Skip to content

Commit

Permalink
1.11: allow external abstract interpreter compilation
Browse files Browse the repository at this point in the history
As mentioned in #53478, the precompilation support for external abstract
interpreters in v1.11 isn't perfect, and directly cherry-picking the
refined test cases from #53478 into the v1.11 backport branch leads to
a test failure (note that this particular problem has likely been fixed
in the master branch, probably thanks to #53300). To address this, this
commit does more than just cherry-pick the test case, and it also
modifies the `CodeInstance(::AbstractInterpreter, ::InferenceResult)`
constructor to allow precompilation for external abstract interpreters
in v1.11.
  • Loading branch information
aviatesk committed Feb 27, 2024
1 parent 0f0f8eb commit e9e2f2f
Show file tree
Hide file tree
Showing 5 changed files with 214 additions and 179 deletions.
2 changes: 1 addition & 1 deletion base/compiler/typeinfer.jl
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ function CodeInstance(interp::AbstractInterpreter, result::InferenceResult,
t = @_gc_preserve_begin inferred_result
relocatability = unsafe_load(unsafe_convert(Ptr{UInt8}, inferred_result), Core.sizeof(inferred_result))
@_gc_preserve_end t
elseif inferred_result === nothing
elseif !(inferred_result isa CodeInfo)
relocatability = 0x1
end
end
Expand Down
185 changes: 7 additions & 178 deletions test/precompile.jl
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

original_depot_path = copy(Base.DEPOT_PATH)
original_load_path = copy(Base.LOAD_PATH)

using Test, Distributed, Random, Logging
using REPL # doc lookup function

include("precompile_utils.jl")

Foo_module = :Foo4b3a94a1a081a8cb
Foo2_module = :F2oo4b3a94a1a081a8cb
FooBase_module = :FooBase4b3a94a1a081a8cb
Expand All @@ -16,37 +15,6 @@ FooBase_module = :FooBase4b3a94a1a081a8cb
end
using .ConflictingBindings

function precompile_test_harness(@nospecialize(f), testset::String)
@testset "$testset" begin
precompile_test_harness(f, true)
end
end
function precompile_test_harness(@nospecialize(f), separate::Bool)
load_path = mktempdir()
load_cache_path = separate ? mktempdir() : load_path
try
pushfirst!(LOAD_PATH, load_path)
pushfirst!(DEPOT_PATH, load_cache_path)
f(load_path)
finally
try
rm(load_path, force=true, recursive=true)
catch err
@show err
end
if separate
try
rm(load_cache_path, force=true, recursive=true)
catch err
@show err
end
end
filter!(()(load_path), LOAD_PATH)
separate && filter!(()(load_cache_path), DEPOT_PATH)
end
nothing
end

# method root provenance

rootid(m::Module) = Base.module_build_id(Base.parentmodule(m)) % UInt64
Expand Down Expand Up @@ -1712,146 +1680,10 @@ precompile_test_harness("issue #46296") do load_path
(@eval (using CodeInstancePrecompile))
end

let newinterp_path = abspath("compiler/newinterp.jl")
precompile_test_harness("AbstractInterpreter caching") do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

write(joinpath(load_path, "CustomAbstractInterpreterCaching.jl"), :(module CustomAbstractInterpreterCaching
import SimpleModule: basic_caller, basic_callee

module Custom
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching"))
@eval let
using CustomAbstractInterpreterCaching
cache_owner = Core.Compiler.cache_owner(
CustomAbstractInterpreterCaching.Custom.PrecompileInterpreter())
let m = only(methods(CustomAbstractInterpreterCaching.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi in Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end

write(joinpath(load_path, "CustomAbstractInterpreterCaching2.jl"), :(module CustomAbstractInterpreterCaching2
import SimpleModule: basic_caller, basic_callee

module Custom
const CC = Core.Compiler
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
struct CustomData
inferred
CustomData(@nospecialize inferred) = new(inferred)
end
function CC.transform_result_for_cache(interp::PrecompileInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
return CustomData(inferred_result)
end
function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src),
@nospecialize(info::CC.CallInfo), stmt_flag::UInt32)
if src isa CustomData
src = src.inferred
end
return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any,
info::CC.CallInfo, stmt_flag::UInt32)
end
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching2"))
@eval let
using CustomAbstractInterpreterCaching2
cache_owner = Core.Compiler.cache_owner(
CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter())
let m = only(methods(CustomAbstractInterpreterCaching2.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi = Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end
@testset "Precompile external abstract interpreter" begin
dir = @__DIR__
@test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint1.jl`; dir); stdout, stderr))
@test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint2.jl`; dir); stdout, stderr))
end

precompile_test_harness("Recursive types") do load_path
Expand Down Expand Up @@ -2093,7 +1925,4 @@ precompile_test_harness("Test flags") do load_path
@test !Base.isprecompiled(id, ;flags=current_flags)
end

empty!(Base.DEPOT_PATH)
append!(Base.DEPOT_PATH, original_depot_path)
empty!(Base.LOAD_PATH)
append!(Base.LOAD_PATH, original_load_path)
finish_precompile_test!()
73 changes: 73 additions & 0 deletions test/precompile_absint1.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test

include("precompile_utils.jl")

precompile_test_harness() do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

newinterp_path = abspath("compiler/newinterp.jl")
write(joinpath(load_path, "TestAbsIntPrecompile1.jl"), :(module TestAbsIntPrecompile1
import SimpleModule: basic_caller, basic_callee

module Custom
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("TestAbsIntPrecompile1"))

@eval let
using TestAbsIntPrecompile1
cache_owner = Core.Compiler.cache_owner(
TestAbsIntPrecompile1.Custom.PrecompileInterpreter())
let m = only(methods(TestAbsIntPrecompile1.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi in Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end

finish_precompile_test!()
92 changes: 92 additions & 0 deletions test/precompile_absint2.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test

include("precompile_utils.jl")

precompile_test_harness() do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

newinterp_path = abspath("compiler/newinterp.jl")
write(joinpath(load_path, "TestAbsIntPrecompile2.jl"), :(module TestAbsIntPrecompile2
import SimpleModule: basic_caller, basic_callee

module Custom
const CC = Core.Compiler
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
struct CustomData
inferred
CustomData(@nospecialize inferred) = new(inferred)
end
function CC.transform_result_for_cache(interp::PrecompileInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
return CustomData(inferred_result)
end
function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src),
@nospecialize(info::CC.CallInfo), stmt_flag::UInt32)
if src isa CustomData
src = src.inferred
end
return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any,
info::CC.CallInfo, stmt_flag::UInt32)
end
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("TestAbsIntPrecompile2"))

@eval let
using TestAbsIntPrecompile2
cache_owner = Core.Compiler.cache_owner(
TestAbsIntPrecompile2.Custom.PrecompileInterpreter())
let m = only(methods(TestAbsIntPrecompile2.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi = Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end

finish_precompile_test!()
Loading

0 comments on commit e9e2f2f

Please sign in to comment.