Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

1.11: allow external abstract interpreter compilation #53488

Open
wants to merge 1 commit into
base: release-1.11
Choose a base branch
from
+214 −179
Open

1.11: allow external abstract interpreter compilation #53488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion base/compiler/typeinfer.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ function CodeInstance(interp::AbstractInterpreter, result::InferenceResult,
t = @_gc_preserve_begin inferred_result
relocatability = unsafe_load(unsafe_convert(Ptr{UInt8}, inferred_result), Core.sizeof(inferred_result))
@_gc_preserve_end t
elseif inferred_result === nothing
elseif !(inferred_result isa CodeInfo)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the actual fix. The other changes are the refinements on the test cases (which were done in #53478).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm but on master we still have

julia/base/compiler/typeinfer.jl

Line 331 in b3b2736

elseif inferred_result === nothing

So I am not sure how it would be fixed there. IIRC even on master I needed a abs int overload that set relocatability?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The relocatability indicates that we know the inferred field is safe to duplicate, which doesn't seem would be the case if this is not one of the expected objects? (i.e. String or Nothing)

relocatability = 0x1
end
end
Expand Down
185 changes: 7 additions & 178 deletions test/precompile.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

original_depot_path = copy(Base.DEPOT_PATH)
original_load_path = copy(Base.LOAD_PATH)

using Test, Distributed, Random, Logging
using REPL # doc lookup function

include("precompile_utils.jl")

Foo_module = :Foo4b3a94a1a081a8cb
Foo2_module = :F2oo4b3a94a1a081a8cb
FooBase_module = :FooBase4b3a94a1a081a8cb
Expand All @@ -16,37 +15,6 @@ FooBase_module = :FooBase4b3a94a1a081a8cb
end
using .ConflictingBindings

function precompile_test_harness(@nospecialize(f), testset::String)
@testset "$testset" begin
precompile_test_harness(f, true)
end
end
function precompile_test_harness(@nospecialize(f), separate::Bool)
load_path = mktempdir()
load_cache_path = separate ? mktempdir() : load_path
try
pushfirst!(LOAD_PATH, load_path)
pushfirst!(DEPOT_PATH, load_cache_path)
f(load_path)
finally
try
rm(load_path, force=true, recursive=true)
catch err
@show err
end
if separate
try
rm(load_cache_path, force=true, recursive=true)
catch err
@show err
end
end
filter!((≠)(load_path), LOAD_PATH)
separate && filter!((≠)(load_cache_path), DEPOT_PATH)
end
nothing
end

# method root provenance

rootid(m::Module) = Base.module_build_id(Base.parentmodule(m)) % UInt64
Expand Down Expand Up @@ -1712,146 +1680,10 @@ precompile_test_harness("issue #46296") do load_path
(@eval (using CodeInstancePrecompile))
end

let newinterp_path = abspath("compiler/newinterp.jl")
precompile_test_harness("AbstractInterpreter caching") do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

write(joinpath(load_path, "CustomAbstractInterpreterCaching.jl"), :(module CustomAbstractInterpreterCaching
import SimpleModule: basic_caller, basic_callee

module Custom
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching"))
@eval let
using CustomAbstractInterpreterCaching
cache_owner = Core.Compiler.cache_owner(
CustomAbstractInterpreterCaching.Custom.PrecompileInterpreter())
let m = only(methods(CustomAbstractInterpreterCaching.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi in Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end

write(joinpath(load_path, "CustomAbstractInterpreterCaching2.jl"), :(module CustomAbstractInterpreterCaching2
import SimpleModule: basic_caller, basic_callee

module Custom
const CC = Core.Compiler
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
struct CustomData
inferred
CustomData(@nospecialize inferred) = new(inferred)
end
function CC.transform_result_for_cache(interp::PrecompileInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
return CustomData(inferred_result)
end
function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src),
@nospecialize(info::CC.CallInfo), stmt_flag::UInt32)
if src isa CustomData
src = src.inferred
end
return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any,
info::CC.CallInfo, stmt_flag::UInt32)
end
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("CustomAbstractInterpreterCaching2"))
@eval let
using CustomAbstractInterpreterCaching2
cache_owner = Core.Compiler.cache_owner(
CustomAbstractInterpreterCaching2.Custom.PrecompileInterpreter())
let m = only(methods(CustomAbstractInterpreterCaching2.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi = Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end
@testset "Precompile external abstract interpreter" begin
dir = @__DIR__
@test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint1.jl`; dir); stdout, stderr))
@test success(pipeline(Cmd(`$(Base.julia_cmd()) precompile_absint2.jl`; dir); stdout, stderr))
end

precompile_test_harness("Recursive types") do load_path
Expand Down Expand Up @@ -2093,7 +1925,4 @@ precompile_test_harness("Test flags") do load_path
@test !Base.isprecompiled(id, ;flags=current_flags)
end

empty!(Base.DEPOT_PATH)
append!(Base.DEPOT_PATH, original_depot_path)
empty!(Base.LOAD_PATH)
append!(Base.LOAD_PATH, original_load_path)
finish_precompile_test!()
73 changes: 73 additions & 0 deletions test/precompile_absint1.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test

include("precompile_utils.jl")

precompile_test_harness() do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

newinterp_path = abspath("compiler/newinterp.jl")
write(joinpath(load_path, "TestAbsIntPrecompile1.jl"), :(module TestAbsIntPrecompile1
import SimpleModule: basic_caller, basic_callee

module Custom
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("TestAbsIntPrecompile1"))

@eval let
using TestAbsIntPrecompile1
cache_owner = Core.Compiler.cache_owner(
TestAbsIntPrecompile1.Custom.PrecompileInterpreter())
let m = only(methods(TestAbsIntPrecompile1.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi in Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end

finish_precompile_test!()
92 changes: 92 additions & 0 deletions test/precompile_absint2.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

using Test

include("precompile_utils.jl")

precompile_test_harness() do load_path
write(joinpath(load_path, "SimpleModule.jl"), :(module SimpleModule
basic_callee(x) = x
basic_caller(x) = basic_callee(x)
end) |> string)

newinterp_path = abspath("compiler/newinterp.jl")
write(joinpath(load_path, "TestAbsIntPrecompile2.jl"), :(module TestAbsIntPrecompile2
import SimpleModule: basic_caller, basic_callee

module Custom
const CC = Core.Compiler
include("$($newinterp_path)")
@newinterp PrecompileInterpreter
struct CustomData
inferred
CustomData(@nospecialize inferred) = new(inferred)
end
function CC.transform_result_for_cache(interp::PrecompileInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
inferred_result = @invoke CC.transform_result_for_cache(interp::CC.AbstractInterpreter,
mi::Core.MethodInstance, valid_worlds::CC.WorldRange, result::CC.InferenceResult)
return CustomData(inferred_result)
end
function CC.inlining_policy(interp::PrecompileInterpreter, @nospecialize(src),
@nospecialize(info::CC.CallInfo), stmt_flag::UInt32)
if src isa CustomData
src = src.inferred
end
return @invoke CC.inlining_policy(interp::CC.AbstractInterpreter, src::Any,
info::CC.CallInfo, stmt_flag::UInt32)
end
end

Base.return_types((Float64,)) do x
basic_caller(x)
end
Base.return_types((Float64,); interp=Custom.PrecompileInterpreter()) do x
basic_caller(x)
end
Base.return_types((Vector{Float64},)) do x
sum(x)
end
Base.return_types((Vector{Float64},); interp=Custom.PrecompileInterpreter()) do x
sum(x)
end
end) |> string)
Base.compilecache(Base.PkgId("TestAbsIntPrecompile2"))

@eval let
using TestAbsIntPrecompile2
cache_owner = Core.Compiler.cache_owner(
TestAbsIntPrecompile2.Custom.PrecompileInterpreter())
let m = only(methods(TestAbsIntPrecompile2.basic_callee))
mi = only(Base.specializations(m))
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
end
let m = only(methods(sum, (Vector{Float64},)))
found = false
for mi = Base.specializations(m)
if mi isa Core.MethodInstance && mi.specTypes == Tuple{typeof(sum),Vector{Float64}}
ci = mi.cache
@test isdefined(ci, :next)
@test ci.owner === cache_owner
@test ci.max_world == typemax(UInt)
ci = ci.next
@test !isdefined(ci, :next)
@test ci.owner === nothing
@test ci.max_world == typemax(UInt)
found = true
break
end
end
@test found
end
end
end

finish_precompile_test!()
Loading