Skip to content

Commit

Permalink
Move safepoint emission to llvm-final-gc-lowering (JuliaLang#47393)
Browse files Browse the repository at this point in the history
  • Loading branch information
@vchuravy @kpamnany
vchuravy authored and kpamnany committed Sep 28, 2023
1 parent 11bb88d commit f7dfc94
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 25 deletions.
48 changes: 34 additions & 14 deletions src/codegen_shared.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <utility>
#include <llvm/ADT/ArrayRef.h>
#include <llvm/Support/Debug.h>
#include <llvm/IR/Attributes.h>
#include <llvm/IR/DebugLoc.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/MDBuilder.h>
Expand Down Expand Up @@ -233,20 +234,39 @@ static inline void emit_signal_fence(llvm::IRBuilder<> &builder)
builder.CreateFence(AtomicOrdering::SequentiallyConsistent, SyncScope::SingleThread);
}

static inline void emit_gc_safepoint(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::MDNode *tbaa)
static inline void emit_gc_safepoint(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::MDNode *tbaa, bool final = false)
{
using namespace llvm;
llvm::Value *signal_page = get_current_signal_page_from_ptls(builder, ptls, tbaa);
emit_signal_fence(builder);
builder.CreateLoad(getSizeTy(builder.getContext()), get_current_signal_page_from_ptls(builder, ptls, tbaa), true);
Module *M = builder.GetInsertBlock()->getModule();
LLVMContext &C = builder.getContext();
// inline jlsafepoint_func->realize(M)
if (final) {
auto T_size = getSizeTy(builder.getContext());
builder.CreateLoad(T_size, signal_page, true);
}
else {
Function *F = M->getFunction("julia.safepoint");
if (!F) {
auto T_size = getSizeTy(builder.getContext());
auto T_psize = T_size->getPointerTo();
FunctionType *FT = FunctionType::get(Type::getVoidTy(C), {T_psize}, false);
F = Function::Create(FT, Function::ExternalLinkage, "julia.safepoint", M);
F->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
}
builder.CreateCall(F, {signal_page});
}
emit_signal_fence(builder);
}

static inline llvm::Value *emit_gc_state_set(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::Value *state, llvm::Value *old_state)
static inline llvm::Value *emit_gc_state_set(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::Value *state, llvm::Value *old_state, bool final)
{
using namespace llvm;
Type *T_int8 = state->getType();
ptls = emit_bitcast_with_builder(builder, ptls, builder.getInt8PtrTy());
llvm::Value *ptls_i8 = emit_bitcast_with_builder(builder, ptls, builder.getInt8PtrTy());
Constant *offset = ConstantInt::getSigned(builder.getInt32Ty(), offsetof(jl_tls_states_t, gc_state));
Value *gc_state = builder.CreateInBoundsGEP(T_int8, ptls, ArrayRef<Value*>(offset), "gc_state");
Value *gc_state = builder.CreateInBoundsGEP(T_int8, ptls_i8, ArrayRef<Value*>(offset), "gc_state");
if (old_state == nullptr) {
old_state = builder.CreateLoad(T_int8, gc_state);
cast<LoadInst>(old_state)->setOrdering(AtomicOrdering::Monotonic);
Expand All @@ -266,38 +286,38 @@ static inline llvm::Value *emit_gc_state_set(llvm::IRBuilder<> &builder, llvm::V
passBB, exitBB);
builder.SetInsertPoint(passBB);
MDNode *tbaa = get_tbaa_const(builder.getContext());
emit_gc_safepoint(builder, ptls, tbaa);
emit_gc_safepoint(builder, ptls, tbaa, final);
builder.CreateBr(exitBB);
builder.SetInsertPoint(exitBB);
return old_state;
}

static inline llvm::Value *emit_gc_unsafe_enter(llvm::IRBuilder<> &builder, llvm::Value *ptls)
static inline llvm::Value *emit_gc_unsafe_enter(llvm::IRBuilder<> &builder, llvm::Value *ptls, bool final)
{
using namespace llvm;
Value *state = builder.getInt8(0);
return emit_gc_state_set(builder, ptls, state, nullptr);
return emit_gc_state_set(builder, ptls, state, nullptr, final);
}

static inline llvm::Value *emit_gc_unsafe_leave(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::Value *state)
static inline llvm::Value *emit_gc_unsafe_leave(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::Value *state, bool final)
{
using namespace llvm;
Value *old_state = builder.getInt8(0);
return emit_gc_state_set(builder, ptls, state, old_state);
return emit_gc_state_set(builder, ptls, state, old_state, final);
}

static inline llvm::Value *emit_gc_safe_enter(llvm::IRBuilder<> &builder, llvm::Value *ptls)
static inline llvm::Value *emit_gc_safe_enter(llvm::IRBuilder<> &builder, llvm::Value *ptls, bool final)
{
using namespace llvm;
Value *state = builder.getInt8(JL_GC_STATE_SAFE);
return emit_gc_state_set(builder, ptls, state, nullptr);
return emit_gc_state_set(builder, ptls, state, nullptr, final);
}

static inline llvm::Value *emit_gc_safe_leave(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::Value *state)
static inline llvm::Value *emit_gc_safe_leave(llvm::IRBuilder<> &builder, llvm::Value *ptls, llvm::Value *state, bool final)
{
using namespace llvm;
Value *old_state = builder.getInt8(JL_GC_STATE_SAFE);
return emit_gc_state_set(builder, ptls, state, old_state);
return emit_gc_state_set(builder, ptls, state, old_state, final);
}

// Compatibility shims for LLVM attribute APIs that were renamed in LLVM 14.
Expand Down
31 changes: 28 additions & 3 deletions src/llvm-final-gc-lowering.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ STATISTIC(GetGCFrameSlotCount, "Number of lowered getGCFrameSlotFunc intrinsics"
STATISTIC(GCAllocBytesCount, "Number of lowered GCAllocBytesFunc intrinsics");
STATISTIC(QueueGCRootCount, "Number of lowered queueGCRootFunc intrinsics");
STATISTIC(QueueGCBindingCount, "Number of lowered queueGCBindingFunc intrinsics");
STATISTIC(SafepointCount, "Number of lowered safepoint intrinsics");

using namespace llvm;

Expand Down Expand Up @@ -72,6 +73,9 @@ struct FinalLowerGC: private JuliaPassContext {

// Lowers a `julia.queue_gc_binding` intrinsic.
Value *lowerQueueGCBinding(CallInst *target, Function &F);

// Lowers a `julia.safepoint` intrinsic.
Value *lowerSafepoint(CallInst *target, Function &F);
};

Value *FinalLowerGC::lowerNewGCFrame(CallInst *target, Function &F)
Expand Down Expand Up @@ -202,6 +206,18 @@ Value *FinalLowerGC::lowerQueueGCBinding(CallInst *target, Function &F)
return target;
}

Value *FinalLowerGC::lowerSafepoint(CallInst *target, Function &F)
{
++SafepointCount;
assert(target->arg_size() == 1);
IRBuilder<> builder(target->getContext());
builder.SetInsertPoint(target);
auto T_size = getSizeTy(builder.getContext());
Value* signal_page = target->getOperand(0);
Value* load = builder.CreateLoad(T_size, signal_page, true);
return load;
}

Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
{
++GCAllocBytesCount;
Expand Down Expand Up @@ -317,16 +333,20 @@ static void replaceInstruction(

bool FinalLowerGC::runOnFunction(Function &F)
{
LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Processing function " << F.getName() << "\n");
// Check availability of functions again since they might have been deleted.
initFunctions(*F.getParent());
if (!pgcstack_getter && !adoptthread_func)
if (!pgcstack_getter && !adoptthread_func) {
LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Skipping function " << F.getName() << "\n");
return false;
}

// Look for a call to 'julia.get_pgcstack'.
pgcstack = getPGCstack(F);
if (!pgcstack)
if (!pgcstack) {
LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Skipping function " << F.getName() << " no pgcstack\n");
return false;
}
LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Processing function " << F.getName() << "\n");

// Acquire intrinsic functions.
auto newGCFrameFunc = getOrNull(jl_intrinsics::newGCFrame);
Expand All @@ -336,6 +356,7 @@ bool FinalLowerGC::runOnFunction(Function &F)
auto GCAllocBytesFunc = getOrNull(jl_intrinsics::GCAllocBytes);
auto queueGCRootFunc = getOrNull(jl_intrinsics::queueGCRoot);
auto queueGCBindingFunc = getOrNull(jl_intrinsics::queueGCBinding);
auto safepointFunc = getOrNull(jl_intrinsics::safepoint);

// Lower all calls to supported intrinsics.
for (BasicBlock &BB : F) {
Expand All @@ -347,6 +368,7 @@ bool FinalLowerGC::runOnFunction(Function &F)
}

Value *callee = CI->getCalledOperand();
assert(callee);

if (callee == newGCFrameFunc) {
replaceInstruction(CI, lowerNewGCFrame(CI, F), it);
Expand All @@ -370,6 +392,9 @@ bool FinalLowerGC::runOnFunction(Function &F)
}
else if (callee == queueGCBindingFunc) {
replaceInstruction(CI, lowerQueueGCBinding(CI, F), it);
else if (callee == safepointFunc) {
lowerSafepoint(CI, F);
it = CI->eraseFromParent();
}
else {
++it;
Expand Down
22 changes: 16 additions & 6 deletions src/llvm-pass-helpers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,12 +119,7 @@ namespace jl_intrinsics {
static const char *POP_GC_FRAME_NAME = "julia.pop_gc_frame";
static const char *QUEUE_GC_ROOT_NAME = "julia.queue_gc_root";
static const char *QUEUE_GC_BINDING_NAME = "julia.queue_gc_binding";

static auto T_size_t(const JuliaPassContext &context) {
return sizeof(size_t) == sizeof(uint32_t) ?
Type::getInt32Ty(context.getLLVMContext()) :
Type::getInt64Ty(context.getLLVMContext());
}
static const char *SAFEPOINT_NAME = "julia.safepoint";

// Annotates a function with attributes suitable for GC allocation
// functions. Specifically, the return value is marked noalias and nonnull.
Expand Down Expand Up @@ -228,6 +223,21 @@ namespace jl_intrinsics {
QUEUE_GC_BINDING_NAME);
intrinsic->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
return intrinsic;

const IntrinsicDescription safepoint(
SAFEPOINT_NAME,
[](const JuliaPassContext &context) {
auto T_size = getSizeTy(context.getLLVMContext());
auto T_psize = T_size->getPointerTo();
auto intrinsic = Function::Create(
FunctionType::get(
Type::getVoidTy(context.getLLVMContext()),
{T_psize},
false),
Function::ExternalLinkage,
SAFEPOINT_NAME);
intrinsic->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
return intrinsic;
});
}

Expand Down
3 changes: 3 additions & 0 deletions src/llvm-pass-helpers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,9 @@ namespace jl_intrinsics {

// `julia.queue_gc_binding`: an intrinsic that queues a binding for GC.
extern const IntrinsicDescription queueGCBinding;

// `julia.safepoint`: an intrinsic that triggers a GC safepoint.
extern const IntrinsicDescription safepoint;
}

// A namespace for well-known Julia runtime function descriptions.
Expand Down
4 changes: 2 additions & 2 deletions src/llvm-ptls.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter,
IRBuilder<> builder(fastTerm->getParent());
fastTerm->removeFromParent();
MDNode *tbaa = tbaa_gcframe;
Value *prior = emit_gc_unsafe_enter(builder, get_current_ptls_from_task(builder, get_current_task_from_pgcstack(builder, pgcstack), tbaa));
Value *prior = emit_gc_unsafe_enter(builder, get_current_ptls_from_task(builder, get_current_task_from_pgcstack(builder, pgcstack), tbaa), true);
builder.Insert(fastTerm);
phi->addIncoming(pgcstack, fastTerm->getParent());
// emit pre-return cleanup
Expand All @@ -219,7 +219,7 @@ void LowerPTLS::fix_pgcstack_use(CallInst *pgcstack, Function *pgcstack_getter,
for (auto &BB : *pgcstack->getParent()->getParent()) {
if (isa<ReturnInst>(BB.getTerminator())) {
IRBuilder<> builder(BB.getTerminator());
emit_gc_unsafe_leave(builder, get_current_ptls_from_task(builder, get_current_task_from_pgcstack(builder, phi), tbaa), last_gc_state);
emit_gc_unsafe_leave(builder, get_current_ptls_from_task(builder, get_current_task_from_pgcstack(builder, phi), tbaa), last_gc_state, true);
}
}
}
Expand Down

0 comments on commit f7dfc94

Please sign in to comment.