diff --git a/src/codegen.cpp b/src/codegen.cpp index a928fdd7e207b..1f9ad2b951c1e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -8100,7 +8100,12 @@ extern "C" void jl_init_llvm(void) } // Allocate a target... Optional codemodel = -#ifdef _P64 +#if defined(JL_USE_JITLINK) + // JITLink can patch up relocations between far objects so we can use the + // small code model – which is good, as the large code model is unmaintained + // on MachO/AArch64. + CodeModel::Small; +#elif defined(_P64) // Make sure we are using the large code model on 64bit // Let LLVM pick a default suitable for jitting on 32bit CodeModel::Large; @@ -8142,13 +8147,15 @@ extern "C" void jl_init_llvm(void) } #endif if (jl_using_gdb_jitevents) - jl_ExecutionEngine->RegisterJITEventListener(JITEventListener::createGDBRegistrationListener()); + jl_ExecutionEngine->enableJITDebuggingSupport(); #if defined(JL_USE_INTEL_JITEVENTS) || \ defined(JL_USE_OPROFILE_JITEVENTS) || \ defined(JL_USE_PERF_JITEVENTS) +#ifdef JL_USE_JITLINK +#error "JIT profiling support (JL_USE_*_JITEVENTS) not yet available on platforms that use JITLink" +#else const char *jit_profiling = getenv("ENABLE_JITPROFILING"); -#endif #if defined(JL_USE_INTEL_JITEVENTS) if (jit_profiling && atoi(jit_profiling)) { @@ -8181,6 +8188,8 @@ extern "C" void jl_init_llvm(void) #ifdef JL_USE_PERF_JITEVENTS if (jl_using_perf_jitevents) jl_ExecutionEngine->RegisterJITEventListener(JITEventListener::createPerfJITEventListener()); +#endif +#endif #endif cl::PrintOptionValues(); diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index f0af463f2a131..bb1fcd88bb73d 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -113,12 +113,6 @@ void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const D } -#ifdef _OS_WINDOWS_ -#if defined(_CPU_X86_64_) -void *lookupWriteAddressFor(RTDyldMemoryManager *memmgr, void *rt_addr); -#endif -#endif - #if defined(_OS_WINDOWS_) static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnname, uint8_t *Section, size_t Allocated, uint8_t *UnwindData) @@ -177,7 +171,14 @@ struct revcomp { }; -class JuliaJITEventListener +// Central registry for resolving function addresses to `jl_method_instance_t`s and +// originating `ObjectFile`s (for the DWARF debug info). +// +// A global singleton instance is notified by the JIT whenever a new object is emitted, +// and later queried by the various function info APIs. We also use the chance to handle +// some platform-specific unwind info registration (which is unrelated to the query +// functionality). +class JITObjectRegistry { std::map objectmap; std::map, revcomp> linfomap; @@ -194,44 +195,16 @@ class JuliaJITEventListener return linfo; } - void NotifyObjectEmitted(const object::ObjectFile &Object, - const RuntimeDyld::LoadedObjectInfo &L, - RTDyldMemoryManager *memmgr) + void registerJITObject(const object::ObjectFile &Object, + std::function getLoadAddress, + std::function lookupWriteAddress) { jl_ptls_t ptls = jl_current_task->ptls; // This function modify codeinst->fptr in GC safe region. // This should be fine since the GC won't scan this field. int8_t gc_state = jl_gc_safe_enter(ptls); - auto SavedObject = L.getObjectForDebug(Object).takeBinary(); - // If the debug object is unavailable, save (a copy of) the original object - // for our backtraces. - // This copy seems unfortunate, but there doesn't seem to be a way to take - // ownership of the original buffer. - if (!SavedObject.first) { - auto NewBuffer = MemoryBuffer::getMemBufferCopy( - Object.getData(), Object.getFileName()); - auto NewObj = cantFail(object::ObjectFile::createObjectFile(NewBuffer->getMemBufferRef())); - SavedObject = std::make_pair(std::move(NewObj), std::move(NewBuffer)); - } - const object::ObjectFile &debugObj = *SavedObject.first.release(); - SavedObject.second.release(); - - object::section_iterator EndSection = debugObj.section_end(); - StringMap loadedSections; - for (const object::SectionRef &lSection: Object.sections()) { - auto sName = lSection.getName(); - if (sName) { - bool inserted = loadedSections.insert(std::make_pair(*sName, lSection)).second; - assert(inserted); (void)inserted; - } - } - auto getLoadAddress = [&] (const StringRef &sName) -> uint64_t { - auto search = loadedSections.find(sName); - if (search == loadedSections.end()) - return 0; - return L.getSectionLoadAddress(search->second); - }; + object::section_iterator EndSection = Object.section_end(); #ifdef _CPU_ARM_ // ARM does not have/use .eh_frame @@ -290,7 +263,7 @@ class JuliaJITEventListener uint8_t *UnwindData = NULL; #if defined(_CPU_X86_64_) uint8_t *catchjmp = NULL; - for (const object::SymbolRef &sym_iter : debugObj.symbols()) { + for (const object::SymbolRef &sym_iter : Object.symbols()) { StringRef sName = cantFail(sym_iter.getName()); uint8_t **pAddr = NULL; if (sName.equals("__UnwindData")) { @@ -313,9 +286,8 @@ class JuliaJITEventListener SectionAddrCheck = SectionAddr; SectionLoadCheck = SectionLoadAddr; SectionWriteCheck = SectionLoadAddr; - if (memmgr) - SectionWriteCheck = (uintptr_t)lookupWriteAddressFor(memmgr, - (void*)SectionLoadAddr); + if (lookupWriteAddress) + SectionWriteCheck = (uintptr_t)lookupWriteAddress((void*)SectionLoadAddr); Addr += SectionWriteCheck - SectionLoadAddr; *pAddr = (uint8_t*)Addr; } @@ -343,7 +315,7 @@ class JuliaJITEventListener #endif // defined(_OS_X86_64_) #endif // defined(_OS_WINDOWS_) - auto symbols = object::computeSymbolSizes(debugObj); + auto symbols = object::computeSymbolSizes(Object); bool first = true; for (const auto &sym_size : symbols) { const object::SymbolRef &sym_iter = sym_size.first; @@ -380,7 +352,7 @@ class JuliaJITEventListener if (codeinst) linfomap[Addr] = std::make_pair(Size, codeinst->def); if (first) { - ObjectInfo tmp = {&debugObj, + ObjectInfo tmp = {&Object, (size_t)SectionSize, (ptrdiff_t)(SectionAddr - SectionLoadAddr), *Section, @@ -394,22 +366,18 @@ class JuliaJITEventListener jl_gc_safe_leave(ptls, gc_state); } - // must implement if we ever start freeing code - // virtual void NotifyFreeingObject(const ObjectImage &Object) {} - // virtual void NotifyFreeingObject(const object::ObjectFile &Obj) {} - std::map& getObjectMap() JL_NOTSAFEPOINT { return objectmap; } }; -static JuliaJITEventListener jl_jit_events; -JL_DLLEXPORT void ORCNotifyObjectEmitted(const object::ObjectFile &Object, - const RuntimeDyld::LoadedObjectInfo &L, - RTDyldMemoryManager *memmgr) +static JITObjectRegistry jl_jit_object_registry; +void jl_register_jit_object(const object::ObjectFile &Object, + std::function getLoadAddress, + std::function lookupWriteAddress) { - jl_jit_events.NotifyObjectEmitted(Object, L, memmgr); + jl_jit_object_registry.registerJITObject(Object, getLoadAddress, lookupWriteAddress); } // TODO: convert the safe names from aotcomile.cpp:makeSafeName back into symbols @@ -1178,7 +1146,7 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, { int found = 0; uv_rwlock_wrlock(&threadsafe); - std::map &objmap = jl_jit_events.getObjectMap(); + std::map &objmap = jl_jit_object_registry.getObjectMap(); std::map::iterator fit = objmap.lower_bound(fptr); if (symsize) @@ -1212,7 +1180,7 @@ extern "C" JL_DLLEXPORT int jl_getFunctionInfo_impl(jl_frame_t **frames_out, siz int64_t slide; uint64_t symsize; if (jl_DI_for_fptr(pointer, &symsize, &slide, &Section, &context)) { - frames[0].linfo = jl_jit_events.lookupLinfo(pointer); + frames[0].linfo = jl_jit_object_registry.lookupLinfo(pointer); int nf = lookup_pointer(Section, context, frames_out, pointer, slide, true, noInline); return nf; } @@ -1221,7 +1189,7 @@ extern "C" JL_DLLEXPORT int jl_getFunctionInfo_impl(jl_frame_t **frames_out, siz extern "C" jl_method_instance_t *jl_gdblookuplinfo(void *p) JL_NOTSAFEPOINT { - return jl_jit_events.lookupLinfo((size_t)p); + return jl_jit_object_registry.lookupLinfo((size_t)p); } #if (defined(_OS_LINUX_) || defined(_OS_FREEBSD_) || (defined(_OS_DARWIN_) && defined(LLVM_SHLIB))) @@ -1643,7 +1611,7 @@ uint64_t jl_getUnwindInfo_impl(uint64_t dwAddr) { // Might be called from unmanaged thread uv_rwlock_rdlock(&threadsafe); - std::map &objmap = jl_jit_events.getObjectMap(); + std::map &objmap = jl_jit_object_registry.getObjectMap(); std::map::iterator it = objmap.lower_bound(dwAddr); uint64_t ipstart = 0; // ip of the start of the section (if found) if (it != objmap.end() && dwAddr < it->first + it->second.SectionSize) { diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 8a360813e2cb0..97209ca040e6e 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -3,7 +3,6 @@ #include "llvm-version.h" #include "platform.h" - #include "llvm/IR/Mangler.h" #include #include @@ -34,7 +33,17 @@ using namespace llvm; #include "jitlayers.h" #include "julia_assert.h" -RTDyldMemoryManager* createRTDyldMemoryManager(void); +#ifdef JL_USE_JITLINK +# if JL_LLVM_VERSION >= 140000 +# include +# endif +# include +# include +#else +# include +#endif + +#define DEBUG_TYPE "jitlayers" void jl_init_jit(void) { } @@ -431,63 +440,6 @@ jl_value_t *jl_dump_method_asm_impl(jl_method_instance_t *mi, size_t world, return jl_dump_function_asm(F, raw_mc, asm_variant, debuginfo, binary); } -// A simple forwarding class, since OrcJIT v2 needs a unique_ptr, while we have a shared_ptr -class ForwardingMemoryManager : public RuntimeDyld::MemoryManager { -private: - std::shared_ptr MemMgr; - -public: - ForwardingMemoryManager(std::shared_ptr MemMgr) : MemMgr(MemMgr) {} - virtual ~ForwardingMemoryManager() = default; - virtual uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - return MemMgr->allocateCodeSection(Size, Alignment, SectionID, SectionName); - } - virtual uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName, - bool IsReadOnly) override { - return MemMgr->allocateDataSection(Size, Alignment, SectionID, SectionName, IsReadOnly); - } - virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, - uintptr_t RODataSize, - uint32_t RODataAlign, - uintptr_t RWDataSize, - uint32_t RWDataAlign) override { - return MemMgr->reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign, RWDataSize, RWDataAlign); - } - virtual bool needsToReserveAllocationSpace() override { - return MemMgr->needsToReserveAllocationSpace(); - } - virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, - size_t Size) override { - return MemMgr->registerEHFrames(Addr, LoadAddr, Size); - } - virtual void deregisterEHFrames() override { - return MemMgr->deregisterEHFrames(); - } - virtual bool finalizeMemory(std::string *ErrMsg = nullptr) override { - return MemMgr->finalizeMemory(ErrMsg); - } - virtual void notifyObjectLoaded(RuntimeDyld &RTDyld, - const object::ObjectFile &Obj) override { - return MemMgr->notifyObjectLoaded(RTDyld, Obj); - } -}; - - -JL_DLLEXPORT void ORCNotifyObjectEmitted(const object::ObjectFile &obj, - const RuntimeDyld::LoadedObjectInfo &L, - RTDyldMemoryManager *memmgr); - -template -void JuliaOJIT::registerObject(const ObjT &Obj, const LoadResult &LO) -{ - const ObjT* Object = &Obj; - ORCNotifyObjectEmitted(*Object, *LO, MemMgr.get()); -} - CodeGenOpt::Level CodeGenOptLevelFor(int optlevel) { #ifdef DISABLE_OPT @@ -603,11 +555,257 @@ CompilerResultT JuliaOJIT::CompilerT::operator()(Module &M) return CompilerResultT(std::move(ObjBuffer)); } +void jl_register_jit_object(const object::ObjectFile &debugObj, + std::function getLoadAddress, + std::function lookupWriteAddress); + +#ifdef JL_USE_JITLINK + +namespace { + +using namespace llvm::orc; + +struct JITObjectInfo { + std::unique_ptr BackingBuffer; + std::unique_ptr Object; + StringMap SectionLoadAddresses; +}; + +class JLDebuginfoPlugin : public ObjectLinkingLayer::Plugin { + std::map> PendingObjs; + // Resources from distinct MaterializationResponsibilitys can get merged + // after emission, so we can have multiple debug objects per resource key. + std::map>> RegisteredObjs; + +public: + void notifyMaterializing(MaterializationResponsibility &MR, jitlink::LinkGraph &G, + jitlink::JITLinkContext &Ctx, + MemoryBufferRef InputObject) override + { + // Keeping around a full copy of the input object file (and re-parsing it) is + // wasteful, but for now, this lets us reuse the existing debuginfo.cpp code. + // Should look into just directly pulling out all the information required in + // a JITLink pass and just keeping the required tables/DWARF sections around + // (perhaps using the LLVM DebuggerSupportPlugin as a reference). + auto NewBuffer = + MemoryBuffer::getMemBufferCopy(InputObject.getBuffer(), G.getName()); + auto NewObj = + cantFail(object::ObjectFile::createObjectFile(NewBuffer->getMemBufferRef())); + + assert(PendingObjs.count(&MR) == 0); + PendingObjs[&MR] = std::unique_ptr( + new JITObjectInfo{std::move(NewBuffer), std::move(NewObj), {}}); + } + + Error notifyEmitted(MaterializationResponsibility &MR) override + { + auto It = PendingObjs.find(&MR); + if (It == PendingObjs.end()) + return Error::success(); + + auto NewInfo = PendingObjs[&MR].get(); + auto getLoadAddress = [NewInfo](const StringRef &Name) -> uint64_t { + auto result = NewInfo->SectionLoadAddresses.find(Name); + if (result == NewInfo->SectionLoadAddresses.end()) { + LLVM_DEBUG({ + dbgs() << "JLDebuginfoPlugin: No load address found for section '" + << Name << "'\n"; + }); + return 0; + } + return result->second; + }; + + jl_register_jit_object(*NewInfo->Object, getLoadAddress, nullptr); + + cantFail(MR.withResourceKeyDo([&](ResourceKey K) { + RegisteredObjs[K].push_back(std::move(PendingObjs[&MR])); + PendingObjs.erase(&MR); + })); + + return Error::success(); + } + + Error notifyFailed(MaterializationResponsibility &MR) override + { + PendingObjs.erase(&MR); + return Error::success(); + } + + Error notifyRemovingResources(ResourceKey K) override + { + RegisteredObjs.erase(K); + // TODO: If we ever unload code, need to notify debuginfo registry. + return Error::success(); + } + + void notifyTransferringResources(ResourceKey DstKey, ResourceKey SrcKey) override + { + auto SrcIt = RegisteredObjs.find(SrcKey); + if (SrcIt != RegisteredObjs.end()) { + for (std::unique_ptr &Info : SrcIt->second) + RegisteredObjs[DstKey].push_back(std::move(Info)); + RegisteredObjs.erase(SrcIt); + } + } + + void modifyPassConfig(MaterializationResponsibility &MR, jitlink::LinkGraph &, + jitlink::PassConfiguration &PassConfig) override + { + auto It = PendingObjs.find(&MR); + if (It == PendingObjs.end()) + return; + + JITObjectInfo &Info = *It->second; + PassConfig.PostAllocationPasses.push_back([&Info](jitlink::LinkGraph &G) -> Error { + for (const jitlink::Section &Sec : G.sections()) { + // Canonical JITLink section names have the segment name included, e.g. + // "__TEXT,__text" or "__DWARF,__debug_str". There are some special internal + // sections without a comma separator, which we can just ignore. + size_t SepPos = Sec.getName().find(','); + if (SepPos >= 16 || (Sec.getName().size() - (SepPos + 1) > 16)) { + LLVM_DEBUG({ + dbgs() << "JLDebuginfoPlugin: Ignoring section '" << Sec.getName() + << "'\n"; + }); + continue; + } + auto SecName = Sec.getName().substr(SepPos + 1); + Info.SectionLoadAddresses[SecName] = jitlink::SectionRange(Sec).getStart(); + } + return Error::success(); + }); + } +}; +} + +# ifdef LLVM_SHLIB +class JLEHFrameRegistrar final : public jitlink::EHFrameRegistrar { +public: + Error registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) override { + register_eh_frames( + jitTargetAddressToPointer(EHFrameSectionAddr), + EHFrameSectionSize); + return Error::success(); + } + + Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) override { + deregister_eh_frames( + jitTargetAddressToPointer(EHFrameSectionAddr), + EHFrameSectionSize); + return Error::success(); + } +}; +# endif + +#else // !JL_USE_JITLINK + +RTDyldMemoryManager* createRTDyldMemoryManager(void); + +// A simple forwarding class, since OrcJIT v2 needs a unique_ptr, while we have a shared_ptr +class ForwardingMemoryManager : public RuntimeDyld::MemoryManager { +private: + std::shared_ptr MemMgr; + +public: + ForwardingMemoryManager(std::shared_ptr MemMgr) : MemMgr(MemMgr) {} + virtual ~ForwardingMemoryManager() = default; + virtual uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override { + return MemMgr->allocateCodeSection(Size, Alignment, SectionID, SectionName); + } + virtual uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName, + bool IsReadOnly) override { + return MemMgr->allocateDataSection(Size, Alignment, SectionID, SectionName, IsReadOnly); + } + virtual void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, + uintptr_t RODataSize, + uint32_t RODataAlign, + uintptr_t RWDataSize, + uint32_t RWDataAlign) override { + return MemMgr->reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign, RWDataSize, RWDataAlign); + } + virtual bool needsToReserveAllocationSpace() override { + return MemMgr->needsToReserveAllocationSpace(); + } + virtual void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, + size_t Size) override { + return MemMgr->registerEHFrames(Addr, LoadAddr, Size); + } + virtual void deregisterEHFrames() override { + return MemMgr->deregisterEHFrames(); + } + virtual bool finalizeMemory(std::string *ErrMsg = nullptr) override { + return MemMgr->finalizeMemory(ErrMsg); + } + virtual void notifyObjectLoaded(RuntimeDyld &RTDyld, + const object::ObjectFile &Obj) override { + return MemMgr->notifyObjectLoaded(RTDyld, Obj); + } +}; + + +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) +void *lookupWriteAddressFor(RTDyldMemoryManager *MemMgr, void *rt_addr); +#endif + +void registerRTDyldJITObject(const object::ObjectFile &Object, + const RuntimeDyld::LoadedObjectInfo &L, + const std::shared_ptr &MemMgr) +{ + auto SavedObject = L.getObjectForDebug(Object).takeBinary(); + // If the debug object is unavailable, save (a copy of) the original object + // for our backtraces. + // This copy seems unfortunate, but there doesn't seem to be a way to take + // ownership of the original buffer. + if (!SavedObject.first) { + auto NewBuffer = + MemoryBuffer::getMemBufferCopy(Object.getData(), Object.getFileName()); + auto NewObj = + cantFail(object::ObjectFile::createObjectFile(NewBuffer->getMemBufferRef())); + SavedObject = std::make_pair(std::move(NewObj), std::move(NewBuffer)); + } + const object::ObjectFile *DebugObj = SavedObject.first.release(); + SavedObject.second.release(); + + StringMap loadedSections; + // Use the original Object, not the DebugObject, as this is used for the + // RuntimeDyld::LoadedObjectInfo lookup. + for (const object::SectionRef &lSection : Object.sections()) { + auto sName = lSection.getName(); + if (sName) { + bool inserted = loadedSections.insert(std::make_pair(*sName, lSection)).second; + assert(inserted); + (void)inserted; + } + } + auto getLoadAddress = [loadedSections = std::move(loadedSections), + &L](const StringRef &sName) -> uint64_t { + auto search = loadedSections.find(sName); + if (search == loadedSections.end()) + return 0; + return L.getSectionLoadAddress(search->second); + }; + + jl_register_jit_object(*DebugObj, getLoadAddress, +#if defined(_OS_WINDOWS_) && defined(_CPU_X86_64_) + [MemMgr](void *p) { return lookupWriteAddressFor(MemMgr.get(), p); } +#else + nullptr +#endif + ); +} +#endif + JuliaOJIT::JuliaOJIT(TargetMachine &TM, LLVMContext *LLVMCtx) : TM(TM), DL(TM.createDataLayout()), ObjStream(ObjBufferSV), - MemMgr(createRTDyldMemoryManager()), TSCtx(std::unique_ptr(LLVMCtx)), #if JL_LLVM_VERSION >= 130000 ES(cantFail(orc::SelfExecutorProcessControl::Create())), @@ -616,6 +814,16 @@ JuliaOJIT::JuliaOJIT(TargetMachine &TM, LLVMContext *LLVMCtx) #endif GlobalJD(ES.createBareJITDylib("JuliaGlobals")), JD(ES.createBareJITDylib("JuliaOJIT")), +#ifdef JL_USE_JITLINK + // TODO: Port our memory management optimisations to JITLink instead of using the + // default InProcessMemoryManager. +# if JL_LLVM_VERSION < 140000 + ObjectLayer(ES, std::make_unique()), +# else + ObjectLayer(ES, cantFail(jitlink::InProcessMemoryManager::Create())), +# endif +#else + MemMgr(createRTDyldMemoryManager()), ObjectLayer( ES, [this]() { @@ -623,14 +831,29 @@ JuliaOJIT::JuliaOJIT(TargetMachine &TM, LLVMContext *LLVMCtx) return result; } ), +#endif CompileLayer(ES, ObjectLayer, std::make_unique(this)) { +#ifdef JL_USE_JITLINK +# if defined(_OS_DARWIN_) && defined(LLVM_SHLIB) + // When dynamically linking against LLVM, use our custom EH frame registration code + // also used with RTDyld to inform both our and the libc copy of libunwind. + auto ehRegistrar = std::make_unique(); +# else + auto ehRegistrar = std::make_unique(); +# endif + ObjectLayer.addPlugin(std::make_unique( + ES, std::move(ehRegistrar))); + + ObjectLayer.addPlugin(std::make_unique()); +#else ObjectLayer.setNotifyLoaded( [this](orc::MaterializationResponsibility &MR, const object::ObjectFile &Object, - const RuntimeDyld::LoadedObjectInfo &LOS) { - registerObject(Object, &LOS); + const RuntimeDyld::LoadedObjectInfo &LO) { + registerRTDyldJITObject(Object, LO, MemMgr); }); +#endif for (int i = 0; i < 4; i++) { TMs[i] = TM.getTarget().createTargetMachine(TM.getTargetTriple().getTriple(), TM.getTargetCPU(), TM.getTargetFeatureString(), TM.Options, Reloc::Static, TM.getCodeModel(), @@ -694,8 +917,8 @@ void JuliaOJIT::addModule(std::unique_ptr M) NewExports.push_back(getMangledName(F.getName())); } } -#ifndef JL_NDEBUG - // validate the relocations for M +#if !defined(JL_NDEBUG) && !defined(JL_USE_JITLINK) + // validate the relocations for M (not implemented for the JITLink memory manager yet) for (Module::global_object_iterator I = M->global_objects().begin(), E = M->global_objects().end(); I != E; ) { GlobalObject *F = &*I; ++I; @@ -788,12 +1011,38 @@ StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_code_instance_t *cod } +#ifdef JL_USE_JITLINK +# if JL_LLVM_VERSION < 140000 +# warning "JIT debugging (GDB integration) not available on LLVM < 14.0 (for JITLink)" +void JuliaOJIT::enableJITDebuggingSupport() {} +# else +extern "C" orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size); + +void JuliaOJIT::enableJITDebuggingSupport() +{ + // We do not use GDBJITDebugInfoRegistrationPlugin::Create, as the runtime name + // lookup is unnecessarily involved/fragile for our in-process JIT use case + // (with the llvm_orc_registerJITLoaderGDBAllocAction symbol being in either + // libjulia-codegen or yet another shared library for LLVM depending on the build + // flags, etc.). + const auto Addr = ExecutorAddr::fromPtr(&llvm_orc_registerJITLoaderGDBAllocAction); + ObjectLayer.addPlugin(std::make_unique(Addr)); +} +# endif +#else +void JuliaOJIT::enableJITDebuggingSupport() +{ + RegisterJITEventListener(JITEventListener::createGDBRegistrationListener()); +} + void JuliaOJIT::RegisterJITEventListener(JITEventListener *L) { if (!L) return; this->ObjectLayer.registerJITEventListener(*L); } +#endif const DataLayout& JuliaOJIT::getDataLayout() const { @@ -817,12 +1066,20 @@ std::string JuliaOJIT::getMangledName(const GlobalValue *GV) return getMangledName(GV->getName()); } +#ifdef JL_USE_JITLINK +size_t JuliaOJIT::getTotalBytes() const +{ + // TODO: Implement in future custom JITLink memory manager. + return 0; +} +#else size_t getRTDyldMemoryManagerTotalBytes(RTDyldMemoryManager *mm); size_t JuliaOJIT::getTotalBytes() const { return getRTDyldMemoryManagerTotalBytes(MemMgr.get()); } +#endif JuliaOJIT *jl_ExecutionEngine; diff --git a/src/jitlayers.h b/src/jitlayers.h index ffb62e3683fe4..ba3f81fa66997 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -7,13 +7,41 @@ #include "llvm/IR/LegacyPassManager.h" #include -#include #include -#include #include #include "julia_assert.h" +// As of LLVM 13, there are two runtime JIT linker implementations, the older +// RuntimeDyld (used via orc::RTDyldObjectLinkingLayer) and the newer JITLink +// (used via orc::ObjectLinkingLayer). +// +// JITLink is not only more flexible (which isn't of great importance for us, as +// we do only single-threaded in-process codegen), but crucially supports using +// the Small code model, where the linker needs to fix up relocations between +// object files that end up far apart in address space. RuntimeDyld can't do +// that and relies on the Large code model instead, which is broken on +// aarch64-darwin (macOS on ARM64), and not likely to ever be supported there +// (see https://bugs.llvm.org/show_bug.cgi?id=52029). +// +// However, JITLink is a relatively young library and lags behind in platform +// and feature support (e.g. Windows, JITEventListeners for various profilers, +// etc.). Thus, we currently only use JITLink where absolutely required, that is, +// for Mac/aarch64. +#if defined(_OS_DARWIN_) && defined(_CPU_AARCH64_) +# if JL_LLVM_VERSION < 130000 +# warning "On aarch64-darwin, LLVM version >= 13 is required for JITLink; fallback suffers from occasional segfaults" +# endif +# define JL_USE_JITLINK +#endif + +#ifdef JL_USE_JITLINK +# include +#else +# include +# include +#endif + using namespace llvm; extern "C" jl_cgparams_t jl_default_cgparams; @@ -159,13 +187,22 @@ class JuliaOJIT { void registerObject(const ObjT &Obj, const LoadResult &LO); public: +#ifdef JL_USE_JITLINK + typedef orc::ObjectLinkingLayer ObjLayerT; +#else typedef orc::RTDyldObjectLinkingLayer ObjLayerT; +#endif typedef orc::IRCompileLayer CompileLayerT; typedef object::OwningBinary OwningObj; JuliaOJIT(TargetMachine &TM, LLVMContext *Ctx); + void enableJITDebuggingSupport(); +#ifndef JL_USE_JITLINK + // JITLink doesn't support old JITEventListeners (yet). void RegisterJITEventListener(JITEventListener *L); +#endif + void addGlobalMapping(StringRef Name, uint64_t Addr); void addModule(std::unique_ptr M); @@ -193,14 +230,15 @@ class JuliaOJIT { legacy::PassManager PM3; TargetMachine *TMs[4]; MCContext *Ctx; - std::shared_ptr MemMgr; - orc::ThreadSafeContext TSCtx; orc::ExecutionSession ES; orc::JITDylib &GlobalJD; orc::JITDylib &JD; +#ifndef JL_USE_JITLINK + std::shared_ptr MemMgr; +#endif ObjLayerT ObjectLayer; CompileLayerT CompileLayer;