diff --git a/src/Makefile b/src/Makefile index ae46d015acc6e..5f7e50c34b190 100644 --- a/src/Makefile +++ b/src/Makefile @@ -117,8 +117,10 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ # additional dependency links $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h -$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp intrinsics.h cgutils.cpp ccall.cpp jitlayers.cpp abi_*.cpp) +$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp jitlayers.cpp intrinsics.h codegen_internal.h cgutils.cpp ccall.cpp abi_*.cpp) $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h +$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(SRCDIR)/codegen_internal.h +$(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/codegen_internal.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc-debug.c $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $(SRCDIR)/,signals-*.c) diff --git a/src/ccall.cpp b/src/ccall.cpp index 0bf253f03d839..833e54e562030 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -4,26 +4,8 @@ // --- the ccall, cglobal, and llvm intrinsics --- -// keep track of llvmcall declarations -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static std::map llvmcallDecls; -#else -static std::set llvmcallDecls; -#endif - -static std::map libMapGV; -static std::map symMapGV; - -static Value *GetStringPtr(llvm::IRBuilder <> *builder, GlobalValue *GV, const Twine &Name) -{ - Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0); - Value *Args[] = { zero, zero }; -#ifdef LLVM37 - return builder->CreateInBoundsGEP(GV->getValueType(), GV, Args, Name); -#else - return builder->CreateInBoundsGEP(GV, Args, Name); -#endif -} +static StringMap libMapGV; +static StringMap symMapGV; static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, const char *f_name, jl_codectx_t *ctx) { @@ -53,52 +35,47 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons libsym = jl_RTLD_DEFAULT_handle; } else { + std::string name = "ccalllib_"; + name += f_lib; runtime_lib = true; libptrgv = libMapGV[f_lib]; if (libptrgv == NULL) { - libptrgv = new GlobalVariable(imaging_mode ? *shadow_module : *active_module, T_pint8, - false, GlobalVariable::PrivateLinkage, - ConstantPointerNull::get((PointerType*)T_pint8), f_lib); - libMapGV[f_lib] = libptrgv; + libptrgv = new GlobalVariable(*jl_Module, T_pint8, + false, GlobalVariable::ExternalLinkage, + NULL, name); + libMapGV[f_lib] = global_proto(libptrgv); libsym = jl_get_library(f_lib); assert(libsym != NULL); -#ifdef USE_MCJIT - jl_llvm_to_jl_value[libptrgv] = libsym; -#else - *((void**)jl_ExecutionEngine->getPointerToGlobal(libptrgv)) = libsym; -#endif + *(void**)jl_emit_and_add_to_shadow(libptrgv) = libsym; + } + else { + libptrgv = prepare_global(libptrgv); } } if (libsym == NULL) { -#ifdef USE_MCJIT - libsym = (void*)jl_llvm_to_jl_value[libptrgv]; -#else - libsym = *((void**)jl_ExecutionEngine->getPointerToGlobal(libptrgv)); -#endif + libsym = *(void**)jl_get_global(libptrgv); } - assert(libsym != NULL); GlobalVariable *llvmgv = symMapGV[f_name]; - Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); if (llvmgv == NULL) { // MCJIT forces this to have external linkage eventually, so we would clobber // the symbol of the actual function. - std::string name = f_name; - name = "ccall_" + name; - llvmgv = new GlobalVariable(imaging_mode ? *shadow_module : *active_module, T_pvoidfunc, - false, GlobalVariable::PrivateLinkage, - initnul, name); - symMapGV[f_name] = llvmgv; -#ifdef USE_MCJIT - jl_llvm_to_jl_value[llvmgv] = jl_dlsym_e(libsym, f_name); -#else - *((void**)jl_ExecutionEngine->getPointerToGlobal(llvmgv)) = jl_dlsym_e(libsym, f_name); -#endif + std::string name = "ccall_"; + name += f_name; + llvmgv = new GlobalVariable(*jl_Module, T_pvoidfunc, + false, GlobalVariable::ExternalLinkage, + NULL, name); + symMapGV[f_name] = global_proto(llvmgv); + *(void**)jl_emit_and_add_to_shadow(llvmgv) = jl_dlsym_e(libsym, f_name); + } + else { + llvmgv = prepare_global(llvmgv); } BasicBlock *dlsym_lookup = BasicBlock::Create(jl_LLVMContext, "dlsym"), *ccall_bb = BasicBlock::Create(jl_LLVMContext, "ccall"); + Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); builder.CreateCondBr(builder.CreateICmpNE(builder.CreateLoad(llvmgv), initnul), ccall_bb, dlsym_lookup); assert(ctx->f->getParent() != NULL); @@ -598,7 +575,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::stringstream name; name << (ctx->f->getName().str()) << "u" << i++; ir_name = name.str(); - if (builtins_module->getFunction(ir_name) == NULL) + if (jl_Module->getFunction(ir_name) == NULL) break; } @@ -615,9 +592,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::string rstring; llvm::raw_string_ostream rtypename(rstring); rettype->print(rtypename); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) std::map localDecls; -#endif if (decl != NULL) { std::stringstream declarations(jl_string_data(decl)); @@ -625,33 +600,16 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c // parse string line by line std::string declstr; while (std::getline(declarations, declstr, '\n')) { - uint64_t declhash = memhash(declstr.c_str(), declstr.length()); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - auto it = llvmcallDecls.find(declhash); - if (it != llvmcallDecls.end()) { - prepare_call(it->second); - } - else { -#else - if (llvmcallDecls.count(declhash) == 0) { -#endif - // Find name of declaration by searching for '@' - std::string::size_type atpos = declstr.find('@') + 1; - // Find end of declaration by searching for '(' - std::string::size_type bracepos = declstr.find('(', atpos); - // Declaration name is the string between @ and ( - std::string declname = declstr.substr(atpos, bracepos - atpos); - - // Check if declaration already present in module - if(jl_Module->getNamedValue(declname) == NULL) { - ir_stream << "; Declarations\n" << declstr << "\n"; - } - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - localDecls[declhash] = declname; -#else - llvmcallDecls.insert(declhash); -#endif + // Find name of declaration by searching for '@' + std::string::size_type atpos = declstr.find('@') + 1; + // Find end of declaration by searching for '(' + std::string::size_type bracepos = declstr.find('(', atpos); + // Declaration name is the string between @ and ( + std::string declname = declstr.substr(atpos, bracepos - atpos); + + // Check if declaration already present in module + if(jl_Module->getNamedValue(declname) == NULL) { + ir_stream << "; Declarations\n" << declstr << "\n"; } } } @@ -662,9 +620,9 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::string ir_string = ir_stream.str(); #ifdef LLVM36 Module *m = NULL; - bool failed = parseAssemblyInto(llvm::MemoryBufferRef(ir_string,"llvmcall"),*builtins_module,Err); + bool failed = parseAssemblyInto(llvm::MemoryBufferRef(ir_string,"llvmcall"),*jl_Module,Err); if (!failed) - m = builtins_module; + m = jl_Module; #else Module *m = ParseAssemblyString(ir_string.c_str(),jl_Module,Err,jl_LLVMContext); #endif @@ -675,12 +633,8 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c jl_error(stream.str().c_str()); } f = m->getFunction(ir_name); + f->removeFromParent(); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - for (auto it : localDecls) { - llvmcallDecls[it.first] = cast(prepare_call(m->getNamedValue(it.second))); - } -#endif } else { assert(isPtr); @@ -692,13 +646,6 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c it != argtypes.end(); ++it, ++i) assert(*it == f->getFunctionType()->getParamType(i)); -#ifdef USE_MCJIT - if (f->getParent() != active_module) { - FunctionMover mover(active_module,f->getParent()); - f = mover.CloneFunction(f); - } -#endif - //f->dump(); #ifndef LLVM35 if (verifyFunction(*f,PrintMessageAction)) { @@ -1174,8 +1121,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) if (jl_is_tuple_type(fargt) && jl_is_leaf_type(fargt)) { frt = jl_tparam0(frt); JL_TRY { - Value *llvmf = prepare_call( - jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt)); + Value *llvmf = prepare_call(jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt)); // make sure to emit any side-effects that may have been part of the original expression emit_expr(args[4], ctx); emit_expr(args[6], ctx); @@ -1353,8 +1299,8 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } if (needStackRestore) { - stacksave = CallInst::Create(prepare_call(Intrinsic::getDeclaration(builtins_module, - Intrinsic::stacksave))); + stacksave = CallInst::Create(Intrinsic::getDeclaration(jl_Module, + Intrinsic::stacksave)); if (savespot) { #ifdef LLVM38 instList.insertAfter(savespot->getIterator(), (Instruction*)stacksave); @@ -1381,10 +1327,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) result = ret; if (needStackRestore) { assert(stacksave != NULL); - builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, - Intrinsic::stackrestore)), - stacksave); + builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::stackrestore), stacksave); } if (0) { // Enable this to turn on SSPREQ (-fstack-protector) on the function containing this ccall ctx->f->addFnAttr(Attribute::StackProtectReq); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 9a557bedbc56e..ae526dc79e9d4 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4,482 +4,96 @@ // utility procedures used in code generation -template // for GlobalObject's -static T *addComdat(T *G) -{ -#if defined(_OS_WINDOWS_) - if (imaging_mode && !G->isDeclaration()) { -#ifdef LLVM35 - // Add comdat information to make MSVC link.exe happy - Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); - jl_Comdat->setSelectionKind(Comdat::NoDuplicates); - G->setComdat(jl_Comdat); - // add __declspec(dllexport) to everything marked for export - if (G->getLinkage() == GlobalValue::ExternalLinkage) - G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); -#endif - } -#endif - return G; -} - static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) { load_or_store->setMetadata( llvm::LLVMContext::MD_tbaa, md ); return load_or_store; } -// Fixing up references to other modules for MCJIT -std::set pending_globals; static GlobalVariable *prepare_global(GlobalVariable *G) { - pending_globals.insert(G); - return G; + Module *M = jl_builderModule; + GlobalValue *local = M->getNamedValue(G->getName()); + if (!local) { + local = global_proto(G, M); + } + return cast(local); } static llvm::Value *prepare_call(llvm::Value *Callee) { - llvm::Function *F = dyn_cast(Callee); - if (!F) - return Callee; - pending_globals.insert(F); - return Callee; -} - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static GlobalValue *realize_pending_global(Module *M, GlobalValue *G, std::map &FixedGlobals) -{ - if (M == G->getParent() && M != builtins_module) { // can happen during bootstrap - //std::cout << "Skipping " << std::string(G->getName()) << " due to parentage" << std::endl; - return nullptr; - } - // If we come across a function that is still being constructed, - // this use needs to remain pending - if (!M || M == builtins_module) { - pending_globals.insert(G); - //std::cout << "Skipping " << std::string(G->getName()) << " due to construction" << std::endl; - return nullptr; - } - if (!FixedGlobals.count(M)) { - if (GlobalVariable *GV = dyn_cast(G)) { - GlobalVariable *NewGV = M->getGlobalVariable(GV->getName()); - if (!NewGV) { - NewGV = new GlobalVariable(*M, GV->getType()->getElementType(), - GV->isConstant(), GlobalVariable::ExternalLinkage, - NULL, GV->getName(), NULL, GV->getThreadLocalMode(), - GV->getType()->getAddressSpace()); - NewGV->setUnnamedAddr(GV->hasUnnamedAddr()); - // Move over initializer - if (GV->hasInitializer()) { - NewGV->setInitializer(GV->getInitializer()); - GV->setInitializer(nullptr); - } - } - FixedGlobals[M] = NewGV; + if (Function *F = dyn_cast(Callee)) { + Module *M = jl_builderModule; + GlobalValue *local = M->getNamedValue(Callee->getName()); + if (!local) { + local = function_proto(F, M); } - else { - Function *F = cast(G); - //std::cout << "Realizing " << std::string(F->getName()) << std::endl; - //if (!F->getParent()) { - //std::cout << "Skipping" << std::endl; - // return nullptr; - //} - Function *NewF = nullptr; - if (!F->isDeclaration() && F->getParent() == builtins_module) { - // It's a definition. Actually move the function and create a - // declaration in the original module - NewF = F; - F->removeFromParent(); - M->getFunctionList().push_back(F); - Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - active_module); - } - else { - assert(F); - NewF = M->getFunction(F->getName()); - if (!NewF) { - NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - M); - } - } - FixedGlobals[M] = NewF; - } - } - return FixedGlobals[M]; -} - -struct ExprChain { - ConstantExpr *Expr; - unsigned OpNo; - struct ExprChain *Next; -}; - -static void handleUse(Use &Use1, llvm::GlobalValue *G, - std::map &FixedGlobals, - struct ExprChain *Chain, struct ExprChain *ChainEnd) -{ - Instruction *User = dyn_cast(Use1.getUser()); - GlobalValue *GVUser = dyn_cast(Use1.getUser()); - if (!User && !GVUser) { - ConstantExpr *Expr = cast(Use1.getUser()); - Value::use_iterator UI2 = Expr->use_begin(), E2 = Expr->use_end(); - for (; UI2 != E2;) { - Use &Use2 = *UI2; - ++UI2; - struct ExprChain NextChain; - NextChain.Expr = Expr; - NextChain.OpNo = Use1.getOperandNo(); - NextChain.Next = nullptr; - if (ChainEnd) - ChainEnd->Next = &NextChain; - handleUse(Use2,G,FixedGlobals,Chain ? Chain : &NextChain,&NextChain); - } - return; - } - llvm::Module *M = nullptr; - if (User) { - Function *UsedInHere = User->getParent()->getParent(); - assert(UsedInHere); - M = UsedInHere->getParent(); - } else { - assert(GVUser); - M = GVUser->getParent(); - } - llvm::Constant *Replacement = realize_pending_global(M,G,FixedGlobals); - if (!Replacement) - return; - while (Chain) { - Replacement = Chain->Expr->getWithOperandReplaced(Chain->OpNo,Replacement); - Chain->Expr = cast(Replacement); - Chain = Chain->Next; - } - Use1.set(Replacement); -} - -// RAUW, but only for those users which live in a module, and create a module -// specific copy -static void realize_pending_globals() -{ - std::set local_pending_globals; - std::swap(local_pending_globals,pending_globals); - std::map FixedGlobals; - for (auto *G : local_pending_globals) { - Value::use_iterator UI = G->use_begin(), E = G->use_end(); - for (; UI != E;) - handleUse(*(UI++),G,FixedGlobals,nullptr,nullptr); - FixedGlobals.clear(); - } -} - -static void realize_cycle(jl_cyclectx_t *cyclectx) -{ - // These need to be resolved together - for (auto *F : cyclectx->functions) { - F->removeFromParent(); - active_module->getFunctionList().push_back(F); - } - for (auto *CU : cyclectx->CUs) { - NamedMDNode *NMD = active_module->getOrInsertNamedMetadata("llvm.dbg.cu"); - NMD->addOperand(CU); - } -} -#endif - -template -#ifdef LLVM35 -static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) -#else -static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) -#endif -{ - // cast through integer to avoid c++ pedantic warning about casting between - // data and code pointers - void *addr = (void*)(uintptr_t)_addr; -#ifdef LLVM34 - StringRef name = gv->getName(); -#ifdef _OS_WINDOWS_ - std::string imp_name; -#endif -#endif - -#ifdef _OS_WINDOWS_ - // setting JL_DLLEXPORT correctly only matters when building a binary - if (dllimport && imaging_mode) { - assert(gv->getLinkage() == GlobalValue::ExternalLinkage); -#ifdef LLVM35 - // add the __declspec(dllimport) attribute - gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); - // this will cause llvm to rename it, so we do the same - imp_name = Twine("__imp_", name).str(); - name = StringRef(imp_name); -#else - gv->setLinkage(GlobalValue::DLLImportLinkage); -#endif -#if defined(_P64) || defined(LLVM35) - // __imp_ variables are indirection pointers, so use malloc to simulate that - void **imp_addr = (void**)malloc(sizeof(void**)); - *imp_addr = addr; - addr = (void*)imp_addr; -#endif + return local; } -#endif // _OS_WINDOWS_ - -#ifdef USE_ORCJIT - addComdat(gv); - jl_ExecutionEngine->addGlobalMapping(name, addr); -#elif defined(USE_MCJIT) - addComdat(gv); - sys::DynamicLibrary::AddSymbol(name, addr); -#else // USE_MCJIT - jl_ExecutionEngine->addGlobalMapping(gv, addr); -#endif // USE_MCJIT + return Callee; } // --- string constants --- -static std::map stringConstants; - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static GlobalVariable *global_proto(GlobalVariable *G) -{ - GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), - G->isConstant(), GlobalVariable::ExternalLinkage, - NULL, G->getName(), G->getThreadLocalMode()); - return proto; -} -#else -static GlobalVariable *global_proto(GlobalVariable *G) -{ - return G; -} -#endif - -static GlobalVariable *stringConst(const std::string &txt) +static StringMap stringConstants; +static GlobalValue *stringConst(const std::string &txt) { - GlobalVariable *gv = stringConstants[txt]; static int strno = 0; - // in inference, we can not share string constants between - // modules as there might be multiple compiles on the stack - // with calls in between them. - if (gv == NULL) { - std::stringstream ssno; - std::string vname; - ssno << strno; - vname += "_j_str"; - vname += ssno.str(); - gv = new GlobalVariable(*active_module, - ArrayType::get(T_int8, txt.length()+1), - true, - imaging_mode ? GlobalVariable::PrivateLinkage : GlobalVariable::ExternalLinkage, - ConstantDataArray::get(getGlobalContext(), - ArrayRef( - (const unsigned char*)txt.c_str(), - txt.length()+1)), - vname); - gv->setUnnamedAddr(true); - gv = imaging_mode ? gv : prepare_global(global_proto(gv)); - stringConstants[txt] = gv; - strno++; - } - else { - prepare_global(gv); - } + std::stringstream ssno; + ssno << "_j_str" << strno++; + GlobalVariable *gv = new GlobalVariable(*jl_builderModule, + ArrayType::get(T_int8, txt.length()+1), + true, + GlobalVariable::PrivateLinkage, + ConstantDataArray::get(getGlobalContext(), + ArrayRef( + (const unsigned char*)txt.c_str(), + txt.length()+1)), + ssno.str()); + gv->setUnnamedAddr(true); return gv; -} - -// --- Shadow module handling --- - -typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing -static std::map jl_value_to_llvm; -JL_DLLEXPORT std::map jl_llvm_to_jl_value; - -// In imaging mode, cache a fast mapping of Function * to code address -// because this is queried in the hot path -static std::map emitted_function_symtab; - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static Function *function_proto(Function *F) -{ - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName()); - NewF->setAttributes(AttributeSet()); - - // FunctionType does not include any attributes. Copy them over manually - // as codegen may make decisions based on the presence of certain attributes - NewF->copyAttributesFrom(F); - +// GlobalVariable *gv = stringConstants[txt]; +// // in inference, we can not share string constants between +// // modules as there might be multiple compiles on the stack +// // with calls in between them. +// if (gv == NULL) { +// std::stringstream ssno; +// std::string vname; +// ssno << strno; +// vname += "_j_str"; +// vname += ssno.str(); +// gv = new GlobalVariable(*jl_builderModule, +// T_int8, +// true, +// GlobalVariable::ExternalLinkage, +// NULL, +// vname); +// stringConstants[txt] = global_proto(gv); +// jl_sym_t *sym = jl_symbol(txt.c_str()); +// *(jl_sym_t**)jl_emit_and_add_to_shadow(gv, sym) = sym; +// strno++; +// } +// else { +// gv = prepare_global(gv); +// } +// return builder.CreateConstInBoundsGEP1_32(T_int8, +// builder.CreateLoad(T_pint8, ConstantExpr::getBitCast(gv, T_ppint8)), +// sizeof(jl_sym_t)); +} + +static Value *GetStringPtr(llvm::IRBuilder <> *builder, GlobalValue *GV, StringRef Name) +{ + Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0); + Value *Args[] = { zero, zero }; #ifdef LLVM37 - // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway, so clear them again manually - NewF->setPersonalityFn(nullptr); -#endif - - return NewF; -} - -class FunctionMover : public ValueMaterializer -{ -public: - FunctionMover(llvm::Module *dest,llvm::Module *src) : - ValueMaterializer(), VMap(), destModule(dest), srcModule(src), - LazyFunctions(0) - { - } - ValueToValueMapTy VMap; - llvm::Module *destModule; - llvm::Module *srcModule; - std::vector LazyFunctions; - - Function *CloneFunctionProto(Function *F) - { - assert(!F->isDeclaration()); - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - destModule); - LazyFunctions.push_back(F); - VMap[F] = NewF; - return NewF; - } - - void CloneFunctionBody(Function *F) - { - Function *NewF = (Function*)(Value*)VMap[F]; - assert(NewF != NULL); - - Function::arg_iterator DestI = NewF->arg_begin(); - for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { - DestI->setName(I->getName()); // Copy the name over... - VMap[&*I] = &*(DestI++); // Add mapping to VMap - } - - #ifdef LLVM36 - // Clone debug info - Not yet public API - // llvm::CloneDebugInfoMetadata(NewF,F,VMap); - #endif - - SmallVector Returns; - llvm::CloneFunctionInto(NewF,F,VMap,true,Returns,"",NULL,NULL,this); - } - - Function *CloneFunction(Function *F) - { - Function *NewF = (llvm::Function*)MapValue(F,VMap,RF_None,NULL,this); - ResolveLazyFunctions(); - return NewF; - } - - void ResolveLazyFunctions() - { - while (!LazyFunctions.empty()) { - Function *F = LazyFunctions.back(); - LazyFunctions.pop_back(); - - CloneFunctionBody(F); - } - } - - Value *InjectFunctionProto(Function *F) - { - Function *NewF = destModule->getFunction(F->getName()); - if (!NewF) { - NewF = function_proto(F); - destModule->getFunctionList().push_back(NewF); - } - return NewF; - } - -#ifdef LLVM38 - virtual Value *materializeDeclFor(Value *V) + return builder->CreateInBoundsGEP(GV->getValueType(), GV, Args, Name); #else - virtual Value *materializeValueFor (Value *V) + return builder->CreateInBoundsGEP(GV, Args, Name); #endif - { - Function *F = dyn_cast(V); - if (F) { - if (F->isIntrinsic()) { - return destModule->getOrInsertFunction(F->getName(),F->getFunctionType()); - } - if (F->isDeclaration() || F->getParent() != destModule) { - if (F->getName().empty()) - return CloneFunctionProto(F); - Function *shadow = srcModule->getFunction(F->getName()); - if (shadow != NULL && !shadow->isDeclaration()) { - // Not truly external - // Check whether we already emitted it once - if (emitted_function_symtab.find(shadow) != emitted_function_symtab.end()) - return InjectFunctionProto(F); - - Function *oldF = destModule->getFunction(F->getName()); - if (oldF) - return oldF; - -#ifndef USE_ORCJIT - // Also check if this function is pending in any other module - if (jl_ExecutionEngine->FindFunctionNamed(F->getName().data())) - return InjectFunctionProto(F); -#endif - - return CloneFunctionProto(shadow); - } - else if (!F->isDeclaration()) { - return CloneFunctionProto(F); - } - } - // Still a declaration and still in a different module - if (F->isDeclaration() && F->getParent() != destModule) { - // Create forward declaration in current module - return InjectFunctionProto(F); - } - } - else if (isa(V)) { - GlobalVariable *GV = cast(V); - assert(GV != NULL); - GlobalVariable *oldGV = destModule->getGlobalVariable(GV->getName()); - if (oldGV != NULL) - return oldGV; - GlobalVariable *newGV = new GlobalVariable(*destModule, - GV->getType()->getElementType(), - GV->isConstant(), - GlobalVariable::ExternalLinkage, - NULL, - GV->getName()); - newGV->copyAttributesFrom(GV); - if (GV->isDeclaration()) - return newGV; - if (!GV->getName().empty()) { - uint64_t addr = jl_ExecutionEngine->getGlobalValueAddress(GV->getName()); - if (addr != 0) { - newGV->setExternallyInitialized(true); - return newGV; - } - } - std::map::iterator it; - it = jl_llvm_to_jl_value.find(GV); - if (it != jl_llvm_to_jl_value.end()) { - newGV->setInitializer(Constant::getIntegerValue(GV->getType()->getElementType(),APInt(sizeof(void*)*8,(intptr_t)it->second))); - newGV->setConstant(true); - } - else if (GV->hasInitializer()) { - Value *C = MapValue(GV->getInitializer(),VMap,RF_None,NULL,this); - newGV->setInitializer(cast(C)); - } - return newGV; - } - return NULL; - }; -}; -#else -static Function *function_proto(Function *F) -{ - return F; +// return GV; } -#endif + +// --- Debug info --- #ifdef LLVM37 static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed = false) @@ -569,218 +183,6 @@ static Value *literal_static_pointer_val(const void *p, Type *t) #endif } -static std::vector jl_sysimg_gvars; -static std::vector jl_sysimg_fvars; - -extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) -{ - // map a jl_value_t memory location to a GlobalVariable - std::map::iterator it; - it = jl_value_to_llvm.find(p); - if (it == jl_value_to_llvm.end()) - return 0; - return it->second.index; -} - -#ifdef HAVE_CPUID -extern "C" { - extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); -} -#endif - -static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, - const char *sysimg_data, size_t sysimg_len) -{ - ArrayType *gvars_type = ArrayType::get(T_psize, jl_sysimg_gvars.size()); - addComdat(new GlobalVariable(*mod, - gvars_type, - true, - GlobalVariable::ExternalLinkage, - MapValue(ConstantArray::get(gvars_type, ArrayRef(jl_sysimg_gvars)), VMap), - "jl_sysimg_gvars")); - ArrayType *fvars_type = ArrayType::get(T_pvoidfunc, jl_sysimg_fvars.size()); - addComdat(new GlobalVariable(*mod, - fvars_type, - true, - GlobalVariable::ExternalLinkage, - MapValue(ConstantArray::get(fvars_type, ArrayRef(jl_sysimg_fvars)), VMap), - "jl_sysimg_fvars")); - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size,globalUnique+1), - "jl_globalUnique")); -#ifdef JULIA_ENABLE_THREADING - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size, jltls_states_func_idx), - "jl_ptls_states_getter_idx")); -#endif - - Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); - addComdat(new GlobalVariable(*mod, - feature_string->getType(), - true, - GlobalVariable::ExternalLinkage, - feature_string, - "jl_sysimg_cpu_target")); - -#ifdef HAVE_CPUID - // For native also store the cpuid - if (strcmp(jl_options.cpu_target,"native") == 0) { - uint32_t info[4]; - - jl_cpuid((int32_t*)info, 1); - addComdat(new GlobalVariable(*mod, - T_int64, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), - "jl_sysimg_cpu_cpuid")); - } -#endif - - if (sysimg_data) { - Constant *data = ConstantDataArray::get(jl_LLVMContext, ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); - addComdat(new GlobalVariable(*mod, data->getType(), true, - GlobalVariable::ExternalLinkage, - data, "jl_system_image_data")); - Constant *len = ConstantInt::get(T_size, sysimg_len); - addComdat(new GlobalVariable(*mod, len->getType(), true, - GlobalVariable::ExternalLinkage, - len, "jl_system_image_size")); - } -} - -static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len, - bool dump_as_bc) -{ -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - realize_pending_globals(); -#endif -#ifdef JL_DEBUG_BUILD - verifyModule(*shadow_module); -#endif - -#ifdef LLVM36 - std::error_code err; - StringRef fname_ref = StringRef(fname); - raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); -#elif defined(LLVM35) - std::string err; - raw_fd_ostream OS(fname, err, sys::fs::F_None); -#else - std::string err; - raw_fd_ostream OS(fname, err); -#endif -#ifdef LLVM37 // 3.7 simplified formatted output; just use the raw stream alone - raw_fd_ostream& FOS(OS); -#else - formatted_raw_ostream FOS(OS); -#endif - - // We don't want to use MCJIT's target machine because - // it uses the large code model and we may potentially - // want less optimizations there. - Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); -#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) -#ifdef LLVM35 - TheTriple.setObjectFormat(Triple::COFF); -#else - TheTriple.setEnvironment(Triple::UnknownEnvironment); -#endif -#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) -#ifdef LLVM35 - TheTriple.setObjectFormat(Triple::MachO); -#else - TheTriple.setEnvironment(Triple::MachO); -#endif -#endif -#ifdef LLVM35 - std::unique_ptr -#else - OwningPtr -#endif - TM(jl_TargetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - jl_TargetMachine->getTargetCPU(), - jl_TargetMachine->getTargetFeatureString(), - jl_TargetMachine->Options, -#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) - Reloc::PIC_, -#else - jit_model ? Reloc::PIC_ : Reloc::Default, -#endif - jit_model ? CodeModel::JITDefault : CodeModel::Default, - CodeGenOpt::Aggressive // -O3 - )); - -#ifdef LLVM38 - legacy::PassManager PM; -#else - PassManager PM; -#endif -#ifndef LLVM37 - PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); -#else - PM.add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); -#endif -#ifdef LLVM37 -// No DataLayout pass needed anymore. -#elif defined(LLVM36) - PM.add(new DataLayoutPass()); -#elif defined(LLVM35) - PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); -#else - PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); -#endif - - addOptimizationPasses(&PM); - if (!dump_as_bc) { - if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { - jl_error("Could not generate obj file for this target"); - } - } - - // now copy the module, since PM.run may modify it - ValueToValueMapTy VMap; -#ifdef LLVM38 - Module *clone = CloneModule(shadow_module, VMap).release(); -#else - Module *clone = CloneModule(shadow_module, VMap); -#endif -#ifdef LLVM37 - // Reset the target triple to make sure it matches the new target machine - clone->setTargetTriple(TM->getTargetTriple().str()); -#ifdef LLVM38 - clone->setDataLayout(TM->createDataLayout()); -#else - clone->setDataLayout(TM->getDataLayout()->getStringRepresentation()); -#endif -#endif - - // add metadata information - jl_gen_llvm_globaldata(clone, VMap, sysimg_data, sysimg_len); - - // do the actual work - finalize_gc_frame(clone); - PM.run(*clone); - if (dump_as_bc) - WriteBitcodeToFile(clone, FOS); - delete clone; -} - -static int32_t jl_assign_functionID(Function *functionObject, int specsig) -{ - // give the function an index in the constant lookup table - if (!imaging_mode) - return 0; - jl_sysimg_fvars.push_back(ConstantExpr::getBitCast(functionObject, T_pvoidfunc)); - return jl_sysimg_fvars.size(); -} static Value *julia_gv(const char *cname, void *addr) { @@ -794,24 +196,11 @@ static Value *julia_gv(const char *cname, void *addr) std::stringstream gvname; gvname << cname << globalUnique++; // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(imaging_mode ? *shadow_module : *builtins_module, T_pjlvalue, - false, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - ConstantPointerNull::get((PointerType*)T_pjlvalue), gvname.str()); + GlobalVariable *gv = new GlobalVariable(*jl_builderModule, T_pjlvalue, + false, GlobalVariable::ExternalLinkage, + NULL, gvname.str()); addComdat(gv); - - // make the pointer valid for this session -#ifdef USE_MCJIT - jl_llvm_to_jl_value[gv] = addr; -#else - void **p = (void**)jl_ExecutionEngine->getPointerToGlobal(gv); - *p = addr; -#endif - // make the pointer valid for future sessions - jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(gv, T_psize)); - jl_value_llvm gv_struct; - gv_struct.gv = prepare_global(gv); - gv_struct.index = jl_sysimg_gvars.size(); - jl_value_to_llvm[addr] = gv_struct; + *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; return builder.CreateLoad(gv); } @@ -1183,11 +572,8 @@ static Value *emit_datatype_isbitstype(Value *dt) static void just_emit_error(const std::string &txt, jl_codectx_t *ctx) { - Value *zeros[2] = { ConstantInt::get(T_int32, 0), - ConstantInt::get(T_int32, 0) }; builder.CreateCall(prepare_call(jlerror_func), - builder.CreateGEP(stringConst(txt), - ArrayRef(zeros))); + GetStringPtr(&builder, stringConst(txt), "just_emit_error")); } static void emit_error(const std::string &txt, jl_codectx_t *ctx) @@ -1252,12 +638,8 @@ static void null_pointer_check(Value *v, jl_codectx_t *ctx) static void emit_type_error(const jl_cgval_t &x, jl_value_t *type, const std::string &msg, jl_codectx_t *ctx) { - Value *zeros[2] = { ConstantInt::get(T_int32, 0), - ConstantInt::get(T_int32, 0) }; - Value *fname_val = builder.CreateGEP(stringConst(ctx->funcName), - ArrayRef(zeros)); - Value *msg_val = builder.CreateGEP(stringConst(msg), - ArrayRef(zeros)); + Value *fname_val = GetStringPtr(&builder, stringConst(ctx->funcName), "fname_val"); + Value *msg_val = GetStringPtr(&builder, stringConst(msg), "msg_val"); #ifdef LLVM37 builder.CreateCall(prepare_call(jltypeerror_func), { fname_val, msg_val, diff --git a/src/codegen.cpp b/src/codegen.cpp index faffd23bb802d..537aaa56d2c81 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -198,31 +198,17 @@ namespace llvm { // for image reloading static bool imaging_mode = false; -#include "jitlayers.cpp" - -#ifdef USE_ORCJIT -JuliaOJIT *jl_ExecutionEngine; -#else -ExecutionEngine *jl_ExecutionEngine; -#endif - -#ifdef USE_MCJIT -static Module *shadow_module; -static Module *builtins_module; -static Module *active_module; -#define jl_Module (builder.GetInsertBlock()->getParent()->getParent()) -#else -static Module *jl_Module; -#define shadow_module jl_Module -#define active_module jl_Module -#define builtins_module jl_Module -#endif +static Module *shadow_output; +#define jl_Module ctx->f->getParent() +#define jl_builderModule builder.GetInsertBlock()->getParent()->getParent() static MDBuilder *mbuilder; static std::map argNumberStrings; +#ifndef USE_ORCJIT #ifdef LLVM38 -static legacy::FunctionPassManager *FPM; +static legacy::PassManager *PM; #else -static FunctionPassManager *FPM; +static PassManager *PM; +#endif #endif #ifdef LLVM37 @@ -438,6 +424,11 @@ static std::vector three_pvalue_llvmt; static std::map builtin_func_map; // --- code generation --- +extern "C" { + int globalUnique = 0; +} + +#include "jitlayers.cpp" // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { @@ -556,13 +547,6 @@ typedef struct { jl_value_t *ty; } jl_arrayvar_t; -// Keeps tracks of all functions and compile units created during this cycle -// to be able to atomically add them to a module. -typedef struct { - std::vector functions; - std::vector CUs; -} jl_cyclectx_t; - // information about the context of a piece of code: its enclosing // function and module, and visible local variables and labels. typedef struct { @@ -595,8 +579,6 @@ typedef struct { llvm::DIBuilder *dbuilder; bool debug_enabled; std::vector to_inline; - - jl_cyclectx_t *cyclectx; } jl_codectx_t; typedef struct { @@ -624,8 +606,7 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol=false); static Value *emit_condition(jl_value_t *cond, const std::string &msg, jl_codectx_t *ctx); static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); -static void finalize_gc_frame(Function *F); -static void finalize_gc_frame(Module *m); +static void jl_finalize_module(std::unique_ptr m); static GlobalVariable *prepare_global(GlobalVariable *G); // --- convenience functions for tagging llvm values with julia types --- @@ -716,10 +697,6 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) // --- utilities --- -extern "C" { - int globalUnique = 0; -} - static void emit_write_barrier(jl_codectx_t*, Value*, Value*); #include "cgutils.cpp" @@ -821,17 +798,16 @@ void jl_dump_compiles(void *s) // --- entry point --- //static int n_emit=0; -static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations, - jl_llvm_functions_t *definitions, jl_cyclectx_t *cyclectx); -static void jl_finalize_module(Module *m); +static Module *emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations); void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataLayout &DL); //static int n_compile=0; -static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) +static Module *to_function(jl_lambda_info_t *li) { JL_LOCK(codegen); JL_SIGATOMIC_BEGIN(); assert(!li->inInference); + li->inCompile = 1; BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; DebugLoc olddl = builder.getCurrentDebugLocation(); bool last_n_c = nested_compile; @@ -839,30 +815,12 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) last_time = jl_hrtime(); nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); + Module *m = NULL; Function *f = NULL, *specf = NULL; JL_TRY { - jl_llvm_functions_t definitions; - #if defined(USE_MCJIT) || defined(USE_ORCJIT) - jl_cyclectx_t *newcyclectx = cyclectx; - if (!newcyclectx) - newcyclectx = new jl_cyclectx_t; - emit_function(li, &li->functionObjects, &definitions, newcyclectx); - // If we're the root of the cycle, realize all functions - if (!cyclectx) { - realize_cycle(newcyclectx); - delete newcyclectx; - } - #else - emit_function(li, &li->functionObjects, &definitions, NULL); - #endif - f = (llvm::Function*)definitions.functionObject; - specf = (llvm::Function*)definitions.specFunctionObject; - li->functionID = jl_assign_functionID(f, 0); - if (specf) - li->specFunctionID = jl_assign_functionID(specf, 1); - if (f->getFunctionType() != jl_func_sig) - // mark the pointer as jl_fptr_sparam_t calling convention - li->jlcall_api = 1; + m = emit_function(li, &li->functionObjects); + f = (Function*)li->functionObjects.functionObject; + specf = (Function*)li->functionObjects.specFunctionObject; //n_emit++; } JL_CATCH { @@ -874,45 +832,40 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); } + li->inCompile = 0; JL_SIGATOMIC_END(); JL_UNLOCK(codegen); jl_rethrow_with_add("error compiling %s", jl_symbol_name(li->name)); } - assert(f != NULL); - const DataLayout &DL = -#ifdef LLVM35 - f->getParent()->getDataLayout(); -#else - *jl_data_layout; -#endif + assert(m != NULL); + jl_finalize_module(std::unique_ptr(m)); + // record that this function name came from this linfo, // so we can build a reverse mapping for debug-info. if (li->name != anonymous_sym) { + const DataLayout &DL = +#ifdef LLVM35 + m->getDataLayout(); +#else + *jl_data_layout; +#endif // but don't remember anonymous symbols because // they may not be rooted in the gc for the life of the program, // and the runtime doesn't notify us when the code becomes unreachable :( jl_add_linfo_in_flight((specf ? specf : f)->getName(), li, DL); } -#if !defined(USE_MCJIT) && !defined(USE_ORCJIT) - finalize_gc_frame(f); - if (specf) finalize_gc_frame(specf); -#ifdef JL_DEBUG_BUILD - if (verifyFunction(*f, PrintMessageAction) || - (specf && verifyFunction(*specf, PrintMessageAction))) { - f->dump(); - if (specf) specf->dump(); - gc_debug_critical_error(); - abort(); - } -#endif - FPM->run(*f); - if (specf) FPM->run(*specf); -#endif + li->functionID = jl_assign_functionID(f, 0); + if (specf) + li->specFunctionID = jl_assign_functionID(specf, 1); + if (f->getFunctionType() != jl_func_sig) + // mark the pointer as jl_fptr_sparam_t calling convention + li->jlcall_api = 1; if (old != NULL) { builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); } + li->inCompile = 0; nested_compile = last_n_c; jl_gc_inhibit_finalizers(nested_compile); JL_UNLOCK(codegen); @@ -924,7 +877,7 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) jl_printf(dump_compiles_stream, "\"\n"); last_time = this_time; } - return specf ? specf : f; + return m; } #ifndef LLVM37 @@ -975,185 +928,69 @@ static void jl_setup_module(Module *m) #endif } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static void jl_finalize_module(Module *m) +static void finalize_gc_frame(Module *m); +static void jl_finalize_module(std::unique_ptr uniquem) { -#if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) && defined(USE_MCJIT) - ArrayType *atype = ArrayType::get(T_uint32,3); // want 4-byte alignment of 12-bytes of data - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); -#endif + Module *m = uniquem.release(); finalize_gc_frame(m); - assert(jl_ExecutionEngine); -#if defined(LLVM36) && !defined(USE_ORCJIT) - jl_ExecutionEngine->addModule(std::unique_ptr(m)); -#else - jl_ExecutionEngine->addModule(m); +#if !defined(USE_ORCJIT) +#ifdef LLVM33 +#ifdef JL_DEBUG_BUILD + if (verifyModule(*m, PrintMessageAction)) { + m->dump(); + gc_debug_critical_error(); + abort(); + } #endif -} - -#if !defined(USE_ORCJIT) && defined(JL_DEBUG_BUILD) -static void writeRecoveryFile(llvm::Module *mod) -{ - std::error_code err; - mod->dump(); - std::cout << "Julia emitted a broken LLVM module (about to be __jl_dump.bc)." - << "Please file a bug report.\n" - << "If the module writing below fails," - << "please include the textual representation printed above this error."; - StringRef fname_ref = StringRef("__jl_dump.bc"); - raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); - WriteBitcodeToFile(mod,OS); - OS.flush(); - gc_debug_critical_error(); - abort(); -} #endif + PM->run(*m); +#endif +#ifdef USE_MCJIT + // record the function names that are part of this Module + // so it can be added to the JIT when needed + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) + module_for_fname[F->getName()] = m; + } +#endif + jl_add_to_shadow(m); +} -static uint64_t getAddressForOrCompileFunction(llvm::Function *llvmf) +static uint64_t getAddressForFunction(llvm::Function *llvmf) { - #ifdef JL_DEBUG_BUILD +#ifdef JL_DEBUG_BUILD llvm::raw_fd_ostream out(1,false); - #endif - Function *ActiveF = active_module->getFunction(llvmf->getName()); - // Must have been in a prior module. Safe to ask the execution engine - // to emit it. - if (!ActiveF || ActiveF->isDeclaration()) - return jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); - if (!imaging_mode) { - realize_pending_globals(); - #ifndef USE_ORCJIT - #ifdef JL_DEBUG_BUILD -#ifdef LLVM38 - Module *backup = llvm::CloneModule(active_module).release(); +#endif +#ifdef USE_MCJIT + jl_finalize_function(llvmf, NULL); + return jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); #else - Module *backup = llvm::CloneModule(active_module); + return (uint64_t)jl_ExecutionEngine->getPointerToFunction( + cast(shadow_output->getNamedValue(llvmf->getName()))); #endif - if(verifyModule(*active_module)) - writeRecoveryFile(backup); - #endif - for (auto &F : active_module->functions()) { - if (F.isDeclaration()) - continue; - #ifdef JL_DEBUG_BUILD - if(verifyFunction(F)) - writeRecoveryFile(backup); - #endif - finalize_gc_frame(&F); - FPM->run(F); - #ifdef JL_DEBUG_BUILD - if(verifyFunction(F)) - writeRecoveryFile(backup); - #endif - } - #ifdef JL_DEBUG_BUILD - if(verifyModule(*active_module)) - writeRecoveryFile(backup); - delete backup; - #endif - #endif - jl_finalize_module(active_module); - } - uint64_t addr = jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); - assert(addr != 0); - if (!imaging_mode) { - active_module = new Module("julia", jl_LLVMContext); - jl_setup_module(active_module); - } - return addr; } -#endif extern "C" void jl_generate_fptr(jl_lambda_info_t *li) { JL_LOCK(codegen); // objective: assign li->fptr assert(li->functionObjects.functionObject); + assert(!li->inCompile); if (li->fptr == NULL) { JL_SIGATOMIC_BEGIN(); - #ifdef USE_MCJIT - if (imaging_mode) { - // see if it has been emitted already (as part of compiling something else) - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.functionObject)->getName()); - if (li->fptr == NULL) { - // Copy the function out of the shadow module - Module *m = new Module("julia", jl_LLVMContext); - jl_setup_module(m); - FunctionMover mover(m, shadow_module); - mover.CloneFunction((Function*)li->functionObjects.functionObject); - if (li->functionObjects.specFunctionObject != NULL) - mover.CloneFunction((Function*)li->functionObjects.specFunctionObject); - if (li->functionObjects.cFunctionList != NULL) { - size_t i; - cFunctionList_t *list = (cFunctionList_t*)li->functionObjects.cFunctionList; - for (i = 0; i < list->len; i++) { - list->data()[i].f = mover.CloneFunction(list->data()[i].f); - } - } - jl_finalize_module(m); - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.functionObject)->getName()); - } - } - else { - li->fptr = (jl_fptr_t)getAddressForOrCompileFunction((Function*)li->functionObjects.functionObject); - } - #else - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getPointerToFunction((Function*)li->functionObjects.functionObject); - #endif - + li->fptr = (jl_fptr_t)getAddressForFunction((Function*)li->functionObjects.functionObject); assert(li->fptr != NULL); -#ifndef KEEP_BODIES - if (!imaging_mode) - ((Function*)li->functionObjects.functionObject)->deleteBody(); -#endif - - if (li->functionObjects.cFunctionList != NULL) { - size_t i; - cFunctionList_t *list = (cFunctionList_t*)li->functionObjects.cFunctionList; - for (i = 0; i < list->len; i++) { -#ifdef USE_MCJIT - (void)getAddressForOrCompileFunction(list->data()[i].f); -#else - (void)jl_ExecutionEngine->getPointerToFunction(list->data()[i].f); -#endif -#ifndef KEEP_BODIES - if (!imaging_mode) { - list->data()[i].f->deleteBody(); - } -#endif - } - } - - if (li->functionObjects.specFunctionObject != NULL) { -#ifdef USE_MCJIT - if (imaging_mode) - (void)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.specFunctionObject)->getName()); - else - (void)getAddressForOrCompileFunction((Function*)li->functionObjects.specFunctionObject); -#else - (void)jl_ExecutionEngine->getPointerToFunction((Function*)li->functionObjects.specFunctionObject); -#endif -#ifndef KEEP_BODIES - if (!imaging_mode) - ((Function*)li->functionObjects.specFunctionObject)->deleteBody(); -#endif - } JL_SIGATOMIC_END(); } JL_UNLOCK(codegen); } -extern "C" void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx) +extern "C" void jl_compile_linfo(jl_lambda_info_t *li) { if (li->functionObjects.functionObject == NULL) { // objective: assign li->functionObject - li->inCompile = 1; - (void)to_function(li, (jl_cyclectx_t *)cyclectx); - li->inCompile = 0; + (void)to_function(li); } } @@ -1211,7 +1048,7 @@ static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tuplet } } - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)sigt, NULL); + jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)sigt); if (li != NULL) { for(i=1; i < nargs+1; i++) { jl_value_t *speci = jl_nth_slot_type(li->specTypes, i); @@ -1255,26 +1092,8 @@ void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) } assert(jl_is_tuple_type(argt)); Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - assert(llvmf); JL_GC_POP(); - -#ifdef USE_MCJIT - if (uint64_t addr = getAddressForOrCompileFunction(llvmf)) - return (void*)(intptr_t)addr; - if (llvmf->getParent() == shadow_module) { - // Copy the function out of the shadow module - Module *m = new Module("julia", jl_LLVMContext); - jl_setup_module(m); - FunctionMover mover(m, shadow_module); - (void)mover.CloneFunction(llvmf); - jl_finalize_module(m); - } -#endif -#ifdef USE_MCJIT - return (void*)getAddressForOrCompileFunction(llvmf); -#else - return jl_ExecutionEngine->getPointerToFunction(llvmf); -#endif + return (void*)getAddressForFunction(llvmf); } @@ -1295,33 +1114,24 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) assert(jl_is_tuple_type(argt)); Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); if (llvmf) { - Function *active_llvmf = active_module->getFunction(llvmf->getName()); - // In imaging mode, and in most cases in JIT mode (where the wrapper is specifically - // compiled for this function), we can simply use a global alias. - if (active_llvmf) { - #ifndef LLVM35 - new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, llvmf->getParent()); - #elif defined(LLVM37) && !defined(LLVM38) - GlobalAlias::create(cast(llvmf->getType()), - GlobalValue::ExternalLinkage, name, active_llvmf, active_module); - #else + // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) + uint64_t Addr = getAddressForFunction(llvmf); + llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); + // emit the function pointer and set up an alias in the execution engine +#if defined(USE_ORCJIT) || defined(USE_MCJIT) + jl_ExecutionEngine->addGlobalMapping(name, Addr); + if (imaging_mode) +#endif + // in imaging_mode, also need to add the alias to the shadow_module +#if defined(LLVM38) GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), - GlobalValue::ExternalLinkage, name, active_llvmf, active_module); - #endif - } else { - // Otherwise we use a global mapping - assert(!imaging_mode); -#if defined(USE_ORCJIT) - jl_ExecutionEngine->addGlobalMapping(llvmf->getName(), - (void*)getAddressForOrCompileFunction(llvmf)); -#elif defined(USE_MCJIT) - jl_ExecutionEngine->addGlobalMapping(llvmf->getName(), - getAddressForOrCompileFunction(llvmf)); + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); +#elif defined(LLVM37) + GlobalAlias::create(cast(llvmf->getType()), + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); #else - jl_ExecutionEngine->addGlobalMapping(llvmf, - jl_ExecutionEngine->getPointerToFunction(llvmf)); + new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, shadow_output); #endif - } } } @@ -1333,7 +1143,7 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g jl_lambda_info_t *linfo = NULL; JL_GC_PUSH2(&linfo, &tt); if (tt != NULL) { - linfo = jl_get_specialization1(tt, NULL); + linfo = jl_get_specialization1(tt); if (linfo == NULL) { linfo = jl_method_lookup_by_type(jl_gf_mtable(f), tt, 0, 0); if (linfo == NULL) { @@ -1352,62 +1162,40 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g } if (!getdeclarations) { - Function *llvmDecl = NULL; - if (!getwrapper && linfo->functionObjects.specFunctionObject != NULL) - llvmDecl = (Function*)linfo->functionObjects.specFunctionObject; - else - llvmDecl = (Function*)linfo->functionObjects.functionObject; -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - Function *llvmf = llvmDecl ? active_module->getFunction(llvmDecl->getName()) : NULL; - // Note that in either case, we need to run the FPM manually, - // since this is now usually done as part of object emission -#else - Function *llvmf = llvmDecl && !llvmDecl->isDeclaration() ? llvmDecl : NULL; -#endif - if (!llvmf) { - Function *other; - jl_llvm_functions_t declarations; - emit_function(linfo, NULL, &declarations, NULL); - if (getwrapper || !declarations.specFunctionObject) { - llvmf = (llvm::Function*)declarations.functionObject; - other = (llvm::Function*)declarations.specFunctionObject; - } - else { - llvmf = (llvm::Function*)declarations.specFunctionObject; - other = (llvm::Function*)declarations.functionObject; - } - if (other) - other->eraseFromParent(); -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - realize_pending_globals(); - finalize_gc_frame(llvmf); - FPM->run(*llvmf); -#endif - llvmf->removeFromParent(); - if (llvmDecl) - llvmf->setName(llvmDecl->getName()); + // emit this function into a new module + Function *f, *specf; + jl_llvm_functions_t declarations; + Module *m = emit_function(linfo, &declarations); + f = (llvm::Function*)declarations.functionObject; + specf = (llvm::Function*)declarations.specFunctionObject; + // swap declarations for definitions and destroy declarations + if (specf) { + Function *temp = cast(m->getNamedValue(specf->getName())); + delete specf; + specf = temp; + } + if (specf) { + Function *temp = cast(m->getNamedValue(f->getName())); + delete f; + f = temp; + } + Function *specf_decl = (Function*)linfo->functionObjects.specFunctionObject; + if (specf_decl) { + specf->setName(specf_decl->getName()); + } + Function *f_decl = (Function*)linfo->functionObjects.functionObject; + if (f_decl) { + f->setName(f_decl->getName()); + } + JL_GC_POP(); + if (getwrapper || !specf) { + return f; } else { - ValueToValueMapTy VMap; - Function *oldllvmf = llvmf; - llvmf = CloneFunction(llvmf, VMap, false); -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - active_module->getFunctionList().push_back(llvmf); - realize_pending_globals(); - finalize_gc_frame(llvmf); - FPM->run(*llvmf); - llvmf->removeFromParent(); - llvmf->setName(oldllvmf->getName()); -#else - (void)oldllvmf; -#endif + return specf; } - JL_GC_POP(); - return llvmf; - } - if (linfo->functionObjects.functionObject == NULL) { - jl_compile_linfo(linfo, NULL); } + jl_compile_linfo(linfo); Function *llvmf; if (!getwrapper && linfo->functionObjects.specFunctionObject != NULL) { llvmf = (Function*)linfo->functionObjects.specFunctionObject; @@ -1419,16 +1207,6 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g return llvmf; } -extern "C" JL_DLLEXPORT -uint64_t jl_get_llvm_fptr(llvm::Function *llvmf) -{ -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - return getAddressForOrCompileFunction(llvmf); -#else - return (uint64_t)jl_ExecutionEngine->getPointerToFunction(llvmf); -#endif -} - extern "C" JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump_module) { @@ -1436,57 +1214,50 @@ const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump llvm::raw_string_ostream stream(code); Function *llvmf = dyn_cast((Function*)f); - if (!llvmf) - jl_error("jl_dump_function_ir: Expected Function*"); + if (!llvmf || (!llvmf->isDeclaration() && !llvmf->getParent())) + jl_error("jl_dump_function_ir: Expected Function* in a temporary Module"); - if (llvmf->isDeclaration()) { - // print the function declaration plain + if (!llvmf->getParent()) { + // print the function declaration as-is llvmf->print(stream); } else { - if (llvmf->getParent()) - jl_error("jl_dump_function_ir requires a parentless clone"); - // Put the function in a module - Module *m = new Module(llvmf->getName(), jl_LLVMContext); - jl_setup_module(m); - m->getFunctionList().push_back(llvmf); - Function *f2 = llvmf; + Module *m = llvmf->getParent(); if (strip_ir_metadata) { - // strip metadata from the copy - Function::BasicBlockListType::iterator f2_bb = f2->getBasicBlockList().begin(); - // iterate over all basic blocks in the function - for (; f2_bb != f2->getBasicBlockList().end(); ++f2_bb) { - BasicBlock::InstListType::iterator f2_il = (*f2_bb).getInstList().begin(); - // iterate over instructions in basic block - for (; f2_il != (*f2_bb).getInstList().end(); ) { - Instruction *inst = &*f2_il++; - // remove dbg.declare and dbg.value calls - if (isa(inst) || isa(inst)) { - inst->eraseFromParent(); - continue; - } + // strip metadata from all instructions in the module + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *f2 = &*I; + Function::BasicBlockListType::iterator f2_bb = f2->getBasicBlockList().begin(); + // iterate over all basic blocks in the function + for (; f2_bb != f2->getBasicBlockList().end(); ++f2_bb) { + BasicBlock::InstListType::iterator f2_il = (*f2_bb).getInstList().begin(); + // iterate over instructions in basic block + for (; f2_il != (*f2_bb).getInstList().end(); ) { + Instruction *inst = &*f2_il++; + // remove dbg.declare and dbg.value calls + if (isa(inst) || isa(inst)) { + inst->eraseFromParent(); + continue; + } - SmallVector, 4> MDForInst; - inst->getAllMetadata(MDForInst); - SmallVector, 4>::iterator md_iter = MDForInst.begin(); + SmallVector, 4> MDForInst; + inst->getAllMetadata(MDForInst); + SmallVector, 4>::iterator md_iter = MDForInst.begin(); - // iterate over all metadata kinds and set to NULL to remove - for (; md_iter != MDForInst.end(); ++md_iter) { - inst->setMetadata((*md_iter).first, NULL); + // iterate over all metadata kinds and set to NULL to remove + for (; md_iter != MDForInst.end(); ++md_iter) { + inst->setMetadata((*md_iter).first, NULL); + } } } } } if (dump_module) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - realize_pending_globals(); -#endif m->print(stream, NULL); } - else - f2->print(stream); - f2->eraseFromParent(); - m->dropAllReferences(); + else { + llvmf->print(stream); + } delete m; } @@ -1580,18 +1351,15 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) // Dump assembly code uint64_t symsize = 0; int64_t slide = 0, section_slide = 0; + uint64_t fptr = getAddressForFunction(llvmf); #ifdef USE_MCJIT - uint64_t fptr = getAddressForOrCompileFunction(llvmf); -#ifdef USE_ORCJIT // Look in the system image as well if (fptr == 0) - fptr = jl_ExecutionEngine->findSymbol( - jl_ExecutionEngine->mangle(llvmf->getName()), true).getAddress(); -#endif + fptr = (uintptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable( + jl_ExecutionEngine->getMangledName(llvmf)); llvm::DIContext *context = NULL; llvm::DIContext *&objcontext = context; #else - uint64_t fptr = (uintptr_t)jl_ExecutionEngine->getPointerToFunction(llvmf); std::vector context; llvm::DIContext *objcontext = NULL; #endif @@ -1649,24 +1417,24 @@ static logdata_t coverageData; static void coverageVisitLine(StringRef filename, int line) { - if (filename == "" || filename == "none" || filename == "no file") - return; - logdata_t::iterator it = coverageData.find(filename); - if (it == coverageData.end()) { - coverageData[filename] = std::vector(0); - } - std::vector &vec = coverageData[filename]; - if (vec.size() <= (size_t)line) - vec.resize(line+1, NULL); - if (vec[line] == NULL) { - vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, - GlobalVariable::InternalLinkage, - ConstantInt::get(T_int64,0), "lcnt")); - } - GlobalVariable *v = prepare_global(vec[line]); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), - ConstantInt::get(T_int64,1)), - v, true); +// if (filename == "" || filename == "none" || filename == "no file") +// return; +// logdata_t::iterator it = coverageData.find(filename); +// if (it == coverageData.end()) { +// coverageData[filename] = std::vector(0); +// } +// std::vector &vec = coverageData[filename]; +// if (vec.size() <= (size_t)line) +// vec.resize(line+1, NULL); +// if (vec[line] == NULL) { +// vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, +// GlobalVariable::InternalLinkage, +// ConstantInt::get(T_int64,0), "lcnt")); +// } +// GlobalVariable *v = prepare_global(vec[line]); +// builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), +// ConstantInt::get(T_int64,1)), +// v, true); } extern "C" int isabspath(const char *in); @@ -1699,11 +1467,7 @@ static void write_log_data(logdata_t &logData, const char *extension) if ((size_t)l < values.size()) { GlobalVariable *gv = values[l]; if (gv) { -#ifdef USE_MCJIT - int *p = (int*)(intptr_t)jl_ExecutionEngine->getGlobalValueAddress(gv->getName()); -#else - int *p = (int*)jl_ExecutionEngine->getPointerToGlobal(gv); -#endif + int *p = (int*)jl_get_global(gv); value = *p; } } @@ -1738,30 +1502,30 @@ static logdata_t mallocData; static void mallocVisitLine(StringRef filename, int line) { - if (filename == "" || filename == "none" || filename == "no file") { - jl_gc_sync_total_bytes(); - return; - } - logdata_t::iterator it = mallocData.find(filename); - if (it == mallocData.end()) { - mallocData[filename] = std::vector(0); - } - std::vector &vec = mallocData[filename]; - if (vec.size() <= (size_t)line) - vec.resize(line+1, NULL); - if (vec[line] == NULL) { - vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, - GlobalVariable::InternalLinkage, - ConstantInt::get(T_int64,0), "bytecnt")); - } - GlobalVariable *v = prepare_global(vec[line]); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), - builder.CreateCall(prepare_call(diff_gc_total_bytes_func) -#ifdef LLVM37 - , {} -#endif - )), - v, true); +// if (filename == "" || filename == "none" || filename == "no file") { +// jl_gc_sync_total_bytes(); +// return; +// } +// logdata_t::iterator it = mallocData.find(filename); +// if (it == mallocData.end()) { +// mallocData[filename] = std::vector(0); +// } +// std::vector &vec = mallocData[filename]; +// if (vec.size() <= (size_t)line) +// vec.resize(line+1, NULL); +// if (vec[line] == NULL) { +// vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, +// GlobalVariable::InternalLinkage, +// ConstantInt::get(T_int64,0), "bytecnt")); +// } +// GlobalVariable *v = prepare_global(vec[line]); +// builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), +// builder.CreateCall(prepare_call(diff_gc_total_bytes_func) +//#ifdef LLVM37 +// , {} +//#endif +// )), +// v, true); } // Resets the malloc counts. Needed to avoid including memory usage @@ -1774,11 +1538,7 @@ extern "C" JL_DLLEXPORT void jl_clear_malloc_data(void) std::vector::iterator itb; for (itb = bytes.begin(); itb != bytes.end(); itb++) { if (*itb) { -#ifdef USE_MCJIT - int *p = (int*)(intptr_t)jl_ExecutionEngine->getGlobalValueAddress((*itb)->getName()); -#else - int *p = (int*)jl_ExecutionEngine->getPointerToGlobal(*itb); -#endif + int *p = (int*)jl_get_global(*itb); *p = 0; } } @@ -2942,11 +2702,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval jl_value_t *jlretty = li->rettype; bool retboxed; (void)julia_type_to_llvm(jlretty, &retboxed); - Function *cf = (Function*)li->functionObjects.specFunctionObject; - if (!cf->getParent() || (cf->getParent() == builtins_module && - builtins_module != active_module)) { // Call cycle - prepare_call(cf); - } + Function *cf = cast(prepare_call((Function*)li->functionObjects.specFunctionObject)); FunctionType *cft = cf->getFunctionType(); size_t nfargs = cft->getNumParams(); Value **argvals = (Value**) alloca(nfargs*sizeof(Value*)); @@ -3055,7 +2811,7 @@ static jl_cgval_t emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_sprint(args[0]), jl_sprint((jl_value_t*)aty)); }*/ - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty, ctx->cyclectx); + jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty); if (li != NULL) { assert(li->functionObjects.functionObject != NULL); theFptr = (Value*)li->functionObjects.functionObject; @@ -3139,11 +2895,12 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, b = jl_get_binding(m, s); if (b == NULL) { // var not found. switch to delayed lookup. + std::stringstream name; + name << "delayedvar" << globalUnique++; Constant *initnul = ConstantPointerNull::get((PointerType*)T_pjlvalue); - GlobalVariable *bindinggv = - prepare_global(new GlobalVariable(imaging_mode ? *shadow_module : *builtins_module, T_pjlvalue, - false, GlobalVariable::PrivateLinkage, - initnul, "delayedvar")); + GlobalVariable *bindinggv = new GlobalVariable(*ctx->f->getParent(), T_pjlvalue, + false, GlobalVariable::PrivateLinkage, + initnul, name.str()); Value *cachedval = builder.CreateLoad(bindinggv); BasicBlock *have_val = BasicBlock::Create(jl_LLVMContext, "found"), *not_found = BasicBlock::Create(jl_LLVMContext, "notfound"); @@ -3814,11 +3571,11 @@ static void finalize_gc_frame(Function *F) static void finalize_gc_frame(Module *m) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - for (auto &F : m->functions()) { - if (F.isDeclaration()) + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (F->isDeclaration()) continue; - finalize_gc_frame(&F); + finalize_gc_frame(F); } #ifndef JULIA_ENABLE_THREADING m->getFunction("jl_get_ptls_states")->eraseFromParent(); @@ -3827,7 +3584,6 @@ static void finalize_gc_frame(Module *m) m->getFunction("julia.gc_root_kill")->eraseFromParent(); m->getFunction("julia.gc_store")->eraseFromParent(); m->getFunction("julia.jlcall_frame_decl")->eraseFromParent(); -#endif } // here argt does not include the leading function type argument @@ -3872,7 +3628,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v if (fargt.size() + sret != fargt_sig.size()) jl_error("va_arg syntax not allowed for cfunction argument list"); - jl_compile_linfo(lam, NULL); + jl_compile_linfo(lam); if (!lam->functionObjects.functionObject) { jl_errorf("error compiling %s while creating cfunction", jl_symbol_name(lam->name)); @@ -3889,9 +3645,11 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); // no allocations expected between the top of this function (when last scanned lam->cFunctionList) and here, which might have triggered running julia code + Module *M = new Module(jl_symbol_name(lam->name), jl_LLVMContext); + jl_setup_module(M); Function *cw = Function::Create(FunctionType::get(sret ? T_void : prt, fargt_sig, false), - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(cw); cw->setAttributes(attrs); #ifdef LLVM37 @@ -3918,7 +3676,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v jl_throw(jl_memory_exception); list2->len = len; list2->data()[len-1].isref = isref; - list2->data()[len-1].f = imaging_mode ? cw : cw_proto; + list2->data()[len-1].f = cw_proto; lam->functionObjects.cFunctionList = list2; // See whether this function is specsig or jlcall @@ -4094,13 +3852,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v else builder.CreateRet(r); -#if !defined(USE_MCJIT) && !defined(USE_ORCJIT) - finalize_gc_frame(cw); - FPM->run(*cw); -#endif - - cw->removeFromParent(); - active_module->getFunctionList().push_back(cw); + jl_finalize_module(std::unique_ptr(M)); // Restore the previous compile context if (old != NULL) { @@ -4115,7 +3867,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret) +static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret, Module *M) { std::stringstream funcName; const std::string &fname = f->getName().str(); @@ -4125,8 +3877,8 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct else funcName << fname; - Function *w = Function::Create(jl_func_sig, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(w); #ifdef LLVM37 w->addFnAttr("no-frame-pointer-elim", "true"); @@ -4195,10 +3947,9 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct } // Compile to LLVM IR, using a specialized signature if applicable. -static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations, - jl_llvm_functions_t *definitions, jl_cyclectx_t *cyclectx) +static Module *emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations) { - assert(definitions && "Capturing definitions is always required"); + assert(declarations && "Capturing declarations is always required"); // step 1. unpack AST and allocate codegen context for this function jl_expr_t *ast = (jl_expr_t*)lam->ast; @@ -4224,7 +3975,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio ctx.vaStack = false; ctx.inbounds.push_back(false); ctx.boundsCheck.push_back(false); - ctx.cyclectx = cyclectx; ctx.spvals_ptr = NULL; // step 2. process var-info lists to see what vars need boxing @@ -4327,6 +4077,8 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio funcName << "_" << globalUnique++; ctx.sret = false; + Module *M = new Module(jl_symbol_name(lam->name), jl_LLVMContext); + jl_setup_module(M); if (specsig) { // assumes !va and !needsparams std::vector fsig(0); Type *rt; @@ -4352,37 +4104,28 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio fsig.push_back(ty); } f = Function::Create(FunctionType::get(rt, fsig, false), - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); if (ctx.sret) f->addAttribute(1, Attribute::StructRet); addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif - definitions->specFunctionObject = f; - if (declarations) - declarations->specFunctionObject = function_proto(f); - fwrap = gen_jlcall_wrapper(lam, ast, - (llvm::Function*)lam->functionObjects.specFunctionObject, ctx.sret); - definitions->functionObject = fwrap; - if (declarations) - declarations->functionObject = function_proto(fwrap); + fwrap = gen_jlcall_wrapper(lam, ast, f, ctx.sret, M); + declarations->functionObject = function_proto(fwrap); + declarations->specFunctionObject = function_proto(f); } else { f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif - definitions->functionObject = f; - definitions->specFunctionObject = NULL; - if (declarations) { - declarations->functionObject = function_proto(f); - declarations->specFunctionObject = NULL; - } + declarations->functionObject = function_proto(f); + declarations->specFunctionObject = NULL; } if (jlrettype == (jl_value_t*)jl_bottom_type) f->setDoesNotReturn(); @@ -4440,7 +4183,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio filename = ""; int toplineno = lno; - DIBuilder dbuilder(*builtins_module); + DIBuilder dbuilder(*M); ctx.dbuilder = &dbuilder; #ifdef LLVM37 DIFile *topfile = NULL; @@ -4646,7 +4389,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio /* // step 6. (optional) check for stack overflow (the slower way) Value *cur_sp = - builder.CreateCall(Intrinsic::getDeclaration(jl_Module, + builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::frameaddress), ConstantInt::get(T_int32, 0)); Value *sp_ok = @@ -4678,12 +4421,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio // must be in the first basic block for the llvm mem2reg pass to work // get pointers for locals stored in the gc frame array (argTemp) -#ifdef LLVM36 - if (ctx.debug_enabled) { - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::dbg_declare)); - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::dbg_value)); - } -#endif for (std::map::iterator I = ctx.vars.begin(), E = ctx.vars.end(); I != E; ++I) { jl_sym_t *s = I->first; jl_varinfo_t &varinfo = I->second; @@ -5065,9 +4802,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio for(std::vector::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) { Function *inlinef = (*it)->getCalledFunction(); InlineFunctionInfo info; - // Intrinsics that InlineFunction might create - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::lifetime_start)); - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::lifetime_end)); if (!InlineFunction(*it,info)) jl_error("Inlining Pass failed"); if (inlinef->getParent()) @@ -5078,26 +4812,14 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio } } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - if (cyclectx) { - cyclectx->functions.push_back(f); - if (fwrap) - cyclectx->functions.push_back(fwrap); - } -#endif - // step 15. Perform any delayed instantiations if (ctx.debug_enabled) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - if(cyclectx) - cyclectx->CUs.push_back(CU); -#endif ctx.dbuilder->finalize(); } JL_GC_POP(); - return; + return M; } // --- initialization --- @@ -5140,8 +4862,8 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs } else { // this assigns a function pointer (from loading the system image), to the function object - std::string funcName = jl_symbol_name(lam->name); - funcName = "julia_" + funcName; + std::stringstream funcName; + funcName << "jlsys_" << jl_symbol_name(lam->name) << "_" << globalUnique++; if (specsig) { // assumes !va std::vector fsig(0); jl_value_t *jlrettype = lam->rettype; @@ -5168,8 +4890,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs ty = PointerType::get(ty,0); fsig.push_back(ty); } - Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName, - shadow_module); + Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName.str(), shadow_output); if (sret) f->addAttribute(1, Attribute::StructRet); @@ -5184,7 +4905,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs lam->fptr = fptr; } else { - Function *f = jlcall_func_to_llvm(funcName, fptr, shadow_module); + Function *f = jlcall_func_to_llvm(funcName.str(), fptr, shadow_output); if (lam->functionObjects.functionObject == NULL) { lam->functionObjects.functionObject = (void*)f; assert(lam->fptr == NULL); @@ -5432,17 +5153,11 @@ static void init_julia_llvm_env(Module *m) PointerType *pfunctype = jltls_states_func->getFunctionType()->getPointerTo(); jltls_states_func_ptr = new GlobalVariable(*m, pfunctype, - false, GlobalVariable::InternalLinkage, - ConstantPointerNull::get(pfunctype), - "jl_get_ptls_states.ptr"); + false, GlobalVariable::ExternalLinkage, + NULL, "jl_get_ptls_states.ptr"); addComdat(jltls_states_func_ptr); -#ifdef USE_MCJIT - jl_llvm_to_jl_value[jltls_states_func_ptr] = - (void*)jl_get_ptls_states_getter(); -#else - void **p = (void**)jl_ExecutionEngine->getPointerToGlobal(jltls_states_func_ptr); + void **p = (void**)jl_emit_and_add_to_shadow(jltls_states_func_ptr); *p = (void*)jl_get_ptls_states_getter(); -#endif jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(jltls_states_func_ptr, T_psize)); jltls_states_func_idx = jl_sysimg_gvars.size(); @@ -5869,12 +5584,6 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlcall_frame_func, (void*)NULL, /*dllimport*/false); // set up optimization passes -#ifdef LLVM38 - FPM = new legacy::FunctionPassManager(m); -#else - FPM = new FunctionPassManager(m); -#endif - #ifdef LLVM37 // No DataLayout pass needed anymore. #elif defined(LLVM36) @@ -5885,11 +5594,22 @@ static void init_julia_llvm_env(Module *m) jl_data_layout = new DataLayout(*jl_ExecutionEngine->getDataLayout()); #endif +#ifndef USE_ORCJIT +#ifdef LLVM38 + PM = new legacy::PassManager(); +#else + PM = new PassManager(); +#endif +#ifndef LLVM37 + PM->add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); +#else + PM->add(new TargetLibraryInfoWrapperPass(Triple(jl_TargetMachine->getTargetTriple()))); +#endif #ifndef LLVM37 - FPM->add(jl_data_layout); + PM->add(jl_data_layout); +#endif + addOptimizationPasses(PM); #endif - addOptimizationPasses(FPM); - FPM->doInitialization(); } // Helper to figure out what features to set for the LLVM target @@ -5976,24 +5696,13 @@ extern "C" void jl_init_codegen(void) InitializeNativeTargetAsmParser(); Module *m, *engine_module; - + engine_module = new Module("julia", jl_LLVMContext); #ifdef USE_MCJIT - m = shadow_module = new Module("shadow", jl_LLVMContext); - builtins_module = new Module("julia_builtins", jl_LLVMContext); - if (imaging_mode) { - engine_module = new Module("engine_module", jl_LLVMContext); - active_module = shadow_module; - } - else { - active_module = new Module("julia", jl_LLVMContext); - engine_module = m; -#ifdef USE_ORCJIT - engine_module = new Module("engine_module", jl_LLVMContext); -#endif - } + m = new Module("julia", jl_LLVMContext); #else - engine_module = m = jl_Module = new Module("julia", jl_LLVMContext); + m = engine_module; #endif + shadow_output = m; TargetOptions options = TargetOptions(); //options.PrintMachineCode = true; //Print machine code produced during JIT compiling @@ -6095,19 +5804,9 @@ extern "C" void jl_init_codegen(void) mbuilder = new MDBuilder(getGlobalContext()); // Now that the execution engine exists, initialize all modules -#ifdef USE_MCJIT - jl_setup_module(engine_module); - jl_setup_module(shadow_module); - jl_setup_module(active_module); - jl_setup_module(builtins_module); -#else jl_setup_module(engine_module); -#endif - - if (imaging_mode) - init_julia_llvm_env(shadow_module); - else - init_julia_llvm_env(builtins_module); + jl_setup_module(m); + init_julia_llvm_env(m); #ifndef USE_ORCJIT jl_ExecutionEngine->RegisterJITEventListener(CreateJuliaJITEventListener()); diff --git a/src/codegen_internal.h b/src/codegen_internal.h index 02d08b8aa3034..860193849e59c 100644 --- a/src/codegen_internal.h +++ b/src/codegen_internal.h @@ -1,4 +1,6 @@ -// Pre-declaration. Definition in disasm.cpp +// This file is a part of Julia. License is MIT: http://julialang.org/license + +// Declarations for disasm.cpp extern "C" void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #ifndef USE_MCJIT @@ -12,6 +14,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #endif ); +// Declarations for debuginfo.cpp extern int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *section_slide, const object::ObjectFile **object, #ifdef USE_MCJIT @@ -20,8 +23,17 @@ extern int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int6 std::vector *lines #endif ); + extern bool jl_dylib_DI_for_fptr(size_t pointer, const object::ObjectFile **object, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename); + #ifdef USE_MCJIT extern void jl_cleanup_DI(llvm::DIContext *context); #endif + +#ifdef USE_ORCJIT +extern JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, + const object::ObjectFile &obj, + const object::ObjectFile &debugObj, + const RuntimeDyld::LoadedObjectInfo &L); +#endif diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 8acd29dac0f83..d28b399a9a932 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -238,6 +238,10 @@ class JuliaJITEventListener: public JITEventListener #endif FuncInfo tmp = {&F, Size, Details.LineStarts, linfo}; info[(size_t)(Code)] = tmp; +#ifndef KEEP_BODIES + if (!jl_generating_output()) + const_cast(&F)->deleteBody(); +#endif uv_rwlock_wrunlock(&threadsafe); } diff --git a/src/dump.c b/src/dump.c index a62366553cfc7..4b9c9cea88d8a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1921,7 +1921,7 @@ JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) } // Get handle to sys.so - jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL); + jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_LOCAL | JL_RTLD_NOW); // set cpu target if unspecified by user and available from sysimg // otherwise default to native. diff --git a/src/gf.c b/src/gf.c index 5d2286911d5d0..880c807f26d63 100644 --- a/src/gf.c +++ b/src/gf.c @@ -870,7 +870,7 @@ static jl_value_t *jl_call_unspecialized(jl_svec_t *sparam_vals, jl_lambda_info_ jl_value_t **args, uint32_t nargs) { if (__unlikely(meth->fptr == NULL)) { - jl_compile_linfo(meth, NULL); + jl_compile_linfo(meth); jl_generate_fptr(meth); } assert(jl_svec_len(meth->sparam_syms) == jl_svec_len(sparam_vals)); @@ -1414,7 +1414,7 @@ jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_value_t *types, int lim); // compile-time method lookup -jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx) +jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) { assert(jl_nparams(types) > 0); if (!jl_is_leaf_type((jl_value_t*)types)) @@ -1451,7 +1451,7 @@ jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx) if (sf->functionObjects.functionObject == NULL) { if (sf->fptr != NULL) goto not_found; - jl_compile_linfo(sf, cyclectx); + jl_compile_linfo(sf); } JL_GC_POP(); return sf; @@ -1506,7 +1506,7 @@ static int _compile_all_tvar_union(jl_methlist_t *meth, jl_tupletype_t *methsig) if (jl_is_leaf_type((jl_value_t*)methsig)) { // usually can create a specialized version of the function, // if the signature is already a leaftype - jl_lambda_info_t *spec = jl_get_specialization1(methsig, NULL); + jl_lambda_info_t *spec = jl_get_specialization1(methsig); if (spec) { if (methsig == meth->sig) { // replace unspecialized func with newly specialized version @@ -1545,7 +1545,7 @@ static int _compile_all_tvar_union(jl_methlist_t *meth, jl_tupletype_t *methsig) goto getnext; // signature wouldn't be callable / is invalid -- skip it } if (jl_is_leaf_type(sig)) { - if (jl_get_specialization1((jl_tupletype_t*)sig, NULL)) { + if (jl_get_specialization1((jl_tupletype_t*)sig)) { if (!jl_has_typevars((jl_value_t*)sig)) goto getnext; // success } } @@ -1680,7 +1680,7 @@ static void _compile_all_deq(jl_array_t *found) linfo->functionID = -1; } else { - jl_compile_linfo(linfo, NULL); + jl_compile_linfo(linfo); assert(linfo->functionID > 0); } } @@ -1783,7 +1783,7 @@ void jl_compile_all(void) JL_DLLEXPORT void jl_compile_hint(jl_tupletype_t *types) { - (void)jl_get_specialization1(types, NULL); + (void)jl_get_specialization1(types); } #ifdef JL_TRACE diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 165de5ae1dfae..281197279fa11 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1146,12 +1146,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, case fma_float: { assert(y->getType() == x->getType()); assert(z->getType() == y->getType()); - Value *fmaintr = Intrinsic::getDeclaration(builtins_module, Intrinsic::fma, + Value *fmaintr = Intrinsic::getDeclaration(jl_Module, Intrinsic::fma, ArrayRef(x->getType())); #ifdef LLVM37 - return builder.CreateCall(prepare_call(fmaintr),{ FP(x), FP(y), FP(z) }); + return builder.CreateCall(fmaintr,{ FP(x), FP(y), FP(z) }); #else - return builder.CreateCall3(prepare_call(fmaintr), FP(x), FP(y), FP(z)); + return builder.CreateCall3(fmaintr, FP(x), FP(y), FP(z)); #endif } case muladd_float: @@ -1164,8 +1164,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, #else return builder.CreateCall3 #endif - (prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::fmuladd, - ArrayRef(x->getType()))), + (Intrinsic::getDeclaration(jl_Module, Intrinsic::fmuladd, + ArrayRef(x->getType())), #ifdef LLVM37 {FP(x), FP(y), FP(z)} #else @@ -1187,7 +1187,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *ix = JL_INT(x); Value *iy = JL_INT(y); assert(ix->getType() == iy->getType()); Value *intr = - Intrinsic::getDeclaration(builtins_module, + Intrinsic::getDeclaration(jl_Module, f==checked_sadd_int ? Intrinsic::sadd_with_overflow : (f==checked_uadd_int ? @@ -1201,9 +1201,9 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Intrinsic::umul_with_overflow)))), ArrayRef(ix->getType())); #ifdef LLVM37 - Value *res = builder.CreateCall(prepare_call(intr),{ix, iy}); + Value *res = builder.CreateCall(intr,{ix, iy}); #else - Value *res = builder.CreateCall2(prepare_call(intr), ix, iy); + Value *res = builder.CreateCall2(intr, ix, iy); #endif Value *obit = builder.CreateExtractValue(res, ArrayRef(1)); raise_exception_if(obit, prepare_global(jlovferr_var), ctx); @@ -1341,33 +1341,33 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, builder.CreateAShr(x, uint_cnvt(t,y))); case bswap_int: x = JL_INT(x); - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::bswap, - ArrayRef(x->getType()))), x); + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::bswap, + ArrayRef(x->getType())), x); case ctpop_int: x = JL_INT(x); - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::ctpop, - ArrayRef(x->getType()))), x); + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::ctpop, + ArrayRef(x->getType())), x); case ctlz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; - Value *ctlz = Intrinsic::getDeclaration(builtins_module, Intrinsic::ctlz, + Value *ctlz = Intrinsic::getDeclaration(jl_Module, Intrinsic::ctlz, ArrayRef(types)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(ctlz), {x, ConstantInt::get(T_int1,0)}); + return builder.CreateCall(ctlz, {x, ConstantInt::get(T_int1,0)}); #else - return builder.CreateCall2(prepare_call(ctlz), x, ConstantInt::get(T_int1,0)); + return builder.CreateCall2(ctlz, x, ConstantInt::get(T_int1,0)); #endif } case cttz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; - Value *cttz = Intrinsic::getDeclaration(builtins_module, Intrinsic::cttz, ArrayRef(types)); + Value *cttz = Intrinsic::getDeclaration(jl_Module, Intrinsic::cttz, ArrayRef(types)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(cttz), {x, ConstantInt::get(T_int1, 0)}); + return builder.CreateCall(cttz, {x, ConstantInt::get(T_int1, 0)}); #else - return builder.CreateCall2(prepare_call(cttz), x, ConstantInt::get(T_int1, 0)); + return builder.CreateCall2(cttz, x, ConstantInt::get(T_int1, 0)); #endif } @@ -1384,9 +1384,9 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, { x = FP(x); #ifdef LLVM34 - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::fabs, - ArrayRef(x->getType()))), + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::fabs, + ArrayRef(x->getType())), x); #else Type *intt = JL_INTT(x->getType()); @@ -1437,34 +1437,34 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, } case ceil_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::ceil, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::ceil, + ArrayRef(x->getType())), x); } case floor_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::floor, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::floor, + ArrayRef(x->getType())), x); } case trunc_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::trunc, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::trunc, + ArrayRef(x->getType())), x); } case rint_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::rint, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::rint, + ArrayRef(x->getType())), x); } case sqrt_llvm: { x = FP(x); raise_exception_unless(builder.CreateFCmpUGE(x, ConstantFP::get(x->getType(),0.0)), prepare_global(jldomerr_var), ctx); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::sqrt, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, + ArrayRef(x->getType())), x); } case powi_llvm: { @@ -1473,12 +1473,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Type *tx = x->getType(); // TODO: LLVM expects this to be i32 #ifdef LLVM36 Type *ts[1] = { tx }; - Value *powi = Intrinsic::getDeclaration(builtins_module, Intrinsic::powi, + Value *powi = Intrinsic::getDeclaration(jl_Module, Intrinsic::powi, ArrayRef(ts)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(powi), {x, y}); + return builder.CreateCall(powi, {x, y}); #else - return builder.CreateCall2(prepare_call(powi), x, y); + return builder.CreateCall2(powi, x, y); #endif #else // issue #6506 @@ -1488,8 +1488,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, } case sqrt_llvm_fast: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::sqrt, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, + ArrayRef(x->getType())), x); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 78a56b3bfe9a2..9b26aaae7f65c 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1,5 +1,6 @@ // This file is part of Julia. -// Parts of this file are copied from LLVM, under the UIUC license. + +// Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). template static void addOptimizationPasses(T *PM) @@ -158,11 +159,6 @@ extern "C" { LLVM_ATTRIBUTE_NOINLINE extern void __jit_debug_register_code(); } -extern JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, - const object::ObjectFile &obj, - const object::ObjectFile &debugObj, - const RuntimeDyld::LoadedObjectInfo &L); - #if defined(_OS_DARWIN_) && defined(LLVM37) && defined(LLVM_SHLIB) #define CUSTOM_MEMORY_MANAGER createRTDyldMemoryManagerOSX extern RTDyldMemoryManager *createRTDyldMemoryManagerOSX(); @@ -313,23 +309,47 @@ class JuliaOJIT { report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr); } - std::string mangle(const std::string &Name) + std::string getMangledName(const std::string &Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return MangledName; + SmallString<128> FullName; + Mangler::getNameWithPrefix(FullName, Name, DL); + return FullName.str(); } - void addGlobalMapping(StringRef Name, void *Addr) + std::string getMangledName(const GlobalValue *GV) { - GlobalSymbolTable[mangle(Name)] = Addr; + return getMangledName(GV->getName()); } - ModuleHandleT addModule(Module *M) + void addGlobalMapping(StringRef Name, uint64_t Addr) { + bool successful = GlobalSymbolTable.insert(make_pair(getMangledName(Name), (void*)Addr)).second; + assert(successful); + } + + void *getPointerToGlobalIfAvailable(StringRef S) { + GlobalSymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); + if (pos != GlobalSymbolTable.end()) + return pos->second; + return nullptr; + } + + ModuleHandleT addModule(std::unique_ptr M) + { +#ifndef NDEBUG + // validate the relocations for M + for (Module::iterator I = M->begin(), E = M->end(); I != E; ) { + Function *F = &*I; + ++I; + if (F->isDeclaration()) { + if (F->use_empty()) + F->eraseFromParent(); + else + assert(F->isIntrinsic() || findUnmangledSymbol(F->getName()) || + SectionMemoryManager::getSymbolAddressInProcess(F->getName())); + } + } +#endif // We need a memory manager to allocate memory and resolve symbols for this // new module. Create one that resolves symbols by looking back into the // JIT. @@ -350,7 +370,7 @@ class JuliaOJIT { [](const std::string &S) { return nullptr; } ); SmallVector,1> Ms; - Ms.push_back(std::unique_ptr{M}); + Ms.push_back(std::move(M)); return CompileLayer->addModuleSet(std::move(Ms), MemMgr, std::move(Resolver)); @@ -362,9 +382,9 @@ class JuliaOJIT { { if (ExportedSymbolsOnly) { // Step 1: Check against list of known external globals - GlobalSymbolTableT::const_iterator pos = GlobalSymbolTable.find(Name); - if (pos != GlobalSymbolTable.end()) - return orc::JITSymbol((uintptr_t)pos->second, JITSymbolFlags::Exported); + void *Addr = getPointerToGlobalIfAvailable(Name); + if (Addr != nullptr) + return orc::JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); } // Step 2: Search all previously emitted symbols return CompileLayer->findSymbol(Name, ExportedSymbolsOnly); @@ -372,17 +392,17 @@ class JuliaOJIT { orc::JITSymbol findUnmangledSymbol(const std::string Name) { - return findSymbol(mangle(Name), true); + return findSymbol(getMangledName(Name), true); } uint64_t getGlobalValueAddress(const std::string &Name) { - return findSymbol(mangle(Name), false).getAddress(); + return findSymbol(getMangledName(Name), false).getAddress(); } uint64_t getFunctionAddress(const std::string &Name) { - return findSymbol(mangle(Name), false).getAddress(); + return findSymbol(getMangledName(Name), false).getAddress(); } Function *FindFunctionNamed(const std::string &Name) @@ -422,3 +442,517 @@ class JuliaOJIT { } #endif + +#ifdef USE_ORCJIT +JuliaOJIT *jl_ExecutionEngine; +#else +ExecutionEngine *jl_ExecutionEngine; +#endif + +// destructively move the contents of src into dest +// this assumes that the targets of the two modules are the same +// including the DataLayout and ModuleFlags (for example) +// and that there is no module-level assembly +static void jl_merge_module(Module *dest, std::unique_ptr src) +{ + for (Module::global_iterator I = src->global_begin(), E = src->global_end(); I != E;) { + GlobalVariable *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (sG->isDeclaration()) { + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getGlobalList().push_back(sG); + } + + for (Module::iterator I = src->begin(), E = src->end(); I != E;) { + Function *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (sG->isDeclaration()) { + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getFunctionList().push_back(sG); + } + + for (Module::alias_iterator I = src->alias_begin(), E = src->alias_end(); I != E;) { + GlobalAlias *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getAliasList().push_back(sG); + } + +// for (Module::named_metadata_iterator I = src->named_metadata_begin(), E = src->named_metadata_end(); I != E;) { +// NamedMDNode *sG = &*I; +// NamedMDNode *dG = dest->getOrInsertNamedMetadata(sG->getName()); +// ++I; +// for (NamedMDNode::op_iterator OpI = sG->op_begin(), OpE = sG->op_end(); OpI != OpE; ++OpI) { +// dG->addOperand(*OpI); +// } +// sG->eraseFromParent(); +// } +#ifdef LLVM35 + NamedMDNode *sNMD = src->getNamedMetadata("llvm.dbg.cu"); + if (sNMD) { + NamedMDNode *dNMD = dest->getOrInsertNamedMetadata("llvm.dbg.cu"); + for (NamedMDNode::op_iterator I = sNMD->op_begin(), E = sNMD->op_end(); I != E; ++I) { + dNMD->addOperand(*I); + } + } +#endif +} + +#ifdef USE_MCJIT +static StringMap module_for_fname; +static void jl_finalize_function(Function *F, Module *collector = NULL) +{ + std::unique_ptr m(module_for_fname.lookup(F->getName())); + if (m) { + std::vector tofinalize; + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) { + module_for_fname.erase(F->getName()); + } + else if (!F->isIntrinsic()) { + tofinalize.push_back(F); + } + } + for (std::vector::iterator I = tofinalize.begin(), E = tofinalize.end(); I != E; ++I) { + jl_finalize_function(*I, collector ? collector : m.get()); + } + if (collector) { + jl_merge_module(collector, std::move(m)); + } + else { +#if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) && defined(USE_MCJIT) + ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); +#endif + assert(jl_ExecutionEngine); +#if defined(LLVM36) + jl_ExecutionEngine->addModule(std::move(m)); +#else + jl_ExecutionEngine->addModule(m.release()); +#endif + } + } +} +#endif + +template // for GlobalObject's +static T *addComdat(T *G) +{ +#if defined(_OS_WINDOWS_) + if (imaging_mode && !G->isDeclaration()) { +#ifdef LLVM35 + // Add comdat information to make MSVC link.exe happy + Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); + jl_Comdat->setSelectionKind(Comdat::NoDuplicates); + G->setComdat(jl_Comdat); + // add __declspec(dllexport) to everything marked for export + if (G->getLinkage() == GlobalValue::ExternalLinkage) + G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); +#endif + } +#endif + return G; +} + +// Connect Modules via prototypes + +static GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) +{ + GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), + G->isConstant(), GlobalVariable::ExternalLinkage, + NULL, G->getName(), G->getThreadLocalMode()); + if (M) + M->getGlobalList().push_back(proto); + return proto; +} + +static Function *function_proto(Function *F, Module *M = NULL) +{ + Function *NewF = Function::Create(F->getFunctionType(), + Function::ExternalLinkage, + F->getName(), M); + NewF->setAttributes(AttributeSet()); + + // FunctionType does not include any attributes. Copy them over manually + // as codegen may make decisions based on the presence of certain attributes + NewF->copyAttributesFrom(F); + +#ifdef LLVM37 + // Declarations are not allowed to have personality routines, but + // copyAttributesFrom sets them anyway, so clear them again manually + NewF->setPersonalityFn(nullptr); +#endif + + return NewF; +} + +template +#ifdef LLVM35 +static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) +#else +static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) +#endif +{ + // cast through integer to avoid c++ pedantic warning about casting between + // data and code pointers + void *addr = (void*)(uintptr_t)_addr; +#ifdef LLVM34 + StringRef name = gv->getName(); +#ifdef _OS_WINDOWS_ + std::string imp_name; +#endif +#endif + +#ifdef _OS_WINDOWS_ + // setting JL_DLLEXPORT correctly only matters when building a binary + if (dllimport && imaging_mode) { + assert(gv->getLinkage() == GlobalValue::ExternalLinkage); +#ifdef LLVM35 + // add the __declspec(dllimport) attribute + gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); + // this will cause llvm to rename it, so we do the same + imp_name = Twine("__imp_", name).str(); + name = StringRef(imp_name); +#else + gv->setLinkage(GlobalValue::DLLImportLinkage); +#endif +#if defined(_P64) || defined(LLVM35) + // __imp_ variables are indirection pointers, so use malloc to simulate that + void **imp_addr = (void**)malloc(sizeof(void**)); + *imp_addr = addr; + addr = (void*)imp_addr; +#endif + } +#endif // _OS_WINDOWS_ + +#ifdef USE_ORCJIT + addComdat(gv); + jl_ExecutionEngine->addGlobalMapping(name, (uintptr_t)addr); +#elif defined(USE_MCJIT) + addComdat(gv); + sys::DynamicLibrary::AddSymbol(name, addr); +#else // USE_MCJIT + jl_ExecutionEngine->addGlobalMapping(gv, addr); +#endif // USE_MCJIT +} + +static std::vector jl_sysimg_gvars; +static std::vector jl_sysimg_fvars; +typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing +static std::map jl_value_to_llvm; + +static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL) +{ + PointerType *T = cast(gv->getType()->getElementType()); // pointer is the only supported type currently + + GlobalVariable *shadowvar = NULL; +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + if (imaging_mode) +#endif + shadowvar = global_proto(gv, shadow_output); + + if (shadowvar) { + shadowvar->setInitializer(ConstantPointerNull::get(T)); + shadowvar->setLinkage(GlobalVariable::InternalLinkage); + addComdat(shadowvar); + if (imaging_mode && gvarinit) { + // make the pointer valid for future sessions + jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(shadowvar, T_psize)); + jl_value_llvm gv_struct; + gv_struct.gv = global_proto(gv); + gv_struct.index = jl_sysimg_gvars.size(); + jl_value_to_llvm[gvarinit] = gv_struct; + } + } + + // make the pointer valid for this session +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + void *slot = calloc(1, sizeof(void*)); + jl_ExecutionEngine->addGlobalMapping(gv->getName(), (uintptr_t)slot); + return slot; +#else + return jl_ExecutionEngine->getPointerToGlobal(shadowvar); +#endif +} + +static void* jl_get_global(GlobalVariable *gv) +{ +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable( + jl_ExecutionEngine->getMangledName(gv)); +#else + void *p = jl_ExecutionEngine->getPointerToGlobal( + shadow_output->getNamedValue(gv->getName())); +#endif + assert(p); + return p; +} + +static void jl_add_to_shadow(Module *m) +{ +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(m, VMap)); + for (Module::iterator I = clone->begin(), E = clone->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) { + F->setLinkage(Function::InternalLinkage); + addComdat(F); + } + } +#else + std::unique_ptr clone(m); +#endif + jl_merge_module(shadow_output, std::move(clone)); +} + +#ifdef HAVE_CPUID +extern "C" { + extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); +} +#endif + +static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, + const char *sysimg_data, size_t sysimg_len) +{ + ArrayType *gvars_type = ArrayType::get(T_psize, jl_sysimg_gvars.size()); + addComdat(new GlobalVariable(*mod, + gvars_type, + true, + GlobalVariable::ExternalLinkage, + MapValue(ConstantArray::get(gvars_type, ArrayRef(jl_sysimg_gvars)), VMap), + "jl_sysimg_gvars")); + ArrayType *fvars_type = ArrayType::get(T_pvoidfunc, jl_sysimg_fvars.size()); + addComdat(new GlobalVariable(*mod, + fvars_type, + true, + GlobalVariable::ExternalLinkage, + MapValue(ConstantArray::get(fvars_type, ArrayRef(jl_sysimg_fvars)), VMap), + "jl_sysimg_fvars")); + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, globalUnique+1), + "jl_globalUnique")); +#ifdef JULIA_ENABLE_THREADING + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, jltls_states_func_idx), + "jl_ptls_states_getter_idx")); +#endif + + Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); + addComdat(new GlobalVariable(*mod, + feature_string->getType(), + true, + GlobalVariable::ExternalLinkage, + feature_string, + "jl_sysimg_cpu_target")); + +#ifdef HAVE_CPUID + // For native also store the cpuid + if (strcmp(jl_options.cpu_target,"native") == 0) { + uint32_t info[4]; + + jl_cpuid((int32_t*)info, 1); + addComdat(new GlobalVariable(*mod, + T_int64, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), + "jl_sysimg_cpu_cpuid")); + } +#endif + + if (sysimg_data) { + Constant *data = ConstantDataArray::get(jl_LLVMContext, ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); + addComdat(new GlobalVariable(*mod, data->getType(), true, + GlobalVariable::ExternalLinkage, + data, "jl_system_image_data")); + Constant *len = ConstantInt::get(T_size, sysimg_len); + addComdat(new GlobalVariable(*mod, len->getType(), true, + GlobalVariable::ExternalLinkage, + len, "jl_system_image_size")); + } +} + +static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len, + bool dump_as_bc) +{ +#ifdef JL_DEBUG_BUILD + verifyModule(*shadow_output); +#endif + +#ifdef LLVM36 + std::error_code err; + StringRef fname_ref = StringRef(fname); + raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); +#elif defined(LLVM35) + std::string err; + raw_fd_ostream OS(fname, err, sys::fs::F_None); +#else + std::string err; + raw_fd_ostream OS(fname, err); +#endif +#ifdef LLVM37 // 3.7 simplified formatted output; just use the raw stream alone + raw_fd_ostream& FOS(OS); +#else + formatted_raw_ostream FOS(OS); +#endif + + // We don't want to use MCJIT's target machine because + // it uses the large code model and we may potentially + // want less optimizations there. + Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); +#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) +#ifdef LLVM35 + TheTriple.setObjectFormat(Triple::COFF); +#else + TheTriple.setEnvironment(Triple::UnknownEnvironment); +#endif +#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) +#ifdef LLVM35 + TheTriple.setObjectFormat(Triple::MachO); +#else + TheTriple.setEnvironment(Triple::MachO); +#endif +#endif +#ifdef LLVM35 + std::unique_ptr +#else + OwningPtr +#endif + TM(jl_TargetMachine->getTarget().createTargetMachine( + TheTriple.getTriple(), + jl_TargetMachine->getTargetCPU(), + jl_TargetMachine->getTargetFeatureString(), + jl_TargetMachine->Options, +#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + Reloc::PIC_, +#else + jit_model ? Reloc::PIC_ : Reloc::Default, +#endif + jit_model ? CodeModel::JITDefault : CodeModel::Default, + CodeGenOpt::Aggressive // -O3 + )); + +#ifdef LLVM38 + legacy::PassManager PM; +#else + PassManager PM; +#endif +#ifndef LLVM37 + PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); +#else + PM.add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); +#endif + + + // set up optimization passes +#ifdef LLVM37 + // No DataLayout pass needed anymore. +#elif defined(LLVM36) + PM.add(new DataLayoutPass()); +#elif defined(LLVM35) + PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); +#else + PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); +#endif + + addOptimizationPasses(&PM); + if (!dump_as_bc) { + if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { + jl_error("Could not generate obj file for this target"); + } + } + + // now copy the module, since PM.run may modify it + ValueToValueMapTy VMap; +#ifdef LLVM38 + Module *clone = CloneModule(shadow_output, VMap).release(); +#else + Module *clone = CloneModule(shadow_output, VMap); +#endif +#ifdef LLVM37 + // Reset the target triple to make sure it matches the new target machine + clone->setTargetTriple(TM->getTargetTriple().str()); +#ifdef LLVM38 + clone->setDataLayout(TM->createDataLayout()); +#else + clone->setDataLayout(TM->getDataLayout()->getStringRepresentation()); +#endif +#endif + + // add metadata information + jl_gen_llvm_globaldata(clone, VMap, sysimg_data, sysimg_len); + + // do the actual work + PM.run(*clone); + if (dump_as_bc) + WriteBitcodeToFile(clone, FOS); + delete clone; +} + +static int32_t jl_assign_functionID(Function *functionObject, int specsig) +{ + // give the function an index in the constant lookup table + if (!imaging_mode) + return 0; + jl_sysimg_fvars.push_back(ConstantExpr::getBitCast( + shadow_output->getNamedValue(functionObject->getName()), + T_pvoidfunc)); + return jl_sysimg_fvars.size(); +} + +extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) +{ + // map a jl_value_t memory location to a GlobalVariable + std::map::iterator it; + it = jl_value_to_llvm.find(p); + if (it == jl_value_to_llvm.end()) + return 0; + return it->second.index; +} diff --git a/src/julia.h b/src/julia.h index d293159bca69b..367d7d2a6b6b8 100644 --- a/src/julia.h +++ b/src/julia.h @@ -211,6 +211,7 @@ typedef struct _jl_lambda_info_t { // with the same name as the generated functions for this linfo, suitable // for referencing in LLVM IR jl_llvm_functions_t functionObjects; + void* moduleObject; // llvm::Module* containing definition int32_t functionID; // index that this function will have in the codegen table int32_t specFunctionID; // index that this specFunction will have in the codegen table diff --git a/src/julia_internal.h b/src/julia_internal.h index 4a50cf2e682e6..72edff4c96b54 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -56,13 +56,13 @@ STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) } void jl_generate_fptr(jl_lambda_info_t *li); -void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx); +void jl_compile_linfo(jl_lambda_info_t *li); // invoke (compiling if necessary) the jlcall function pointer for a method STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs) { if (__unlikely(meth->fptr == NULL)) { - jl_compile_linfo(meth, NULL); + jl_compile_linfo(meth); jl_generate_fptr(meth); } if (meth->jlcall_api == 0) @@ -263,7 +263,7 @@ int32_t jl_get_llvm_gv(jl_value_t *p); void jl_idtable_rehash(jl_array_t **pa, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); -jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx); +jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); jl_function_t *jl_module_get_initializer(jl_module_t *m); uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); diff --git a/src/toplevel.c b/src/toplevel.c index 1eb3fe14742ac..44a010370fc5b 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -81,7 +81,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = (jl_value_t*)jl_apply_tuple_type_v(&tt, 1); - jl_get_specialization1((jl_tupletype_t*)tt, NULL); + jl_get_specialization1((jl_tupletype_t*)tt); JL_GC_POP(); } } diff --git a/test/llvmcall.jl b/test/llvmcall.jl index 90e5ccc6d3d29..2c58958d01237 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -70,19 +70,12 @@ end Int32(1), Int32(2))) == 3 # Test whether declarations work properly -# This test only work properly for llvm 3.5+ -# On llvm <3.5+ even though the the compilation fails on the first try, -# llvm still adds the intrinsice declaration to the module and subsequent calls -# are succesfull. -#if convert(VersionNumber, Base.libllvm_version) > v"3.5-" -# -#function undeclared_ceil(x::Float64) -# llvmcall("""%2 = call double @llvm.ceil.f64(double %0) -# ret double %2""", Float64, Tuple{Float64}, x) -#end -#@test_throws ErrorException undeclared_ceil(4.2) -# -#end +function undeclared_ceil(x::Float64) + llvmcall("""%2 = call double @llvm.ceil.f64(double %0) + ret double %2""", Float64, Tuple{Float64}, x) +end +@test_throws ErrorException undeclared_ceil(4.2) +@test_throws ErrorException undeclared_ceil(4.2) function declared_floor(x::Float64) llvmcall(