From 1d02b58bb2b3220f5a01d8a09c0c0c81ec71243a Mon Sep 17 00:00:00 2001 From: Dwight Guth Date: Wed, 29 Sep 2021 15:53:40 -0500 Subject: [PATCH] Use statepoints to compute stack roots when invoking koreCollect (#426) * add llvm file with opaque function definitions * add llvm pass * implement getMangledTypeStr * introduce references to address space 1 * refactor allocateTerm * add statepoint gc strategy * add noreturn to finish_rewriting * add missing typed alloction functions * add code to retrieve stack map * parse stack map when application boots * remove manual root tracking * use libunwind to walk stack for roots * wire up gc in scripts * use tailcc instead of fastcc * add gc-leaf-function attribute to tail calls * fix bug involving match reason functions * reverse order of relocations in llvm pass * fixes for llvm 10 * fix llvm pass for llvm 10 * fix bug on mac os * fix cmake * use LLVM_VERSION_MAJOR * fix mac os * link against libunwind * fix name mangling issue on mac os * fix ciscript * fix linking against libunwind on mac os * add back missing dependencies on libunwind * try to fix nix on mac os * add comments * add LLVM_LINK variable to cmake * install opaque.ll in separate directory * change llvm-kompile script to invoke llvm-link * make sure opaque functions will actually get inlined * break long line * skip constants in stack map * remove gc-leaf-function attribute from tail calls * add llvm pass to mark tail calls as gc-leaf-function * fix gc offsets * format new files * fix bug on llvm 10 * fix format * don't reverse on older llvm * use slightly lower threshold for iterated optimization on nix This lowered setting is required because nix still uses MacOS SDK 10.12 which has a bug when the binary size is too big * fix code review comments * fix llvm 12 * Fix merge conflict * Update lib/llvm/EmitGCLayoutInfo.cpp Co-authored-by: Bruce Collie * Update MarkTailCallsAsGCLeaf.cpp Co-authored-by: Bruce Collie --- CMakeLists.txt | 16 +- Dockerfile | 2 +- Dockerfile.arch | 2 +- INSTALL.md | 2 +- bin/llvm-kompile | 21 ++- bin/llvm-kompile-clang | 10 +- ciscript | 6 +- default.nix | 2 +- include/kllvm/codegen/CreateTerm.h | 24 ++- include/kllvm/codegen/Util.h | 5 + include/runtime/collect.h | 4 +- lib/CMakeLists.txt | 1 + lib/codegen/CreateTerm.cpp | 185 ++++++++++++++++------- lib/codegen/Decision.cpp | 221 ++++++++-------------------- lib/codegen/EmitConfigParser.cpp | 75 ++++++---- lib/codegen/Util.cpp | 74 +++++++++- lib/llvm/CMakeLists.txt | 14 ++ lib/llvm/EmitGCLayoutInfo.cpp | 133 +++++++++++++++++ lib/llvm/MarkTailCallsAsGCLeaf.cpp | 59 ++++++++ nix/llvm-backend.nix | 4 +- runtime/alloc/alloc.cpp | 39 +++++ runtime/collect/CMakeLists.txt | 1 + runtime/collect/StackMapParser.cpp | 75 ++++++++++ runtime/collect/collect.cpp | 63 +++++++- runtime/finish_rewriting.ll | 4 +- runtime/fresh.ll | 4 +- runtime/opaque/opaque.ll | 101 +++++++++++++ runtime/take_steps.ll | 10 +- tools/llvm-kompile-codegen/main.cpp | 9 ++ 29 files changed, 874 insertions(+), 292 deletions(-) create mode 100644 lib/llvm/CMakeLists.txt create mode 100644 lib/llvm/EmitGCLayoutInfo.cpp create mode 100644 lib/llvm/MarkTailCallsAsGCLeaf.cpp create mode 100644 runtime/collect/StackMapParser.cpp create mode 100644 runtime/opaque/opaque.ll diff --git a/CMakeLists.txt b/CMakeLists.txt index e5d8ba38f..426e6aa85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,10 +113,19 @@ install( DESTINATION lib/kllvm/llvm/main ) +install( + FILES runtime/opaque/opaque.ll + DESTINATION lib/kllvm/llvm/opaque +) + find_program(OPT opt-12) find_program(OPT opt-11) find_program(OPT opt-10) find_program(OPT opt) +find_program(LLVM_LINK llvm-link-12) +find_program(LLVM_LINK llvm-link-11) +find_program(LLVM_LINK llvm-link-10) +find_program(LLVM_LINK llvm-link) find_program(LLC llc-12) find_program(LLC llc-11) find_program(LLC llc-10) @@ -124,8 +133,11 @@ find_program(LLC llc) if(${OPT} STREQUAL "OPT-NOTFOUND") message(FATAL_ERROR "Could not find an opt binary. Is llvm installed on your PATH?") endif() -if(${LLC} STREQUAL "OPT-NOTFOUND") - message(FATAL_ERROR "Could not find an llvm binary. Is llvm installed on your PATH?") +if(${LLC} STREQUAL "LLC-NOTFOUND") + message(FATAL_ERROR "Could not find an llc binary. Is llvm installed on your PATH?") +endif() +if(${LLVM_LINK} STREQUAL "LLVM_LINK-NOTFOUND") + message(FATAL_ERROR "Could not find an llvm-link binary. Is llvm installed on your PATH?") endif() file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/bin) diff --git a/Dockerfile b/Dockerfile index cf50fe5cf..54f683b08 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ENV TZ America/Chicago ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && \ - apt-get install -y git cmake clang-10 llvm-10-tools lld-10 zlib1g-dev flex libboost-test-dev libgmp-dev libmpfr-dev libyaml-dev libjemalloc-dev curl maven pkg-config + apt-get install -y git cmake clang-10 llvm-10-tools lld-10 zlib1g-dev flex libboost-test-dev libgmp-dev libmpfr-dev libyaml-dev libjemalloc-dev libunwind-dev curl maven pkg-config ARG USER_ID=1000 ARG GROUP_ID=1000 diff --git a/Dockerfile.arch b/Dockerfile.arch index d8406aaba..efd3b2af4 100644 --- a/Dockerfile.arch +++ b/Dockerfile.arch @@ -1,7 +1,7 @@ FROM archlinux:base RUN pacman -Syyu --noconfirm && \ - pacman -S --noconfirm base-devel git cmake clang llvm lld flex boost gmp mpfr libyaml jemalloc curl maven pkg-config + pacman -S --noconfirm base-devel git cmake clang llvm lld flex boost gmp mpfr libyaml jemalloc libunwind curl maven pkg-config ARG USER_ID=1000 ARG GROUP_ID=1000 diff --git a/INSTALL.md b/INSTALL.md index d00ae1523..fdf02b514 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,7 +1,7 @@ On Ubuntu 20.04: ```shell sudo apt-get update -sudo apt-get install git cmake clang-10 llvm-10-tools lld-10 zlib1g-dev flex libboost-test-dev libgmp-dev libmpfr-dev libyaml-dev libjemalloc-dev curl maven pkg-config +sudo apt-get install git cmake clang-10 llvm-10-tools lld-10 zlib1g-dev flex libboost-test-dev libgmp-dev libmpfr-dev libyaml-dev libjemalloc-dev libunwind-dev curl maven pkg-config git clone https://github.com/kframework/llvm-backend --recursive cd llvm-backend mkdir build diff --git a/bin/llvm-kompile b/bin/llvm-kompile index 5edb5bcb1..d7e78740c 100755 --- a/bin/llvm-kompile +++ b/bin/llvm-kompile @@ -9,7 +9,9 @@ if [ $# -lt 3 ]; then fi mod="$(mktemp tmp.XXXXXXXXXX)" modopt="$(mktemp tmp.XXXXXXXXXX)" -trap "rm -rf $dt_dir $mod $modopt" INT TERM EXIT +modcombined="$(mktemp tmp.XXXXXXXXXX)" +modcombinedopt="$(mktemp tmp.XXXXXXXXXX)" +trap "rm -rf $dt_dir $mod $modopt $modcombined $modcombinedopt" INT TERM EXIT definition="$1" shift compile=true @@ -31,12 +33,25 @@ if $compile; then -g) debug=1 ;; + -O[0-3]) + opt_flags="$arg" + ;; + -O.) + echo "$0: invalid optimization level" + exit 1 + ;; *) ;; esac done + + # code generation "$(dirname "$0")"/llvm-kompile-codegen "$definition" "$dt_dir"/dt.yaml "$dt_dir" $debug > "$mod" - @OPT@ -mem2reg -tailcallelim -tailcallopt "$mod" -o "$modopt" + + # optimization + @OPT@ -load "$(dirname "$0")"/../lib/kllvm/libLLVMPass.so -mem2reg -tailcallelim $opt_flags -mark-tail-calls-as-gc-leaf -rewrite-statepoints-for-gc -dce -emit-gc-layout-info "$mod" -o "$modopt" + @LLVM_LINK@ "$modopt" "$(dirname "$0")"/../lib/kllvm/llvm/opaque/opaque.ll -o "$modcombined" + @OPT@ "$modcombined" -always-inline -o "$modcombinedopt" else main="$1" shift @@ -45,4 +60,4 @@ fi if [[ "$OSTYPE" != "darwin"* ]]; then flags=-fuse-ld=lld fi -"$(dirname "$0")"/llvm-kompile-clang "$modopt" "$main" @LLVM_KOMPILE_LTO@ -fno-stack-protector $flags "$@" +"$(dirname "$0")"/llvm-kompile-clang "$modcombinedopt" "$main" @LLVM_KOMPILE_LTO@ -fno-stack-protector $flags "$@" diff --git a/bin/llvm-kompile-clang b/bin/llvm-kompile-clang index dad2850d3..b64c35aad 100644 --- a/bin/llvm-kompile-clang +++ b/bin/llvm-kompile-clang @@ -60,7 +60,7 @@ fi if [[ "$OSTYPE" == "darwin"* ]]; then libraries="-liconv -lncurses" - flags="-L/usr/local/opt/libffi/lib -L/usr/local/lib -Wl,-u,_sort_table -I /usr/local/include" + flags="-L/usr/local/opt/libffi/lib -L/usr/local/lib -L$(dirname "@LLC@")/../lib -Wl,-u,_sort_table -I /usr/local/include" else libraries="-ltinfo" flags="-Wl,-u,sort_table" @@ -84,7 +84,7 @@ if ! $save_temps; then fi if [ "$lto" = "lto" ]; then - flags="$flags -flto -Wl,-mllvm,-tailcallopt" + flags="$flags -flto" files=("$LIBDIR"/llvm/*.ll) if ! $link; then mv "$modopt" "$output_file" @@ -96,13 +96,13 @@ else if ! $link; then modasm="$output_file" fi - run @LLC@ -tailcallopt "$modopt" -mtriple=@BACKEND_TARGET_TRIPLE@ -filetype=obj $llc_opt_flags $llc_flags -o "$modasm" + run @LLC@ "$modopt" -mtriple=@BACKEND_TARGET_TRIPLE@ -filetype=obj $llc_opt_flags $llc_flags -o "$modasm" modopt="$modasm" fi if $link; then for file in "$LIBDIR"/llvm/*.ll; do tmp="$tmpdir/`basename "$file"`.o" - run @LLC@ -tailcallopt "$file" -mtriple=@BACKEND_TARGET_TRIPLE@ -filetype=obj $llc_opt_flags $llc_flags -o "$tmp" + run @LLC@ "$file" -mtriple=@BACKEND_TARGET_TRIPLE@ -filetype=obj $llc_opt_flags $llc_flags -o "$tmp" files+=("$tmp") done fi @@ -111,7 +111,7 @@ fi if [ "$main" = "static" ]; then all_libraries= else - all_libraries="$libraries -lgmp -lmpfr -lpthread -ldl -lffi -ljemalloc" + all_libraries="$libraries -lgmp -lmpfr -lpthread -ldl -lffi -ljemalloc -lunwind" fi if $link; then diff --git a/ciscript b/ciscript index cb4f9b8ea..d822abdb8 100755 --- a/ciscript +++ b/ciscript @@ -2,9 +2,9 @@ export PATH=$HOME/.cargo/bin:$HOME/.local/bin:$PATH if [[ "$OSTYPE" == "darwin"* ]]; then - export PATH="/usr/local/opt/flex/bin:/usr/local/opt/llvm@6/bin:$PATH" - export LDFLAGS="-L/usr/local/opt/llvm@6/lib -L/usr/local/opt/flex/lib" - export CPPFLAGS="-I/usr/local/opt/llvm@6/include -I/usr/local/opt/flex/include" + export PATH="/usr/local/opt/flex/bin:/usr/local/opt/llvm@12/bin:$PATH" + export LDFLAGS="-L/usr/local/opt/llvm@12/lib -L/usr/local/opt/flex/lib" + export CPPFLAGS="-I/usr/local/opt/llvm@12/include -I/usr/local/opt/flex/include" export FLEX_EXECUTABLE="/usr/local/opt/flex/bin/flex" fi diff --git a/default.nix b/default.nix index b4695f27f..514405c97 100644 --- a/default.nix +++ b/default.nix @@ -81,7 +81,7 @@ let mkdir -p "$out/bin" cp ${llvm-backend.src}/bin/llvm-kompile-testing "$out/bin" sed -i "$out/bin/llvm-kompile-testing" \ - -e '/@PROJECT_SOURCE_DIR@/ c ${java} -jar ${jar} $definition qbaL $dt_dir 1' + -e '/@PROJECT_SOURCE_DIR@/ c ${java} -jar ${jar} $definition qbaL $dt_dir 9/10' chmod +x "$out/bin/llvm-kompile-testing" patchShebangs "$out/bin/llvm-kompile-testing" ''; diff --git a/include/kllvm/codegen/CreateTerm.h b/include/kllvm/codegen/CreateTerm.h index db8f7f3da..bd394d38a 100644 --- a/include/kllvm/codegen/CreateTerm.h +++ b/include/kllvm/codegen/CreateTerm.h @@ -22,7 +22,7 @@ class CreateTerm { llvm::Value * createHook(KORECompositePattern *hookAtt, KORECompositePattern *pattern); llvm::Value *createFunctionCall( - std::string name, KORECompositePattern *pattern, bool sret, bool fastcc); + std::string name, KORECompositePattern *pattern, bool sret, bool tailcc); llvm::Value * notInjectionCase(KORECompositePattern *constructor, llvm::Value *val); @@ -52,12 +52,12 @@ class CreateTerm { * function actually returns void and the return value is via a pointe. Note * that this can be set to true even if the function does not return a struct, * in which case its value is ignored. load: if the function returns a struct - * via sret, then if load is true, we load the value fastcc: true if we should - * use the fastcc calling convention returned from the function before + * via sret, then if load is true, we load the value tailcc: true if we should + * use the tailcc calling convention returned from the function before * returning it. */ llvm::Value *createFunctionCall( std::string name, ValueType returnCat, - const std::vector &args, bool sret, bool fastcc); + const std::vector &args, bool sret, bool tailcc); llvm::BasicBlock *getCurrentBlock() const { return CurrentBlock; } }; @@ -103,11 +103,23 @@ llvm::Type *getParamType(ValueType sort, llvm::Module *Module); void addAbort(llvm::BasicBlock *block, llvm::Module *Module); llvm::Value *allocateTerm( - llvm::Type *AllocType, llvm::BasicBlock *block, + ValueType Cat, llvm::Type *AllocType, llvm::BasicBlock *block, const char *allocFn = "koreAlloc"); llvm::Value *allocateTerm( + ValueType Cat, llvm::Type *AllocType, llvm::Value *Len, + llvm::BasicBlock *block, const char *allocFn = "koreAlloc"); +llvm::Value *allocateTermNoReloc( + llvm::Type *AllocType, llvm::BasicBlock *block, + const char *allocFn = "koreAllocAlwaysGC"); +llvm::Value *allocateTermNoReloc( llvm::Type *AllocType, llvm::Value *Len, llvm::BasicBlock *block, - const char *allocFn = "koreAlloc"); + const char *allocFn = "koreAllocAlwaysGC"); + +// see comment on runtime/opaque/opaque.ll for explanation about why these +// functions exist +llvm::Value *addrspaceCast0to1(llvm::Value *val, llvm::BasicBlock *block); +llvm::Value *addrspaceCast1to0(llvm::Value *val, llvm::BasicBlock *block); +llvm::Value *ptrToInt(llvm::Value *val, llvm::BasicBlock *block); } // namespace kllvm #endif // CREATE_TERM_H diff --git a/include/kllvm/codegen/Util.h b/include/kllvm/codegen/Util.h index 0e0ae22e6..ed5278eb7 100644 --- a/include/kllvm/codegen/Util.h +++ b/include/kllvm/codegen/Util.h @@ -1,6 +1,7 @@ #ifndef KLLVM_UTIL_H #define KLLVM_UTIL_H +#include "kllvm/ast/AST.h" #include "llvm/IR/Module.h" namespace kllvm { @@ -8,6 +9,8 @@ namespace kllvm { // Returns a reference to the function declaration for a memory allocation // function with the given name, adding a declaration to the current module if // one does not yet exist +llvm::Function * +koreHeapAlloc(ValueType Cat, std::string name, llvm::Module *module); llvm::Function *koreHeapAlloc(std::string name, llvm::Module *module); // If Value is an instance of llvm::Function, cast and return. Otherwise, print @@ -29,6 +32,8 @@ static llvm::Function *getOrInsertFunction(llvm::Module *module, Ts... Args) { llvm::StructType *getTypeByName(llvm::Module *module, std::string name); +std::string getMangledTypeStr(llvm::Type *Ty, bool &HasUnnamedType); + } // namespace kllvm #endif // KLLVM_UTIL_H diff --git a/include/runtime/collect.h b/include/runtime/collect.h index 944353b60..5e3caf77c 100644 --- a/include/runtime/collect.h +++ b/include/runtime/collect.h @@ -25,6 +25,8 @@ using map_impl = map::iterator::tree_t; using set_node = set::iterator::node_t; using set_impl = set::iterator::tree_t; +void parseStackMap(void); + extern "C" { extern size_t numBytesLiveAtCollection[1 << AGE_WIDTH]; bool during_gc(void); @@ -36,7 +38,7 @@ void migrate_map(void *m); void migrate_set(void *s); void migrate_collection_node(void **nodePtr); void setKoreMemoryFunctionsForGMP(void); -void koreCollect(void **, uint8_t, layoutitem *); +void koreCollect(void); } #ifdef GC_DBG diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 790d53728..fb4199373 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(parser) add_subdirectory(ast) add_subdirectory(codegen) +add_subdirectory(llvm) separate_arguments(LLVM_DEFINITIONS_LIST NATIVE_COMMAND ${LLVM_DEFINITIONS}) add_definitions(${LLVM_DEFINITIONS_LIST}) diff --git a/lib/codegen/CreateTerm.cpp b/lib/codegen/CreateTerm.cpp index ac02bb24a..488085645 100644 --- a/lib/codegen/CreateTerm.cpp +++ b/lib/codegen/CreateTerm.cpp @@ -97,6 +97,12 @@ target triple = "x86_64-unknown-linux-gnu" ; Interface to the configuration parser declare %block* @parseConfiguration(i8*) declare void @printConfiguration(i32, %block *) + +@__LLVM_StackMaps = external dso_local global i8, align 8 + +define i8* @getStackMap() { + ret i8* @__LLVM_StackMaps +} )LLVM"; std::unique_ptr @@ -144,7 +150,7 @@ llvm::Type *getParamType(ValueType sort, llvm::Module *Module) { switch (sort.cat) { case SortCategory::Map: case SortCategory::List: - case SortCategory::Set: type = llvm::PointerType::getUnqual(type); break; + case SortCategory::Set: type = llvm::PointerType::get(type, 1); break; default: break; } return type; @@ -156,17 +162,17 @@ llvm::Type *getValueType(ValueType sort, llvm::Module *Module) { case SortCategory::List: return getTypeByName(Module, LIST_STRUCT); case SortCategory::Set: return getTypeByName(Module, SET_STRUCT); case SortCategory::Int: - return llvm::PointerType::getUnqual(getTypeByName(Module, INT_STRUCT)); + return llvm::PointerType::get(getTypeByName(Module, INT_STRUCT), 1); case SortCategory::Float: - return llvm::PointerType::getUnqual(getTypeByName(Module, FLOAT_STRUCT)); + return llvm::PointerType::get(getTypeByName(Module, FLOAT_STRUCT), 1); case SortCategory::StringBuffer: - return llvm::PointerType::getUnqual(getTypeByName(Module, BUFFER_STRUCT)); + return llvm::PointerType::get(getTypeByName(Module, BUFFER_STRUCT), 1); case SortCategory::Bool: return llvm::Type::getInt1Ty(Module->getContext()); case SortCategory::MInt: return llvm::IntegerType::get(Module->getContext(), sort.bits); case SortCategory::Symbol: case SortCategory::Variable: - return llvm::PointerType::getUnqual(getTypeByName(Module, BLOCK_STRUCT)); + return llvm::PointerType::get(getTypeByName(Module, BLOCK_STRUCT), 1); case SortCategory::Uncomputed: abort(); } } @@ -203,21 +209,83 @@ llvm::Value *getBlockHeader( llvm::Type::getInt64Ty(Module->getContext()), headerVal)); } +static llvm::Value * +addrspaceCast(llvm::Value *val, llvm::BasicBlock *block, int from, int to) { + auto AllocType = val->getType()->getPointerElementType(); + bool hasUnnamed = false; + std::string name = "addrspace_" + std::to_string(from) + "_to_" + + std::to_string(to) + "_" + + getMangledTypeStr(AllocType, hasUnnamed); + auto C = getOrInsertFunction( + block->getModule(), name, llvm::PointerType::get(AllocType, to), + llvm::PointerType::get(AllocType, from)); + auto F = llvm::cast(C); + F->addFnAttr(llvm::Attribute::AlwaysInline); + F->addFnAttr("gc-leaf-function"); + auto addrspace = llvm::CallInst::Create(F, {val}, "", block); + return addrspace; +} + +llvm::Value *addrspaceCast0to1(llvm::Value *val, llvm::BasicBlock *block) { + return addrspaceCast(val, block, 0, 1); +} + +llvm::Value *addrspaceCast1to0(llvm::Value *val, llvm::BasicBlock *block) { + return addrspaceCast(val, block, 1, 0); +} + +llvm::Value *ptrToInt(llvm::Value *val, llvm::BasicBlock *block) { + bool hasUnnamed = false; + std::string name + = "ptrtoint_i64." + getMangledTypeStr(val->getType(), hasUnnamed); + auto C = getOrInsertFunction( + block->getModule(), name, + llvm::Type::getInt64Ty(block->getModule()->getContext()), val->getType()); + auto F = llvm::cast(C); + F->addFnAttr(llvm::Attribute::AlwaysInline); + F->addFnAttr("gc-leaf-function"); + auto ptr = llvm::CallInst::Create(F, {val}, "", block); + return ptr; +} + llvm::Value *allocateTerm( - llvm::Type *AllocType, llvm::BasicBlock *block, const char *allocFn) { + ValueType Cat, llvm::Type *AllocType, llvm::BasicBlock *block, + const char *allocFn) { return allocateTerm( - AllocType, llvm::ConstantExpr::getSizeOf(AllocType), block, allocFn); + Cat, AllocType, llvm::ConstantExpr::getSizeOf(AllocType), block, allocFn); } llvm::Value *allocateTerm( + ValueType Cat, llvm::Type *AllocType, llvm::Value *Len, + llvm::BasicBlock *block, const char *allocFn) { + auto malloc = llvm::CallInst::Create( + koreHeapAlloc(Cat, allocFn, block->getModule()), Len, "malloccall", + block); + if (malloc->getType()->getPointerElementType() == AllocType) { + return malloc; + } + auto cast = new llvm::BitCastInst( + malloc, llvm::PointerType::get(AllocType, 1), "", block); + return cast; +} + +llvm::Value *allocateTermNoReloc( + llvm::Type *AllocType, llvm::BasicBlock *block, const char *allocFn) { + return allocateTermNoReloc( + AllocType, llvm::ConstantExpr::getSizeOf(AllocType), block, allocFn); +} + +llvm::Value *allocateTermNoReloc( llvm::Type *AllocType, llvm::Value *Len, llvm::BasicBlock *block, const char *allocFn) { - llvm::Instruction *Malloc = llvm::CallInst::CreateMalloc( - block, llvm::Type::getInt64Ty(block->getContext()), AllocType, Len, - nullptr, koreHeapAlloc(allocFn, block->getModule())); - setDebugLoc(&block->getInstList().back()); - block->getInstList().push_back(Malloc); - return Malloc; + auto malloc = llvm::CallInst::Create( + koreHeapAlloc(allocFn, block->getModule()), Len, "malloccall", block); + if (malloc->getType()->getPointerElementType() == AllocType) { + return malloc; + } + auto cast = new llvm::BitCastInst( + malloc, llvm::PointerType::get(AllocType, 0), "", block); + return cast; } ValueType termType( @@ -319,8 +387,10 @@ llvm::Value *CreateTerm::createToken(ValueType sort, std::string contents) { std::vector Idxs = {llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), 0), llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1)}; - return llvm::ConstantExpr::getInBoundsGetElementPtr( - getTypeByName(Module, INT_WRAPPER_STRUCT), globalVar, Idxs); + return addrspaceCast0to1( + llvm::ConstantExpr::getInBoundsGetElementPtr( + getTypeByName(Module, INT_WRAPPER_STRUCT), globalVar, Idxs), + CurrentBlock); } case SortCategory::Float: { llvm::Constant *global = Module->getOrInsertGlobal( @@ -408,8 +478,10 @@ llvm::Value *CreateTerm::createToken(ValueType sort, std::string contents) { std::vector Idxs = {llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), 0), llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1)}; - return llvm::ConstantExpr::getInBoundsGetElementPtr( - getTypeByName(Module, FLOAT_WRAPPER_STRUCT), globalVar, Idxs); + return addrspaceCast0to1( + llvm::ConstantExpr::getInBoundsGetElementPtr( + getTypeByName(Module, FLOAT_WRAPPER_STRUCT), globalVar, Idxs), + CurrentBlock); } case SortCategory::StringBuffer: assert(false && "not implemented yet: tokens"); @@ -447,8 +519,7 @@ llvm::Value *CreateTerm::createToken(ValueType sort, std::string contents) { llvm::ConstantDataArray::getString(Ctx, contents, false))); } return llvm::ConstantExpr::getPointerCast( - global, - llvm::PointerType::getUnqual(getTypeByName(Module, BLOCK_STRUCT))); + global, llvm::PointerType::get(getTypeByName(Module, BLOCK_STRUCT), 1)); } case SortCategory::Uncomputed: abort(); } @@ -586,10 +657,10 @@ llvm::Value *CreateTerm::createHook( if (nwords == 0) { return createToken({SortCategory::Int, 0}, "0"); } - auto Ptr = allocateTerm( + auto Ptr = allocateTermNoReloc( llvm::Type::getInt64Ty(Ctx), llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), nwords * 8), - CurrentBlock, "koreAllocAlwaysGC"); + CurrentBlock); if (nwords == 1) { llvm::Value *Word; if (cat.bits == 64) { @@ -636,10 +707,10 @@ llvm::Value *CreateTerm::createHook( if (nwords == 0) { return createToken({SortCategory::Int, 0}, "0"); } - auto Ptr = allocateTerm( + auto Ptr = allocateTermNoReloc( llvm::Type::getInt64Ty(Ctx), llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), nwords * 8), - CurrentBlock, "koreAllocAlwaysGC"); + CurrentBlock); if (nwords == 1) { llvm::Value *Word; if (cat.bits == 64) { @@ -859,11 +930,11 @@ llvm::Value *CreateTerm::createHook( } } -// we use fastcc calling convention for apply_rule_* and eval_* functions so -// that the -tailcallopt LLVM pass can be used to make K functions tail -// recursive when their K definitions are tail recursive. +// we use tailcc calling convention for apply_rule_* and eval_* functions so +// that we can add the musttail marker to these calls so the calls become tail +// calls llvm::Value *CreateTerm::createFunctionCall( - std::string name, KORECompositePattern *pattern, bool sret, bool fastcc) { + std::string name, KORECompositePattern *pattern, bool sret, bool tailcc) { std::vector args; auto returnSort = dynamic_cast( pattern->getConstructor()->getSort().get()); @@ -889,12 +960,12 @@ llvm::Value *CreateTerm::createFunctionCall( default: args.push_back(arg); break; } } - return createFunctionCall(name, returnCat, args, sret, fastcc); + return createFunctionCall(name, returnCat, args, sret, tailcc); } llvm::Value *CreateTerm::createFunctionCall( std::string name, ValueType returnCat, - const std::vector &args, bool sret, bool fastcc) { + const std::vector &args, bool sret, bool tailcc) { llvm::Type *returnType = getValueType(returnCat, Module); std::vector types; bool collection = false; @@ -914,13 +985,14 @@ llvm::Value *CreateTerm::createFunctionCall( if (sret) { // we don't use alloca here because the tail call optimization pass for llvm // doesn't handle correctly functions with alloca - AllocSret = allocateTerm(returnType, CurrentBlock, "koreAllocAlwaysGC"); + AllocSret = allocateTerm( + returnCat, returnType, CurrentBlock, "koreAllocAlwaysGC"); sretType = returnType; realArgs.insert(realArgs.begin(), AllocSret); types.insert(types.begin(), AllocSret->getType()); returnType = llvm::Type::getVoidTy(Ctx); } else if (collection) { - returnType = llvm::PointerType::getUnqual(returnType); + returnType = llvm::PointerType::get(returnType, 1); } llvm::FunctionType *funcType @@ -928,8 +1000,8 @@ llvm::Value *CreateTerm::createFunctionCall( llvm::Function *func = getOrInsertFunction(Module, name, funcType); auto call = llvm::CallInst::Create(func, realArgs, "", CurrentBlock); setDebugLoc(call); - if (fastcc) { - call->setCallingConv(llvm::CallingConv::Fast); + if (tailcc) { + call->setCallingConv(llvm::CallingConv::Tail); } if (sret) { #if LLVM_VERSION_MAJOR >= 12 @@ -966,7 +1038,7 @@ llvm::Value *CreateTerm::notInjectionCase( ChildValue = (*this)(child.get()).first; } llvm::Type *ChildPtrType - = llvm::PointerType::get(BlockType->elements()[idx], 0); + = llvm::PointerType::get(BlockType->elements()[idx], 1); if (ChildValue->getType() == ChildPtrType) { ChildValue = new llvm::LoadInst( ChildValue->getType()->getPointerElementType(), ChildValue, "", @@ -975,7 +1047,8 @@ llvm::Value *CreateTerm::notInjectionCase( children.push_back(ChildValue); idx++; } - llvm::Value *Block = allocateTerm(BlockType, CurrentBlock); + llvm::Value *Block + = allocateTerm({SortCategory::Symbol, 0}, BlockType, CurrentBlock); llvm::Value *BlockHeaderPtr = llvm::GetElementPtrInst::CreateInBounds( BlockType, Block, {llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), 0), @@ -993,7 +1066,7 @@ llvm::Value *CreateTerm::notInjectionCase( } auto BlockPtr - = llvm::PointerType::getUnqual(getTypeByName(Module, BLOCK_STRUCT)); + = llvm::PointerType::get(getTypeByName(Module, BLOCK_STRUCT), 1); auto bitcast = new llvm::BitCastInst(Block, BlockPtr, "", CurrentBlock); if (symbolDecl->getAttributes().count("binder")) { auto call = llvm::CallInst::Create( @@ -1047,11 +1120,15 @@ std::pair CreateTerm::operator()(KOREPattern *pattern) { } } else if (symbol->getArguments().empty()) { llvm::StructType *BlockType = getTypeByName(Module, BLOCK_STRUCT); - llvm::IntToPtrInst *Cast = new llvm::IntToPtrInst( - llvm::ConstantInt::get( + llvm::Value *Cast = llvm::CallInst::Create( + getOrInsertFunction( + Module, "inttoptr_i64.p1s_blocks", + llvm::PointerType::get(BlockType, 1), + llvm::Type::getInt64Ty(Ctx)), + {llvm::ConstantInt::get( llvm::Type::getInt64Ty(Ctx), - (((uint64_t)symbol->getTag()) << 32) | 1), - llvm::PointerType::getUnqual(BlockType), "", CurrentBlock); + (((uint64_t)symbol->getTag()) << 32) | 1)}, + "", CurrentBlock); return std::make_pair(Cast, false); } else if ( symbolDecl->getAttributes().count("sortInjection") @@ -1129,7 +1206,7 @@ void addAbort(llvm::BasicBlock *block, llvm::Module *Module) { bool makeFunction( std::string name, KOREPattern *pattern, KOREDefinition *definition, - llvm::Module *Module, bool fastcc, bool bigStep, + llvm::Module *Module, bool tailcc, bool bigStep, KOREAxiomDeclaration *axiom, std::string postfix) { std::map vars; pattern->markVariables(vars); @@ -1154,7 +1231,7 @@ bool makeFunction( case SortCategory::Map: case SortCategory::List: case SortCategory::Set: - paramType = llvm::PointerType::getUnqual(paramType); + paramType = llvm::PointerType::get(paramType, 1); break; default: break; } @@ -1169,13 +1246,14 @@ bool makeFunction( case SortCategory::Map: case SortCategory::List: case SortCategory::Set: - returnType = llvm::PointerType::getUnqual(returnType); + returnType = llvm::PointerType::get(returnType, 1); break; default: break; } llvm::FunctionType *funcType = llvm::FunctionType::get(returnType, paramTypes, false); llvm::Function *applyRule = getOrInsertFunction(Module, name, funcType); + applyRule->setGC("statepoint-example"); initDebugAxiom(axiom->getAttributes()); std::string debugName = name; if (axiom->getAttributes().count("label")) { @@ -1187,8 +1265,8 @@ bool makeFunction( debugName, debugName, getDebugFunctionType(getDebugType(returnCat, Out.str()), debugArgs), definition, applyRule); - if (fastcc) { - applyRule->setCallingConv(llvm::CallingConv::Fast); + if (tailcc) { + applyRule->setCallingConv(llvm::CallingConv::Tail); } llvm::StringMap subst; llvm::BasicBlock *block @@ -1206,9 +1284,10 @@ bool makeFunction( CreateTerm creator = CreateTerm(subst, definition, block, Module, false); llvm::Value *retval = creator(pattern).first; if (funcType->getReturnType() - == llvm::PointerType::getUnqual(retval->getType())) { + == llvm::PointerType::get(retval->getType(), 1)) { auto tempAlloc = allocateTerm( - retval->getType(), creator.getCurrentBlock(), "koreAllocAlwaysGC"); + returnCat, retval->getType(), creator.getCurrentBlock(), + "koreAllocAlwaysGC"); new llvm::StoreInst(retval, tempAlloc, creator.getCurrentBlock()); retval = tempAlloc; } @@ -1219,7 +1298,7 @@ bool makeFunction( auto call = llvm::CallInst::Create(step, {retval}, "", creator.getCurrentBlock()); setDebugLoc(call); - call->setCallingConv(llvm::CallingConv::Fast); + call->setCallingConv(llvm::CallingConv::Tail); retval = call; } auto ret = llvm::ReturnInst::Create( @@ -1269,7 +1348,7 @@ std::string makeApplyRuleFunction( case SortCategory::Map: case SortCategory::List: case SortCategory::Set: - paramType = llvm::PointerType::getUnqual(paramType); + paramType = llvm::PointerType::get(paramType, 1); break; default: break; } @@ -1287,6 +1366,7 @@ std::string makeApplyRuleFunction( false, axiom, ".rhs"); llvm::Function *applyRule = getOrInsertFunction(Module, name, funcType); + applyRule->setGC("statepoint-example"); initDebugAxiom(axiom->getAttributes()); initDebugFunction( name, name, @@ -1294,7 +1374,7 @@ std::string makeApplyRuleFunction( getDebugType({SortCategory::Symbol, 0}, "SortGeneratedTopCell{}"), debugArgs), definition, applyRule); - applyRule->setCallingConv(llvm::CallingConv::Fast); + applyRule->setCallingConv(llvm::CallingConv::Tail); llvm::StringMap subst; llvm::BasicBlock *block = llvm::BasicBlock::Create(Module->getContext(), "entry", applyRule); @@ -1322,7 +1402,8 @@ std::string makeApplyRuleFunction( case SortCategory::Set: if (!arg->getType()->isPointerTy()) { auto ptr = allocateTerm( - arg->getType(), creator.getCurrentBlock(), "koreAllocAlwaysGC"); + cat, arg->getType(), creator.getCurrentBlock(), + "koreAllocAlwaysGC"); new llvm::StoreInst(arg, ptr, creator.getCurrentBlock()); arg = ptr; } @@ -1339,7 +1420,7 @@ std::string makeApplyRuleFunction( auto retval = llvm::CallInst::Create(step, args, "", creator.getCurrentBlock()); setDebugLoc(retval); - retval->setCallingConv(llvm::CallingConv::Fast); + retval->setCallingConv(llvm::CallingConv::Tail); llvm::ReturnInst::Create( Module->getContext(), retval, creator.getCurrentBlock()); return name; diff --git a/lib/codegen/Decision.cpp b/lib/codegen/Decision.cpp index d27316f14..0bf433e05 100644 --- a/lib/codegen/Decision.cpp +++ b/lib/codegen/Decision.cpp @@ -29,7 +29,7 @@ void Decision::operator()(DecisionNode *entry) { "_1", getValueType({SortCategory::Symbol, 0}, Module))); FailSubject->addIncoming( new llvm::BitCastInst( - val, llvm::Type::getInt8PtrTy(Ctx), "", CurrentBlock), + val, llvm::Type::getInt8PtrTy(Ctx, 1), "", CurrentBlock), CurrentBlock); FailPattern->addIncoming( stringLiteral("\\bottom{SortGeneratedTopCell{}}()"), CurrentBlock); @@ -44,10 +44,11 @@ void Decision::operator()(DecisionNode *entry) { llvm::Value *Decision::ptrTerm(llvm::Value *val) { if (val->getType()->isIntegerTy()) { - val = allocateTerm(val->getType(), CurrentBlock, "koreAllocAlwaysGC"); + val = allocateTermNoReloc(val->getType(), CurrentBlock); + val = addrspaceCast0to1(val, CurrentBlock); } return new llvm::BitCastInst( - val, llvm::Type::getInt8PtrTy(Ctx), "", CurrentBlock); + val, llvm::Type::getInt8PtrTy(Ctx, 1), "", CurrentBlock); } bool DecisionNode::beginNode(Decision *d, std::string name) { @@ -154,8 +155,7 @@ void SwitchNode::codegen(Decision *d) { } } if (isCheckNull) { - auto cast = new llvm::PtrToIntInst( - val, llvm::Type::getInt64Ty(d->Ctx), "", d->CurrentBlock); + auto cast = ptrToInt(val, d->CurrentBlock); auto cmp = new llvm::ICmpInst( *d->CurrentBlock, llvm::CmpInst::ICMP_NE, cast, llvm::ConstantExpr::getPtrToInt( @@ -214,7 +214,7 @@ void SwitchNode::codegen(Decision *d) { llvm::StructType *BlockType = getBlockType(d->Module, d->Definition, _case.getConstructor()); llvm::BitCastInst *Cast = new llvm::BitCastInst( - val, llvm::PointerType::getUnqual(BlockType), "", d->CurrentBlock); + val, llvm::PointerType::get(BlockType, 1), "", d->CurrentBlock); KORESymbolDeclaration *symbolDecl = d->Definition->getSymbolDeclarations().at( _case.getConstructor()->getName()); @@ -240,8 +240,8 @@ void SwitchNode::codegen(Decision *d) { binding.first.substr(0, max_name_length), d->CurrentBlock); break; } - auto BlockPtr = llvm::PointerType::getUnqual( - getTypeByName(d->Module, BLOCK_STRUCT)); + auto BlockPtr + = llvm::PointerType::get(getTypeByName(d->Module, BLOCK_STRUCT), 1); if (symbolDecl->getAttributes().count("binder")) { if (offset == 0) { Renamed = llvm::CallInst::Create( @@ -360,9 +360,11 @@ void FunctionNode::codegen(Decision *d) { std::vector functionArgs; functionArgs.push_back(d->stringLiteral(debugName)); functionArgs.push_back(d->stringLiteral(function)); - functionArgs.push_back(d->ptrTerm(Call)); + functionArgs.push_back( + addrspaceCast1to0(d->ptrTerm(Call), d->CurrentBlock)); for (auto arg : args) { - functionArgs.push_back(d->ptrTerm(arg)); + functionArgs.push_back( + addrspaceCast1to0(d->ptrTerm(arg), d->CurrentBlock)); } functionArgs.push_back( llvm::ConstantPointerNull::get(llvm::Type::getInt8PtrTy(d->Ctx))); @@ -393,8 +395,7 @@ void MakeIteratorNode::codegen(Decision *d) { args.push_back(arg); types.push_back(arg->getType()); llvm::Type *sretType = getTypeByName(d->Module, "iter"); - llvm::Value *AllocSret - = allocateTerm(sretType, d->CurrentBlock, "koreAllocAlwaysGC"); + llvm::Value *AllocSret = allocateTermNoReloc(sretType, d->CurrentBlock); AllocSret->setName(name.substr(0, max_name_length)); args.insert(args.begin(), AllocSret); types.insert(types.begin(), AllocSret->getType()); @@ -465,7 +466,7 @@ void LeafNode::codegen(Decision *d) { d->Module, name, llvm::FunctionType::get(type, types, false)), args, "", d->CurrentBlock); setDebugLoc(Call); - Call->setCallingConv(llvm::CallingConv::Fast); + Call->setCallingConv(llvm::CallingConv::Tail); if (child == nullptr) { llvm::ReturnInst::Create(d->Ctx, Call, d->CurrentBlock); } else { @@ -475,8 +476,7 @@ void LeafNode::codegen(Decision *d) { llvm::FunctionType::get( llvm::Type::getVoidTy(d->Ctx), {type, - llvm::PointerType::getUnqual( - llvm::PointerType::getUnqual(type)), + llvm::PointerType::getUnqual(llvm::PointerType::get(type, 1)), llvm::Type::getInt64PtrTy(d->Ctx), llvm::Type::getInt64PtrTy(d->Ctx)}, false)), @@ -608,7 +608,7 @@ void makeEvalOrAnywhereFunction( case SortCategory::Map: case SortCategory::List: case SortCategory::Set: - args.push_back(llvm::PointerType::getUnqual(getValueType(cat, module))); + args.push_back(llvm::PointerType::get(getValueType(cat, module), 1)); cats.push_back(cat); break; default: @@ -623,13 +623,14 @@ void makeEvalOrAnywhereFunction( function->print(Out2, 0, false); std::string name = "eval_" + Out2.str(); llvm::Function *matchFunc = getOrInsertFunction(module, name, funcType); + matchFunc->setGC("statepoint-example"); KORESymbolDeclaration *symbolDecl = definition->getSymbolDeclarations().at(function->getName()); initDebugAxiom(symbolDecl->getAttributes()); initDebugFunction( function->getName(), name, getDebugFunctionType(debugReturnType, debugArgs), definition, matchFunc); - matchFunc->setCallingConv(llvm::CallingConv::Fast); + matchFunc->setCallingConv(llvm::CallingConv::Tail); llvm::BasicBlock *block = llvm::BasicBlock::Create(module->getContext(), "entry", matchFunc); llvm::BasicBlock *stuck @@ -669,16 +670,25 @@ void abortWhenStuck( auto BlockType = getBlockType(Module, d, symbol); llvm::Value *Ptr; auto BlockPtr - = llvm::PointerType::getUnqual(getTypeByName(Module, BLOCK_STRUCT)); + = llvm::PointerType::get(getTypeByName(Module, BLOCK_STRUCT), 1); if (symbol->getArguments().empty()) { - Ptr = llvm::ConstantExpr::getIntToPtr( - llvm::ConstantInt::get( + auto C = getOrInsertFunction( + Module, "inttoptr_i64.p1s_blocks", + getValueType({SortCategory::Symbol, 0}, Module), + llvm::Type::getInt64Ty(Ctx)); + auto F = llvm::cast(C); + F->addFnAttr(llvm::Attribute::AlwaysInline); + F->addFnAttr("gc-leaf-function"); + Ptr = llvm::CallInst::Create( + F, + {llvm::ConstantInt::get( llvm::Type::getInt64Ty(Ctx), - ((uint64_t)symbol->getTag() << 32 | 1)), - getValueType({SortCategory::Symbol, 0}, Module)); + (((uint64_t)symbol->getTag()) << 32) | 1)}, + "", CurrentBlock); } else { llvm::Value *BlockHeader = getBlockHeader(Module, d, symbol, BlockType); - llvm::Value *Block = allocateTerm(BlockType, CurrentBlock); + llvm::Value *Block + = allocateTerm({SortCategory::Symbol, 0}, BlockType, CurrentBlock); llvm::Value *BlockHeaderPtr = llvm::GetElementPtrInst::CreateInBounds( BlockType, Block, {llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), 0), @@ -706,11 +716,13 @@ void abortWhenStuck( } Ptr = new llvm::BitCastInst(Block, BlockPtr, "", CurrentBlock); } + auto FinishRewriting = getOrInsertFunction( + Module, "finish_rewriting", llvm::Type::getVoidTy(Ctx), BlockPtr, + llvm::Type::getInt1Ty(Ctx)); + FinishRewriting->setDoesNotReturn(); llvm::CallInst::Create( - getOrInsertFunction( - Module, "finish_rewriting", llvm::Type::getVoidTy(Ctx), BlockPtr, - llvm::Type::getInt1Ty(Ctx)), - {Ptr, llvm::ConstantInt::getTrue(Ctx)}, "", CurrentBlock); + FinishRewriting, {Ptr, llvm::ConstantInt::getTrue(Ctx)}, "", + CurrentBlock); new llvm::UnreachableInst(Ctx, CurrentBlock); } @@ -748,7 +760,8 @@ void addOwise( case SortCategory::Set: if (retval->getType() == returnType) { auto tempAlloc = allocateTerm( - retval->getType(), creator.getCurrentBlock(), "koreAllocAlwaysGC"); + returnSort, retval->getType(), creator.getCurrentBlock(), + "koreAllocAlwaysGC"); new llvm::StoreInst(retval, tempAlloc, creator.getCurrentBlock()); retval = tempAlloc; } @@ -790,141 +803,21 @@ std::pair, llvm::BasicBlock *> stepFunctionHeader( module->getContext(), "step", block->getParent()); llvm::BranchInst::Create(collect, merge, isCollection, checkCollect); - unsigned nroots = 0; - unsigned i = 0; - std::vector ptrTypes; - std::vector roots; - for (auto type : types) { - switch (type.cat) { - case SortCategory::Map: - case SortCategory::List: - case SortCategory::Set: - nroots++; - ptrTypes.push_back( - llvm::PointerType::getUnqual(getValueType(type, module))); - roots.push_back(args[i]); - break; - case SortCategory::Int: - case SortCategory::Float: - case SortCategory::StringBuffer: - case SortCategory::Symbol: - case SortCategory::Variable: - nroots++; - ptrTypes.push_back(getValueType(type, module)); - roots.push_back(args[i]); - break; - case SortCategory::Bool: - case SortCategory::MInt: break; - case SortCategory::Uncomputed: abort(); - } - i++; - } - auto arr = module->getOrInsertGlobal( - "gc_roots", llvm::ArrayType::get( - llvm::Type::getInt8PtrTy(module->getContext()), 256)); - std::vector rootPtrs; - for (unsigned i = 0; i < nroots; i++) { - auto ptr = llvm::GetElementPtrInst::CreateInBounds( - llvm::dyn_cast(arr->getType())->getElementType(), - arr, - {llvm::ConstantInt::get( - llvm::Type::getInt64Ty(module->getContext()), 0), - llvm::ConstantInt::get( - llvm::Type::getInt64Ty(module->getContext()), i)}, - "", collect); - auto casted = new llvm::BitCastInst( - ptr, llvm::PointerType::getUnqual(ptrTypes[i]), "", collect); - new llvm::StoreInst(roots[i], casted, collect); - rootPtrs.push_back(casted); - } - std::vector elements; - i = 0; - for (auto cat : types) { - switch (cat.cat) { - case SortCategory::Map: - case SortCategory::List: - case SortCategory::Set: - case SortCategory::StringBuffer: - case SortCategory::Symbol: - case SortCategory::Variable: - case SortCategory::Int: - case SortCategory::Float: - elements.push_back(llvm::ConstantStruct::get( - getTypeByName(module, LAYOUTITEM_STRUCT), - llvm::ConstantInt::get( - llvm::Type::getInt64Ty(module->getContext()), i++ * 8), - llvm::ConstantInt::get( - llvm::Type::getInt16Ty(module->getContext()), - (int)cat.cat + cat.bits))); - break; - case SortCategory::Bool: - case SortCategory::MInt: break; - case SortCategory::Uncomputed: abort(); - } - } - auto layoutArr = llvm::ConstantArray::get( - llvm::ArrayType::get( - getTypeByName(module, LAYOUTITEM_STRUCT), elements.size()), - elements); - auto layout = module->getOrInsertGlobal( - "layout_item_rule_" + std::to_string(ordinal), layoutArr->getType()); - llvm::GlobalVariable *globalVar - = llvm::dyn_cast(layout); - if (!globalVar->hasInitializer()) { - globalVar->setInitializer(layoutArr); - } - auto ptrTy = llvm::PointerType::getUnqual( - llvm::ArrayType::get(getTypeByName(module, LAYOUTITEM_STRUCT), 0)); auto koreCollect = getOrInsertFunction( module, "koreCollect", llvm::FunctionType::get( - llvm::Type::getVoidTy(module->getContext()), - {arr->getType(), llvm::Type::getInt8Ty(module->getContext()), ptrTy}, - false)); - auto call = llvm::CallInst::Create( - koreCollect, - {arr, - llvm::ConstantInt::get( - llvm::Type::getInt8Ty(module->getContext()), nroots), - llvm::ConstantExpr::getBitCast(layout, ptrTy)}, - "", collect); + llvm::Type::getVoidTy(module->getContext()), {}, false)); + auto call = llvm::CallInst::Create(koreCollect, {}, "", collect); setDebugLoc(call); - i = 0; - std::vector phis; - for (auto ptr : rootPtrs) { - auto loaded = new llvm::LoadInst( - ptr->getType()->getPointerElementType(), ptr, "", collect); - auto phi = llvm::PHINode::Create(loaded->getType(), 2, "phi", merge); - phi->addIncoming(loaded, collect); - phi->addIncoming(roots[i++], checkCollect); - phis.push_back(phi); - } llvm::BranchInst::Create(merge, collect); - i = 0; - unsigned rootIdx = 0; - std::vector results; - for (auto type : types) { - switch (type.cat) { - case SortCategory::Map: - case SortCategory::List: - case SortCategory::Set: - case SortCategory::StringBuffer: - case SortCategory::Symbol: - case SortCategory::Variable: - case SortCategory::Int: - case SortCategory::Float: results.push_back(phis[rootIdx++]); break; - default: results.push_back(args[i]); - } - i++; - } - return std::make_pair(results, merge); + return std::make_pair(args, merge); } void makeStepFunction( KOREDefinition *definition, llvm::Module *module, DecisionNode *dt, bool search) { auto blockType = getValueType({SortCategory::Symbol, 0}, module); - auto bufType = llvm::PointerType::getUnqual(blockType); + auto bufType = llvm::PointerType::get(blockType, 1); auto debugType = getDebugType({SortCategory::Symbol, 0}, "SortGeneratedTopCell{}"); llvm::FunctionType *funcType; @@ -939,6 +832,7 @@ void makeStepFunction( funcType = llvm::FunctionType::get(blockType, {blockType}, false); } llvm::Function *matchFunc = getOrInsertFunction(module, name, funcType); + matchFunc->setGC("statepoint-example"); resetDebugLoc(); if (search) { initDebugFunction( @@ -952,7 +846,7 @@ void makeStepFunction( name, name, getDebugFunctionType(debugType, {debugType}), definition, matchFunc); } - matchFunc->setCallingConv(llvm::CallingConv::Fast); + matchFunc->setCallingConv(llvm::CallingConv::Tail); auto val = matchFunc->arg_begin(); llvm::BasicBlock *block = llvm::BasicBlock::Create(module->getContext(), "entry", matchFunc); @@ -974,8 +868,8 @@ void makeStepFunction( resultCapacity = new llvm::AllocaInst( llvm::Type::getInt64Ty(module->getContext()), 0, "resultCapacity", block); - llvm::Value *initialBuffer - = allocateTerm(blockType, block, "koreAllocAlwaysGC"); + llvm::Value *initialBuffer = allocateTerm( + {SortCategory::Symbol, 0}, blockType, block, "koreAllocAlwaysGC"); new llvm::StoreInst(initialBuffer, resultBuffer, block); new llvm::StoreInst( llvm::ConstantInt::get(llvm::Type::getInt64Ty(module->getContext()), 0), @@ -1025,6 +919,7 @@ void makeMatchReasonFunction( llvm::Type::getVoidTy(module->getContext()), {blockType}, false); std::string name = "match_" + std::to_string(axiom->getOrdinal()); llvm::Function *matchFunc = getOrInsertFunction(module, name, funcType); + matchFunc->setGC("statepoint-example"); std::string debugName = name; if (axiom->getAttributes().count("label")) { debugName = axiom->getStringAttribute("label") + ".match"; @@ -1036,7 +931,7 @@ void makeMatchReasonFunction( debugName, debugName, getDebugFunctionType(getVoidDebugType(), {debugType}), definition, matchFunc); - matchFunc->setCallingConv(llvm::CallingConv::Fast); + matchFunc->setCallingConv(llvm::CallingConv::Tail); auto val = matchFunc->arg_begin(); llvm::BasicBlock *block = llvm::BasicBlock::Create(module->getContext(), "entry", matchFunc); @@ -1047,7 +942,7 @@ void makeMatchReasonFunction( llvm::BasicBlock *fail = llvm::BasicBlock::Create(module->getContext(), "fail", matchFunc); llvm::PHINode *FailSubject = llvm::PHINode::Create( - llvm::Type::getInt8PtrTy(module->getContext()), 0, "subject", fail); + llvm::Type::getInt8PtrTy(module->getContext(), 1), 0, "subject", fail); llvm::PHINode *FailPattern = llvm::PHINode::Create( llvm::Type::getInt8PtrTy(module->getContext()), 0, "pattern", fail); llvm::PHINode *FailSort = llvm::PHINode::Create( @@ -1057,10 +952,10 @@ void makeMatchReasonFunction( module, "addMatchFailReason", llvm::FunctionType::get( llvm::Type::getVoidTy(module->getContext()), - {FailSubject->getType(), FailPattern->getType(), - FailSort->getType()}, + {llvm::Type::getInt8PtrTy(module->getContext()), + FailPattern->getType(), FailSort->getType()}, false)), - {FailSubject, FailPattern, FailSort}, "", fail); + {addrspaceCast1to0(FailSubject, fail), FailPattern, FailSort}, "", fail); setDebugLoc(call); llvm::AllocaInst *choiceBuffer, *choiceDepth; @@ -1131,8 +1026,7 @@ void makeStepFunction( case SortCategory::Map: case SortCategory::List: case SortCategory::Set: - argTypes.push_back( - llvm::PointerType::getUnqual(getValueType(cat, module))); + argTypes.push_back(llvm::PointerType::get(getValueType(cat, module), 1)); break; default: argTypes.push_back(getValueType(cat, module)); break; } @@ -1143,11 +1037,12 @@ void makeStepFunction( = llvm::FunctionType::get(blockType, argTypes, false); std::string name = "step_" + std::to_string(axiom->getOrdinal()); llvm::Function *matchFunc = getOrInsertFunction(module, name, funcType); + matchFunc->setGC("statepoint-example"); resetDebugLoc(); initDebugFunction( name, name, getDebugFunctionType(blockDebugType, debugTypes), definition, matchFunc); - matchFunc->setCallingConv(llvm::CallingConv::Fast); + matchFunc->setCallingConv(llvm::CallingConv::Tail); llvm::StringMap stuckSubst; llvm::BasicBlock *block diff --git a/lib/codegen/EmitConfigParser.cpp b/lib/codegen/EmitConfigParser.cpp index d5bddd7a6..82cc7d727 100644 --- a/lib/codegen/EmitConfigParser.cpp +++ b/lib/codegen/EmitConfigParser.cpp @@ -282,8 +282,8 @@ static llvm::Value *getArgValue( case SortCategory::Bool: case SortCategory::MInt: { auto cast = new llvm::BitCastInst( - arg, llvm::PointerType::getUnqual(getValueType(cat, mod)), "", - CaseBlock); + addrspaceCast0to1(arg, CaseBlock), + llvm::PointerType::get(getValueType(cat, mod), 1), "", CaseBlock); auto load = new llvm::LoadInst( cast->getType()->getPointerElementType(), cast, "", CaseBlock); arg = load; @@ -293,15 +293,17 @@ static llvm::Value *getArgValue( case SortCategory::List: case SortCategory::Set: arg = new llvm::BitCastInst( - arg, llvm::PointerType::getUnqual(getValueType(cat, mod)), "", - CaseBlock); + addrspaceCast0to1(arg, CaseBlock), + llvm::PointerType::get(getValueType(cat, mod), 1), "", CaseBlock); break; case SortCategory::Int: case SortCategory::Float: case SortCategory::StringBuffer: case SortCategory::Symbol: case SortCategory::Variable: - arg = new llvm::BitCastInst(arg, getValueType(cat, mod), "", CaseBlock); + arg = new llvm::BitCastInst( + addrspaceCast0to1(arg, CaseBlock), getValueType(cat, mod), "", + CaseBlock); break; case SortCategory::Uncomputed: abort(); } @@ -341,8 +343,11 @@ static std::pair getEval( case SortCategory::Map: case SortCategory::List: case SortCategory::Set: - retval = new llvm::BitCastInst( - result, llvm::Type::getInt8PtrTy(Ctx), "", creator.getCurrentBlock()); + retval = addrspaceCast1to0( + new llvm::BitCastInst( + result, llvm::Type::getInt8PtrTy(Ctx, 1), "", + creator.getCurrentBlock()), + creator.getCurrentBlock()); break; case SortCategory::Bool: case SortCategory::MInt: { @@ -521,14 +526,17 @@ static void emitGetToken(KOREDefinition *definition, llvm::Module *module) { } case SortCategory::Float: { llvm::Type *Float = getTypeByName(module, FLOAT_STRUCT); - llvm::Value *Term = allocateTerm(Float, CaseBlock, "koreAllocFloating"); + llvm::Value *Term + = allocateTerm(cat, Float, CaseBlock, "koreAllocFloating"); llvm::Function *InitFloat = getOrInsertFunction( module, "init_float", llvm::Type::getVoidTy(Ctx), - llvm::PointerType::getUnqual(Float), llvm::Type::getInt8PtrTy(Ctx)); + llvm::PointerType::get(Float, 1), llvm::Type::getInt8PtrTy(Ctx)); llvm::CallInst::Create( InitFloat, {Term, func->arg_begin() + 2}, "", CaseBlock); - auto cast = new llvm::BitCastInst( - Term, llvm::Type::getInt8PtrTy(Ctx), "", CaseBlock); + auto cast = addrspaceCast1to0( + new llvm::BitCastInst( + Term, llvm::Type::getInt8PtrTy(Ctx, 1), "", CaseBlock), + CaseBlock); Phi->addIncoming(cast, CaseBlock); llvm::BranchInst::Create(MergeBlock, CaseBlock); break; @@ -557,10 +565,10 @@ static void emitGetToken(KOREDefinition *definition, llvm::Module *module) { phiStr->addIncoming(Pruned, IfIsPlus); CaseBlock = ElseNoPlus; llvm::Type *Int = getTypeByName(module, INT_STRUCT); - llvm::Value *Term = allocateTerm(Int, CaseBlock, "koreAllocInteger"); + llvm::Value *Term = allocateTerm(cat, Int, CaseBlock, "koreAllocInteger"); llvm::Function *MpzInitSet = getOrInsertFunction( module, "__gmpz_init_set_str", llvm::Type::getInt32Ty(Ctx), - llvm::PointerType::getUnqual(Int), llvm::Type::getInt8PtrTy(Ctx), + llvm::PointerType::get(Int, 1), llvm::Type::getInt8PtrTy(Ctx), llvm::Type::getInt32Ty(Ctx)); auto Call = llvm::CallInst::Create( MpzInitSet, @@ -571,8 +579,10 @@ static void emitGetToken(KOREDefinition *definition, llvm::Module *module) { *CaseBlock, llvm::CmpInst::ICMP_EQ, Call, zero32); auto AbortBlock = llvm::BasicBlock::Create(Ctx, "invalid_int", func); addAbort(AbortBlock, module); - auto cast = new llvm::BitCastInst( - Term, llvm::Type::getInt8PtrTy(Ctx), "", CaseBlock); + auto cast = addrspaceCast1to0( + new llvm::BitCastInst( + Term, llvm::Type::getInt8PtrTy(Ctx, 1), "", CaseBlock), + CaseBlock); llvm::BranchInst::Create(MergeBlock, AbortBlock, icmp, CaseBlock); Phi->addIncoming(cast, CaseBlock); break; @@ -589,8 +599,9 @@ static void emitGetToken(KOREDefinition *definition, llvm::Module *module) { auto Len = llvm::BinaryOperator::Create( llvm::Instruction::Add, func->arg_begin() + 1, llvm::ConstantExpr::getSizeOf(StringType), "", CurrentBlock); - llvm::Value *Block - = allocateTerm(StringType, Len, CurrentBlock, "koreAllocToken"); + llvm::Value *Block = allocateTerm( + {SortCategory::Symbol, 0}, StringType, Len, CurrentBlock, + "koreAllocToken"); auto HdrPtr = llvm::GetElementPtrInst::CreateInBounds( Block, {zero, zero32, zero32}, "", CurrentBlock); auto BlockSize @@ -620,10 +631,14 @@ static void emitGetToken(KOREDefinition *definition, llvm::Module *module) { {zero, llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1), zero}, "", CurrentBlock); llvm::CallInst::Create( - Memcpy, {StrPtr, func->arg_begin() + 2, func->arg_begin() + 1}, "", + Memcpy, + {addrspaceCast1to0(StrPtr, CurrentBlock), func->arg_begin() + 2, + func->arg_begin() + 1}, + "", CurrentBlock); + auto cast = addrspaceCast1to0( + new llvm::BitCastInst( + Block, llvm::Type::getInt8PtrTy(Ctx, 1), "", CurrentBlock), CurrentBlock); - auto cast = new llvm::BitCastInst( - Block, llvm::Type::getInt8PtrTy(Ctx), "", CurrentBlock); llvm::BranchInst::Create(MergeBlock, CurrentBlock); Phi->addIncoming(cast, CurrentBlock); llvm::ReturnInst::Create(Ctx, Phi, MergeBlock); @@ -663,18 +678,16 @@ static void emitTraversal( Ctx, file, getValueType({SortCategory::Symbol, 0}, module), 1, 1)); argTypes.push_back(makeVisitorType( Ctx, file, - llvm::PointerType::getUnqual( - getValueType({SortCategory::Map, 0}, module)), + llvm::PointerType::get(getValueType({SortCategory::Map, 0}, module), 1), 3, 0)); argTypes.push_back(makeVisitorType( Ctx, file, - llvm::PointerType::getUnqual( - getValueType({SortCategory::List, 0}, module)), + llvm::PointerType::get( + getValueType({SortCategory::List, 0}, module), 1), 3, 0)); argTypes.push_back(makeVisitorType( Ctx, file, - llvm::PointerType::getUnqual( - getValueType({SortCategory::Set, 0}, module)), + llvm::PointerType::get(getValueType({SortCategory::Set, 0}, module), 1), 3, 0)); argTypes.push_back(makeVisitorType( Ctx, file, getValueType({SortCategory::Int, 0}, module), 1, 0)); @@ -740,8 +753,7 @@ static void getStore( int idx = 0; auto BlockType = getBlockType(module, definition, symbol); auto cast = new llvm::BitCastInst( - func->arg_begin(), llvm::PointerType::getUnqual(BlockType), "", - CaseBlock); + func->arg_begin(), llvm::PointerType::get(BlockType, 1), "", CaseBlock); for (auto &sort : symbol->getArguments()) { ValueType cat = dynamic_cast(sort.get()) ->getCategory(definition); @@ -827,8 +839,7 @@ static void getVisitor( int idx = 0; auto BlockType = getBlockType(module, definition, symbol); auto cast = new llvm::BitCastInst( - func->arg_begin(), llvm::PointerType::getUnqual(BlockType), "", - CaseBlock); + func->arg_begin(), llvm::PointerType::get(BlockType, 1), "", CaseBlock); unsigned i = 0; auto file = llvm::PointerType::getUnqual(llvm::StructType::create(Ctx, "writer")); @@ -920,10 +931,10 @@ static void getVisitor( nbits, CharPtr}, "", CaseBlock); } else { - auto Ptr = allocateTerm( + auto Ptr = allocateTermNoReloc( llvm::Type::getInt64Ty(Ctx), llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), nwords * 8), - CaseBlock, "koreAllocAlwaysGC"); + CaseBlock); if (nwords == 1) { llvm::Value *Word; if (cat.bits == 64) { diff --git a/lib/codegen/Util.cpp b/lib/codegen/Util.cpp index 989e06c4f..bdb327822 100644 --- a/lib/codegen/Util.cpp +++ b/lib/codegen/Util.cpp @@ -1,15 +1,87 @@ #include "kllvm/codegen/Util.h" +#include "kllvm/codegen/CreateTerm.h" #include "llvm/Support/Casting.h" #include "llvm/Support/raw_ostream.h" namespace kllvm { +// this is an adaptation of a function of the same name in +// llvm/lib/IR/Function.cpp that unfortunately was not made public by llvm. We +// use it to assist with the process of generating function names for +// "intrinsics" that we need to generate in order to deal with the limitations +// of the garbage collection API in llvm +std::string getMangledTypeStr(llvm::Type *Ty, bool &HasUnnamedType) { + using namespace llvm; + std::string Result; + if (PointerType *PTyp = dyn_cast(Ty)) { + Result += "p" + std::to_string(PTyp->getAddressSpace()) + + getMangledTypeStr(PTyp->getElementType(), HasUnnamedType); + } else if (ArrayType *ATyp = dyn_cast(Ty)) { + Result += "a" + std::to_string(ATyp->getNumElements()) + + getMangledTypeStr(ATyp->getElementType(), HasUnnamedType); + } else if (StructType *STyp = dyn_cast(Ty)) { + if (!STyp->isLiteral()) { + Result += "s_"; + if (STyp->hasName()) + Result += STyp->getName(); + else + HasUnnamedType = true; + } else { + Result += "sl_"; + for (auto Elem : STyp->elements()) + Result += getMangledTypeStr(Elem, HasUnnamedType); + } + // Ensure nested structs are distinguishable. + Result += "s"; + } else if (FunctionType *FT = dyn_cast(Ty)) { + Result += "f_" + getMangledTypeStr(FT->getReturnType(), HasUnnamedType); + for (size_t i = 0; i < FT->getNumParams(); i++) + Result += getMangledTypeStr(FT->getParamType(i), HasUnnamedType); + if (FT->isVarArg()) + Result += "vararg"; + // Ensure nested function types are distinguishable. + Result += "f"; + } else if (Ty) { + switch (Ty->getTypeID()) { + default: llvm_unreachable("Unhandled type"); + case Type::VoidTyID: Result += "isVoid"; break; + case Type::MetadataTyID: Result += "Metadata"; break; + case Type::HalfTyID: Result += "f16"; break; + case Type::FloatTyID: Result += "f32"; break; + case Type::DoubleTyID: Result += "f64"; break; + case Type::X86_FP80TyID: Result += "f80"; break; + case Type::FP128TyID: Result += "f128"; break; + case Type::PPC_FP128TyID: Result += "ppcf128"; break; + case Type::X86_MMXTyID: Result += "x86mmx"; break; + case Type::IntegerTyID: + Result += "i" + std::to_string(cast(Ty)->getBitWidth()); + break; + } + } + return Result; +} + +llvm::Function * +koreHeapAlloc(ValueType Cat, std::string name, llvm::Module *module) { + llvm::Type *size_type = llvm::Type::getInt64Ty(module->getContext()); + llvm::Type *Ty = getParamType(Cat, module); + auto allocType = llvm::FunctionType::get( + Ty, llvm::ArrayRef(size_type), false); + bool hasUnnamed = false; + auto result = getOrInsertFunction( + module, name + "_" + getMangledTypeStr(Ty, hasUnnamed), allocType); + result->setReturnDoesNotAlias(); + return result; +} + llvm::Function *koreHeapAlloc(std::string name, llvm::Module *module) { llvm::Type *size_type = llvm::Type::getInt64Ty(module->getContext()); auto allocType = llvm::FunctionType::get( llvm::Type::getInt8PtrTy(module->getContext()), llvm::ArrayRef(size_type), false); - return getOrInsertFunction(module, name, allocType); + auto result = getOrInsertFunction(module, name, allocType); + result->setReturnDoesNotAlias(); + return result; } llvm::Function *castToFunctionOrAbort(llvm::Value *value) { diff --git a/lib/llvm/CMakeLists.txt b/lib/llvm/CMakeLists.txt new file mode 100644 index 000000000..e6a60a529 --- /dev/null +++ b/lib/llvm/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(LLVMPass MODULE + EmitGCLayoutInfo.cpp + MarkTailCallsAsGCLeaf.cpp +) + +target_link_libraries(LLVMPass PUBLIC AST) +if (APPLE) + target_link_libraries(LLVMPass PUBLIC "-undefined dynamic_lookup") +endif() + +install( + TARGETS LLVMPass + LIBRARY DESTINATION lib/kllvm +) diff --git a/lib/llvm/EmitGCLayoutInfo.cpp b/lib/llvm/EmitGCLayoutInfo.cpp new file mode 100644 index 000000000..f9c712174 --- /dev/null +++ b/lib/llvm/EmitGCLayoutInfo.cpp @@ -0,0 +1,133 @@ +#include "kllvm/ast/AST.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Statepoint.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +using namespace llvm; +using namespace kllvm; + +#define DEBUG_TYPE "EmitGCLayoutInfo" + +namespace { +struct EmitGCLayoutInfo : public ModulePass { + static char ID; + EmitGCLayoutInfo() + : ModulePass(ID) { } + + void error(Type *Ty) { + llvm::errs() << "Could not identify garbage collection information! This " + "is a bug in the llvm backend of K\n"; + Ty->print(llvm::errs()); + abort(); + } + + bool runOnModule(Module &M) override { + unsigned int nextID = 0; + unsigned int numRelocations = 0; + std::map, ValueType> cats; + auto &Ctx = M.getContext(); + for (Function &F : M) { + for (BasicBlock &BB : F) { + for (Instruction &I : BB) { +#if LLVM_VERSION_MAJOR >= 11 + if (auto *GCSI = dyn_cast(&I)) { + auto S = GCSI; + constexpr auto IDPos = GCStatepointInst::IDPos; +#else + auto SImpl = Statepoint(&I); + if (SImpl) { + auto GCSI = SImpl.getCall(); + auto S = &SImpl; + constexpr auto IDPos = Statepoint::IDPos; +#endif + unsigned int id = nextID++; + GCSI->setArgOperand( + IDPos, + ConstantInt::get(GCSI->getArgOperand(IDPos)->getType(), id)); + unsigned int nrelocs = S->gc_args_end() - S->gc_args_begin(); + unsigned int i = 0; +#if LLVM_VERSION_MAJOR >= 11 + for (auto &R : S->getGCRelocates()) { +#else + for (auto &R : S->getRelocates()) { +#endif + auto Arg = R->getBasePtr(); + auto *Ty = Arg->getType()->getPointerElementType(); + if (Ty->isIntegerTy()) { + i++; + continue; + } + if (!Ty->isStructTy()) { + error(Ty); + } + auto StructTy = llvm::cast(Ty); + if (!StructTy->hasName()) { + error(StructTy); + } + std::string name = StructTy->getName().str(); + ValueType cat; + cat.bits = 0; + if (name == "map") { + cat.cat = SortCategory::Map; + } else if (name == "list") { + cat.cat = SortCategory::List; + } else if (name == "set") { + cat.cat = SortCategory::Set; + } else if (name == "mpz") { + cat.cat = SortCategory::Int; + } else if (name == "floating") { + cat.cat = SortCategory::Float; + } else if (name == "stringbuffer") { + cat.cat = SortCategory::StringBuffer; + } else if (name == "block") { + cat.cat = SortCategory::Symbol; + } else { + error(StructTy); + } + cats[std::make_pair(id, i)] = cat; + i++; + } + if (nrelocs > numRelocations) { + numRelocations = nrelocs; + } + } + } + } + } + + unsigned int numRecords = nextID; + std::vector table; + for (unsigned int i = 0; i < numRecords; i++) { + for (unsigned int j = 0; j < numRelocations; j++) { + table.push_back((char)cats[std::make_pair(i, j)].cat); + } + } + + auto Arr = ConstantDataArray::get(Ctx, table); + Constant *C = M.getOrInsertGlobal("gc_stackmap_layoutinfo", Arr->getType()); + auto G = cast(C); + if (!G->hasInitializer()) { + G->setInitializer(Arr); + } + Constant *C2 = M.getOrInsertGlobal( + "gc_stackmap_num_relocations", Type::getInt32Ty(Ctx)); + auto G2 = cast(C2); + if (!G2->hasInitializer()) { + G2->setInitializer( + ConstantInt::get(Type::getInt32Ty(Ctx), numRelocations)); + } + + return true; + } +}; +} // namespace + +char EmitGCLayoutInfo::ID = 0; + +static RegisterPass + X("emit-gc-layout-info", "Emit Layout Info for Garbage Collection", + false /* Only looks at CFG */, false /* Analysis Pass */); diff --git a/lib/llvm/MarkTailCallsAsGCLeaf.cpp b/lib/llvm/MarkTailCallsAsGCLeaf.cpp new file mode 100644 index 000000000..20a945195 --- /dev/null +++ b/lib/llvm/MarkTailCallsAsGCLeaf.cpp @@ -0,0 +1,59 @@ +#include "kllvm/ast/AST.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Statepoint.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +using namespace llvm; +using namespace kllvm; + +#define DEBUG_TYPE "MarkTailCallsAsGCLeaf" + +namespace { +struct MarkTailCallsAsGCLeaf : public FunctionPass { + static char ID; + MarkTailCallsAsGCLeaf() + : FunctionPass(ID) { } + + bool runOnFunction(Function &F) override { + LLVMContext &Ctx = F.getContext(); + if (F.getCallingConv() != CallingConv::Tail) { + return false; + } + bool dirty = false; + for (BasicBlock &BB : F) { + Instruction *TI = BB.getTerminator(); + if (auto R = dyn_cast(TI)) { + if (&BB.front() == TI) { + continue; + } + BasicBlock::iterator BBI(TI); + --BBI; + if (auto CI = dyn_cast(BBI)) { + Value *RV = R->getReturnValue(); + if (RV != nullptr && RV != CI) { + continue; + } + if (!CI->isTailCall() || CI->getCallingConv() != CallingConv::Tail) { + continue; + } + CI->setAttributes(CI->getAttributes().addAttribute( + Ctx, llvm::AttributeList::FunctionIndex, "gc-leaf-function")); + dirty = true; + } + } + } + return dirty; + } +}; +} // namespace + +char MarkTailCallsAsGCLeaf::ID = 0; + +static RegisterPass + X("mark-tail-calls-as-gc-leaf", + "Mark tailcc tail calls with gc-leaf-function attribute", + false /* Only looks at CFG */, false /* Analysis Pass */); diff --git a/nix/llvm-backend.nix b/nix/llvm-backend.nix index 4fc2607ca..60f791e74 100644 --- a/nix/llvm-backend.nix +++ b/nix/llvm-backend.nix @@ -2,7 +2,7 @@ lib, cleanSourceWith, src, cmake, flex, pkgconfig, llvmPackages, - boost, gmp, jemalloc, libffi, libiconv, libyaml, mpfr, ncurses, + boost, gmp, jemalloc, libffi, libiconv, libyaml, libunwind, mpfr, ncurses, # Runtime dependencies: host, # Options: @@ -34,7 +34,7 @@ stdenv.mkDerivation { nativeBuildInputs = [ cmake flex llvm pkgconfig ]; buildInputs = [ boost libyaml ]; propagatedBuildInputs = - [ gmp jemalloc libffi mpfr ncurses ] + [ gmp jemalloc libffi libunwind llvmPackages.libunwind mpfr ncurses ] ++ lib.optional stdenv.isDarwin libiconv; postPatch = '' diff --git a/runtime/alloc/alloc.cpp b/runtime/alloc/alloc.cpp index e52d3d8c7..1ad603df0 100644 --- a/runtime/alloc/alloc.cpp +++ b/runtime/alloc/alloc.cpp @@ -72,11 +72,20 @@ __attribute__((always_inline)) void *koreAlloc(size_t requested) { return arenaAlloc(&youngspace, requested); } +__attribute__((always_inline)) block *koreAlloc_p1s_blocks(size_t requested) { + return (block *)koreAlloc(requested); +} + __attribute__((always_inline)) void *koreAllocToken(size_t requested) { size_t size = (requested + 7) & ~7; return arenaAlloc(&youngspace, size < 16 ? 16 : size); } +__attribute__((always_inline)) block * +koreAllocToken_p1s_blocks(size_t requested) { + return (block *)koreAllocToken(requested); +} + __attribute__((always_inline)) void *koreAllocOld(size_t requested) { return arenaAlloc(&oldspace, requested); } @@ -90,6 +99,26 @@ __attribute__((always_inline)) void *koreAllocAlwaysGC(size_t requested) { return arenaAlloc(&alwaysgcspace, requested); } +__attribute__((always_inline)) map * +koreAllocAlwaysGC_p1s_maps(size_t requested) { + return (map *)koreAllocAlwaysGC(requested); +} + +__attribute__((always_inline)) set * +koreAllocAlwaysGC_p1s_sets(size_t requested) { + return (set *)koreAllocAlwaysGC(requested); +} + +__attribute__((always_inline)) list * +koreAllocAlwaysGC_p1s_lists(size_t requested) { + return (list *)koreAllocAlwaysGC(requested); +} + +__attribute__((always_inline)) block * +koreAllocAlwaysGC_p1s_blocks(size_t requested) { + return (block *)koreAllocAlwaysGC(requested); +} + void *koreResizeLastAlloc(void *oldptr, size_t newrequest, size_t last_size) { newrequest = (newrequest + 7) & ~7; last_size = (last_size + 7) & ~7; @@ -131,12 +160,22 @@ __attribute__((always_inline)) void *koreAllocInteger(size_t requested) { return &result->i; } +__attribute__((always_inline)) mpz_ptr +koreAllocInteger_p1s_mpzs(size_t requested) { + return (mpz_ptr)koreAllocInteger(requested); +} + __attribute__((always_inline)) void *koreAllocFloating(size_t requested) { floating_hdr *result = (floating_hdr *)koreAlloc(sizeof(floating_hdr)); set_len(result, sizeof(floating_hdr) - sizeof(blockheader)); return &result->f; } +__attribute__((always_inline)) floating * +koreAllocFloating_p1s_floatings(size_t requested) { + return (floating *)koreAllocFloating(requested); +} + __attribute__((always_inline)) void *koreAllocIntegerOld(size_t requested) { mpz_hdr *result = (mpz_hdr *)koreAllocOld(sizeof(mpz_hdr)); set_len(result, sizeof(mpz_hdr) - sizeof(blockheader)); diff --git a/runtime/collect/CMakeLists.txt b/runtime/collect/CMakeLists.txt index 06c3571c9..96682cc55 100644 --- a/runtime/collect/CMakeLists.txt +++ b/runtime/collect/CMakeLists.txt @@ -2,6 +2,7 @@ add_library(collect STATIC collect.cpp migrate_roots.cpp migrate_collection.cpp + StackMapParser.cpp ) install( diff --git a/runtime/collect/StackMapParser.cpp b/runtime/collect/StackMapParser.cpp new file mode 100644 index 000000000..bc31beef1 --- /dev/null +++ b/runtime/collect/StackMapParser.cpp @@ -0,0 +1,75 @@ +#include + +#include "runtime/collect.h" + +extern "C" { +char *getStackMap(); +extern char gc_stackmap_layoutinfo[]; +extern unsigned int gc_stackmap_num_relocations; +} + +std::map> StackMap; + +struct StackMapFunction { + uint64_t FunctionAddress; + uint64_t NumRecords; +}; + +// see https://llvm.org/docs/StackMaps.html +void parseStackMap() { + char *stackMap = getStackMap(); + char version = *stackMap; + if (version != 3) { + abort(); + } + uint32_t NumFunctions = *(uint32_t *)(stackMap + 4); + uint32_t NumConstants = *(uint32_t *)(stackMap + 8); + std::vector functions; + for (int i = 0; i < NumFunctions; i++) { + uint64_t address = *(uint64_t *)(stackMap + 16 + i * 24); + uint64_t records = *(uint64_t *)(stackMap + 32 + i * 24); + functions.push_back({address, records}); + } + char *stackMapRecord = stackMap + 16 + NumFunctions * 24 + NumConstants * 8; + for (StackMapFunction func : functions) { + for (int i = 0; i < func.NumRecords; i++) { + uint64_t StatepointId = *(uint64_t *)(stackMapRecord); + uint32_t InstructionOffset = *(uint32_t *)(stackMapRecord + 8); + uint16_t NumLocations = *(uint16_t *)(stackMapRecord + 14); + void *ip = (void *)(func.FunctionAddress + InstructionOffset); + int32_t NumDeopts = *(int32_t *)(stackMapRecord + 24 + 2 * 12); + uint16_t RelocationOffset = 0; + for (uint16_t j = 4 + NumDeopts; j < NumLocations; j += 2) { + uint8_t type = *(uint8_t *)(stackMapRecord + 16 + j * 12); + if (type == 5) { + // a ConstantOffset gc root is one which corresponds to something the + // compiler was able to statically determine was a constructor with + // zero children. Such terms do not actually live on the heap and + // thus do not need to be relocated. + RelocationOffset++; + continue; + } + if (type != 3) { + abort(); + } + int32_t offset = *(int32_t *)(stackMapRecord + 24 + j * 12); + layoutitem layout; + layout.offset = offset; + size_t layout_offset + = gc_stackmap_num_relocations * StatepointId + RelocationOffset; + layout.cat = gc_stackmap_layoutinfo[layout_offset]; + StackMap[ip].push_back(layout); + RelocationOffset++; + } + stackMapRecord += 16 + NumLocations * 12; + if (NumLocations % 2 == 1) { + stackMapRecord += 4; + } + uint16_t NumLiveOuts = *(uint16_t *)(stackMapRecord + 2); + stackMapRecord += 4 + 4 * NumLiveOuts; + if (NumLiveOuts % 2 == 0) { + stackMapRecord += 4; + } + } + } +} diff --git a/runtime/collect/collect.cpp b/runtime/collect/collect.cpp index 86e534cef..b61227050 100644 --- a/runtime/collect/collect.cpp +++ b/runtime/collect/collect.cpp @@ -1,13 +1,18 @@ -#include "runtime/collect.h" -#include "runtime/alloc.h" -#include "runtime/arena.h" -#include "runtime/header.h" #include #include #include #include #include #include +#include + +#include "runtime/alloc.h" +#include "runtime/arena.h" +#include "runtime/collect.h" +#include "runtime/header.h" + +#define UNW_LOCAL_ONLY +#include extern "C" { @@ -254,12 +259,52 @@ void initStaticObjects(void) { list l = list(); set s = set(); setKoreMemoryFunctionsForGMP(); + parseStackMap(); } -void koreCollect(void **roots, uint8_t nroots, layoutitem *typeInfo) { +struct gc_root { + void *bp; + layoutitem layout; +}; + +extern std::map> StackMap; + +static std::vector scanStackRoots(void) { + unw_cursor_t cursor; + unw_context_t context; + + unw_getcontext(&context); + unw_init_local(&cursor, &context); + + std::vector gc_roots; + + while (unw_step(&cursor)) { + unw_word_t ip_word, sp_word; + + unw_get_reg(&cursor, UNW_REG_IP, &ip_word); + unw_get_reg(&cursor, UNW_REG_SP, &sp_word); + + void *ip, *sp; + ip = (void *)ip_word; + sp = (void *)sp_word; + + if (StackMap.count(ip)) { + std::vector &Relocs = StackMap[ip]; + for (auto &Reloc : Relocs) { + gc_roots.push_back({sp, Reloc}); + } + } + } + return gc_roots; +} + +void koreCollect(void) { is_gc = true; collect_old = shouldCollectOldGen(); MEM_LOG("Starting garbage collection\n"); + + std::vector roots = scanStackRoots(); + #ifdef GC_DBG if (!last_alloc_ptr) { last_alloc_ptr = youngspace_ptr(); @@ -273,9 +318,11 @@ void koreCollect(void **roots, uint8_t nroots, layoutitem *typeInfo) { } #endif char *previous_oldspace_alloc_ptr = *old_alloc_ptr(); - for (int i = 0; i < nroots; i++) { - migrate_child(roots, typeInfo, i, true); + // migrate stack roots + for (int i = 0; i < roots.size(); i++) { + migrate_child(roots[i].bp, &roots[i].layout, 0, true); } + // migrate global variable roots migrateRoots(); char *scan_ptr = youngspace_ptr(); if (scan_ptr != *young_alloc_ptr()) { @@ -326,7 +373,7 @@ void koreCollect(void **roots, uint8_t nroots, layoutitem *typeInfo) { } void freeAllKoreMem() { - koreCollect(nullptr, 0, nullptr); + koreCollect(); } bool is_collection() { diff --git a/runtime/finish_rewriting.ll b/runtime/finish_rewriting.ll index fb468efa3..773eedace 100644 --- a/runtime/finish_rewriting.ll +++ b/runtime/finish_rewriting.ll @@ -19,7 +19,7 @@ declare i8* @getStderr() @exit_int_0 = global %mpz { i32 0, i32 0, i64* getelementptr inbounds ([0 x i64], [0 x i64]* @exit_int_0_limbs, i32 0, i32 0) } @exit_int_0_limbs = global [0 x i64] zeroinitializer -define weak fastcc %mpz* @"eval_LblgetExitCode{SortGeneratedTopCell{}}"(%block*) { +define weak tailcc %mpz* @"eval_LblgetExitCode{SortGeneratedTopCell{}}"(%block*) { ret %mpz* @exit_int_0 } @@ -48,7 +48,7 @@ printConfig: call void @printConfiguration(i8* %output, %block* %subject) br i1 %error, label %exit, label %exitCode exitCode: - %exit_z = call fastcc %mpz* @"eval_LblgetExitCode{SortGeneratedTopCell{}}"(%block* %subject) + %exit_z = call tailcc %mpz* @"eval_LblgetExitCode{SortGeneratedTopCell{}}"(%block* %subject) %exit_ul = call i64 @__gmpz_get_ui(%mpz* %exit_z) %exit_trunc = trunc i64 %exit_ul to i32 br label %exit diff --git a/runtime/fresh.ll b/runtime/fresh.ll index a29b9a2e4..ef0c0eaf4 100644 --- a/runtime/fresh.ll +++ b/runtime/fresh.ll @@ -8,7 +8,7 @@ target triple = "x86_64-unknown-linux-gnu" declare void @abort() #0 -define weak fastcc %block* @"eval_LblgetGeneratedCounterCell{SortGeneratedTopCell{}}"(%block*) { +define weak tailcc %block* @"eval_LblgetGeneratedCounterCell{SortGeneratedTopCell{}}"(%block*) { call void @abort() unreachable } @@ -23,7 +23,7 @@ declare i8* @getTerminatedString(%string*) define i8* @get_fresh_constant(%string* %sort, %block* %top) { entry: - %counterCell = call fastcc %block* @"eval_LblgetGeneratedCounterCell{SortGeneratedTopCell{}}"(%block* %top) + %counterCell = call tailcc %block* @"eval_LblgetGeneratedCounterCell{SortGeneratedTopCell{}}"(%block* %top) %counterCellPointer = getelementptr %block, %block* %counterCell, i64 0, i32 1, i64 0 %mpzPtrPtr = bitcast i64** %counterCellPointer to %mpz** %currCounter = load %mpz*, %mpz** %mpzPtrPtr diff --git a/runtime/opaque/opaque.ll b/runtime/opaque/opaque.ll new file mode 100644 index 000000000..6813344c8 --- /dev/null +++ b/runtime/opaque/opaque.ll @@ -0,0 +1,101 @@ +target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +%blockheader = type { i64 } +%mpz = type { i32, i32, i64* } +%block = type { %blockheader, [0 x i64*] } +%map = type { { i8*, i64 } } +%list = type { { i64, i32, i8*, i8* } } +%set = type { { i8*, i64 } } +%floating = type { i64, { i64, i32, i64, i64* } } +%string = type { %blockheader, [0 x i8] } + +; this file contains a set of functions templated by type that fall into 4 basic categories: +; * address space cast from addrspace(1) to addrspace(0) +; * address space cast from addrspace(0) to addrspace(1) +; * ptrtoint cast +; * inttoptr cast +; +; the reason these instructions live in separate functions is because the RewriteStatepointsForGC +; pass doesn't understand these instructions, and would optimize incorrectly if it knew these +; instructions were being called. The intent is that after performing all the regular optimization +; passes, we will use llvm-link to link this file together with the file generated by code generation +; and then run the always inline llvm pass to inline these instructions, so that at the end of the day, +; they get implemented as a no-op in the resulting binary. + +define %block addrspace(1)* addrspace(1)* @addrspace_0_to_1_p1s_blocks(%block addrspace(1)** %in) #0 { + %out = addrspacecast %block addrspace(1)** %in to %block addrspace(1)* addrspace(1)* + ret %block addrspace(1)* addrspace(1)* %out +} + +define %block addrspace(1)* @addrspace_0_to_1_s_blocks(%block* %in) #0 { + %out = addrspacecast %block* %in to %block addrspace(1)* + ret %block addrspace(1)* %out +} + +define %mpz addrspace(1)* @addrspace_0_to_1_s_mpzs(%mpz* %in) #0 { + %out = addrspacecast %mpz* %in to %mpz addrspace(1)* + ret %mpz addrspace(1)* %out +} + +define %floating addrspace(1)* @addrspace_0_to_1_s_floatings(%floating* %in) #0 { + %out = addrspacecast %floating* %in to %floating addrspace(1)* + ret %floating addrspace(1)* %out +} + +define %map addrspace(1)* @addrspace_0_to_1_s_maps(%map* %in) #0 { + %out = addrspacecast %map* %in to %map addrspace(1)* + ret %map addrspace(1)* %out +} + +define %list addrspace(1)* @addrspace_0_to_1_s_lists(%list* %in) #0 { + %out = addrspacecast %list* %in to %list addrspace(1)* + ret %list addrspace(1)* %out +} + +define %set addrspace(1)* @addrspace_0_to_1_s_sets(%set* %in) #0 { + %out = addrspacecast %set* %in to %set addrspace(1)* + ret %set addrspace(1)* %out +} + +define %string addrspace(1)* @addrspace_0_to_1_s_strings(%string* %in) #0 { + %out = addrspacecast %string* %in to %string addrspace(1)* + ret %string addrspace(1)* %out +} + +define i8 addrspace(1)* @addrspace_0_to_1_i8(i8* %in) #0 { + %out = addrspacecast i8* %in to i8 addrspace(1)* + ret i8 addrspace(1)* %out +} + +define i1 addrspace(1)* @addrspace_0_to_1_i1(i1* %in) #0 { + %out = addrspacecast i1* %in to i1 addrspace(1)* + ret i1 addrspace(1)* %out +} + +define i32 addrspace(1)* @addrspace_0_to_1_i32(i32* %in) #0 { + %out = addrspacecast i32* %in to i32 addrspace(1)* + ret i32 addrspace(1)* %out +} + +define i64 addrspace(1)* @addrspace_0_to_1_i64(i64* %in) #0 { + %out = addrspacecast i64* %in to i64 addrspace(1)* + ret i64 addrspace(1)* %out +} + +define i8* @addrspace_1_to_0_i8(i8 addrspace(1)* %in) #0 { + %out = addrspacecast i8 addrspace(1)* %in to i8* + ret i8* %out +} + +define %block addrspace(1)* @inttoptr_i64.p1s_blocks(i64 %in) #0 { + %out = inttoptr i64 %in to %block addrspace(1)* + ret %block addrspace(1)* %out +} + +define i64 @ptrtoint_i64.p1s_blocks(%block addrspace(1)* %in) #0 { + %out = ptrtoint %block addrspace(1)* %in to i64 + ret i64 %out +} + +attributes #0 = { alwaysinline "gc-leaf-function" } diff --git a/runtime/take_steps.ll b/runtime/take_steps.ll index c38365dd4..1d1c88221 100644 --- a/runtime/take_steps.ll +++ b/runtime/take_steps.ll @@ -4,16 +4,14 @@ target triple = "x86_64-unknown-linux-gnu" %blockheader = type { i64 } %block = type { %blockheader, [0 x i64 *] } ; 16-bit layout, 8-bit length, 32-bit tag, children -declare fastcc %block* @step(%block*) -declare fastcc %block** @stepAll(%block*, i64*) +declare tailcc %block* @step(%block*) +declare tailcc %block** @stepAll(%block*, i64*) @depth = thread_local global i64 zeroinitializer @steps = thread_local global i64 zeroinitializer @current_interval = thread_local global i64 0 @GC_THRESHOLD = thread_local global i64 @GC_THRESHOLD@ -@gc_roots = global [256 x i8 *] zeroinitializer - define void @set_gc_threshold(i64 %threshold) { store i64 %threshold, i64* @GC_THRESHOLD ret void @@ -43,13 +41,13 @@ else: define %block* @take_steps(i64 %depth, %block* %subject) { store i64 %depth, i64* @depth - %result = call fastcc %block* @step(%block* %subject) + %result = call tailcc %block* @step(%block* %subject) ret %block* %result } define %block** @take_search_step(%block* %subject, i64* %count) { store i64 -1, i64* @depth - %result = call fastcc %block** @stepAll(%block* %subject, i64* %count) + %result = call tailcc %block** @stepAll(%block* %subject, i64* %count) ret %block** %result } diff --git a/tools/llvm-kompile-codegen/main.cpp b/tools/llvm-kompile-codegen/main.cpp index 68808818f..39ce68c63 100644 --- a/tools/llvm-kompile-codegen/main.cpp +++ b/tools/llvm-kompile-codegen/main.cpp @@ -125,6 +125,15 @@ int main(int argc, char **argv) { } } +#ifdef __APPLE__ + // apple symbols are mangled slightly with an underscore in front, so we need + // to adjust the name of the __LLVM_StackMaps symbol slightly + auto StackMap = mod->getOrInsertGlobal( + "__LLVM_StackMaps", llvm::Type::getInt8Ty(Context)); + auto StackMapGlobal = llvm::cast(StackMap); + StackMapGlobal->setName("_LLVM_StackMaps"); +#endif + if (CODEGEN_DEBUG) { finalizeDebugInfo(); }