From ae90407140fff10c6f63cef3ae6270ae6ef91f0f Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 3 Dec 2024 00:02:50 +0000 Subject: [PATCH 1/4] Support VO bit --- src/datatype.c | 10 ++++++++++ src/gc-mmtk.c | 23 +++++++++++++--------- src/julia.h | 6 ++++++ src/llvm-late-gc-lowering-mmtk.cpp | 31 ++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index 3a2ebf2bb303a..35c9b033bbe94 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -290,6 +290,11 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, if ((void*)ret == HT_NOTFOUND) { if (!should_malloc) { char *perm_mem = (char *)jl_gc_perm_alloc(flddesc_sz, 0, 4, 0); +// #ifdef MMTK_GC +// // FIXME: Why do we need this? This is not a Julia object at all? +// jl_ptls_t ptls = jl_current_task->ptls; +// mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, jl_valueof(perm_mem), flddesc_sz); +// #endif assert(perm_mem); ret = (jl_datatype_layout_t *)perm_mem; memcpy(perm_mem, flddesc, flddesc_sz); @@ -973,6 +978,11 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name, jl_datatype_layout_t *layout = (jl_datatype_layout_t *) jl_gc_perm_alloc(sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t), 0, 4, 0); +// #ifdef MMTK_GC +// // FIXME: Why do we need this? jl_datatype_layout_t is not a Julia object? +// jl_ptls_t ptls = jl_current_task->ptls; +// mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, jl_valueof(layout), sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t)); +// #endif layout->size = large ? GC_MAX_SZCLASS+1 : 0; layout->nfields = 0; layout->alignment = sizeof(void *); diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 6519c6c34c48d..6782e0370e373 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -865,13 +865,10 @@ inline void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, siz return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 0); } -inline void mmtk_immix_post_alloc_slow(MMTkMutatorContext* mutator, void* obj, size_t size) { - mmtk_post_alloc(mutator, obj, size, 0); -} - inline void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { - // FIXME: for now, we do nothing - // but when supporting moving, this is where we set the valid object (VO) bit + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); + } } inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { @@ -879,10 +876,9 @@ inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (uintptr_t)allocator->limit, size, align, offset, 1); } -inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { - if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { +inline void mmtk_set_side_metadata(void* side_metadata_base, void* obj) { intptr_t addr = (intptr_t) obj; - uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6); + uint8_t* meta_addr = (uint8_t*) side_metadata_base + (addr >> 6); intptr_t shift = (addr >> 3) & 0b111; while(1) { uint8_t old_val = *meta_addr; @@ -891,6 +887,15 @@ inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj break; } } +} + +inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { + if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { + mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj); + } + + if (MMTK_NEEDS_VO_BIT) { + mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj); } } diff --git a/src/julia.h b/src/julia.h index 27d9331944f9d..b81d3bec3889a 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2748,7 +2748,13 @@ extern void mmtk_object_reference_write_slow(void* mutator, const void* parent, #define MMTK_DEFAULT_IMMIX_ALLOCATOR (0) #define MMTK_IMMORTAL_BUMP_ALLOCATOR (0) +// VO bit is required to support conservative stack scanning and moving. +#define MMTK_NEEDS_VO_BIT (1) + +void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size); + extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS; +extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS; // Directly call into MMTk for write barrier (debugging only) STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSAFEPOINT diff --git a/src/llvm-late-gc-lowering-mmtk.cpp b/src/llvm-late-gc-lowering-mmtk.cpp index 6342fc9883845..ca333305c631a 100644 --- a/src/llvm-late-gc-lowering-mmtk.cpp +++ b/src/llvm-late-gc-lowering-mmtk.cpp @@ -132,6 +132,37 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t))); auto v_as_ptr = builder.CreateIntToPtr(v_raw, smallAllocFunc->getReturnType()); + + // Post alloc + if (1) { + auto intptr_ty = Type::getInt64Ty(target->getContext()); + auto i8_ty = Type::getInt8Ty(F.getContext()); + intptr_t metadata_base_address = reinterpret_cast(MMTK_SIDE_VO_BIT_BASE_ADDRESS); + auto metadata_base_val = ConstantInt::get(intptr_ty, metadata_base_address); + auto metadata_base_ptr = ConstantExpr::getIntToPtr(metadata_base_val, PointerType::get(i8_ty, 0)); + + // intptr_t addr = (intptr_t) v; + auto addr = v_raw; + + // uint8_t* vo_meta_addr = (uint8_t*) (MMTK_SIDE_VO_BIT_BASE_ADDRESS) + (addr >> 6); + auto shr = builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 6)); + auto metadata_ptr = builder.CreateGEP(i8_ty, metadata_base_ptr, shr); + + // intptr_t shift = (addr >> 3) & 0b111; + auto shift = builder.CreateAnd(builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 3)), ConstantInt::get(intptr_ty, 7)); + + // uint8_t byte_val = *vo_meta_addr; + auto byte_val = builder.CreateAlignedLoad(i8_ty, metadata_ptr, Align()); + + // uint8_t new_val = byte_val | (1 << shift); + auto shifted_val = builder.CreateShl(ConstantInt::get(intptr_ty, 1), shift); + auto shifted_val_i8 = builder.CreateTruncOrBitCast(shifted_val, i8_ty); + auto new_val = builder.CreateOr(byte_val, shifted_val_i8); + + // (*vo_meta_addr) = new_val; + builder.CreateStore(new_val, metadata_ptr); + } + builder.CreateBr(next_instr->getParent()); phiNode->addIncoming(new_call, slowpath); From ca994fa14d10175e4c582221c722fe7dfa435a53 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 3 Dec 2024 02:44:54 +0000 Subject: [PATCH 2/4] Introduce jl_gc_persmsymbol --- src/gc-interface.h | 4 +++- src/gc-mmtk.c | 10 ++++++++++ src/gc-stock.c | 9 +++++++++ src/symbol.c | 7 ++----- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/gc-interface.h b/src/gc-interface.h index 043aeb019d395..fb42795c62704 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -204,7 +204,9 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align, // immortal region that is never swept. The second parameter specifies the type of the // object being allocated and will be used to set the object header. struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT; - +// permanently allocates a symbol (jl_sym_t). The object needs to be word aligned, +// and tagged with jl_sym_tag. +struct _jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT; // This function notifies the GC about memory addresses that are set when loading the boot image. // The GC may use that information to, for instance, determine that all objects in that chunk of memory should // be treated as marked and belonged to the old generation in nursery collections. diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index 6782e0370e373..d61f5be6813f1 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -1074,6 +1074,16 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT return jl_valueof(o); } +jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT +{ + jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); + jl_value_t *sym = (jl_sym_t*)jl_valueof(tag); + jl_ptls_t ptls = jl_current_task->ptls; + jl_set_typetagof(sym, jl_symbol_tag, 0); + mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, sym, sz); + return sym; +} + JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) { jl_ptls_t ptls = jl_current_task->ptls; diff --git a/src/gc-stock.c b/src/gc-stock.c index aa7f2a42c712a..a03ba07a0e14c 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3836,6 +3836,15 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT return jl_valueof(o); } +jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT +{ + jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); + jl_value_t *sym = (jl_sym_t*)jl_valueof(tag); + // set to old marked so that we won't look at it in the GC or write barrier. + jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); + return sym; +} + JL_DLLEXPORT int jl_gc_enable_conservative_gc_support(void) { if (jl_is_initialized()) { diff --git a/src/symbol.c b/src/symbol.c index ef2c11e0842e8..cf53743b60602 100644 --- a/src/symbol.c +++ b/src/symbol.c @@ -10,6 +10,7 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "gc-interface.h" #ifdef __cplusplus extern "C" { @@ -34,12 +35,8 @@ static size_t symbol_nbytes(size_t len) JL_NOTSAFEPOINT static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT { - jl_sym_t *sym; size_t nb = symbol_nbytes(len); - jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(nb, 0, sizeof(void*), 0); - sym = (jl_sym_t*)jl_valueof(tag); - // set to old marked so that we won't look at it in the GC or write barrier. - jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); + jl_sym_t *sym = (jl_sym_t*)jl_gc_permsymbol(nb); jl_atomic_store_relaxed(&sym->left, NULL); jl_atomic_store_relaxed(&sym->right, NULL); sym->hash = hash_symbol(str, len); From d8dca099d842136bfbcbc70d32a58a0a6f8f803a Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 3 Dec 2024 02:54:20 +0000 Subject: [PATCH 3/4] Cleanup --- src/gc-mmtk.c | 28 ++++++++++++++-------------- src/gc-stock.c | 2 +- src/llvm-late-gc-lowering-mmtk.cpp | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index d61f5be6813f1..eaa755a656839 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -860,6 +860,19 @@ inline void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* cursor, uin } } +inline void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) { + intptr_t addr = (intptr_t) obj; + uint8_t* meta_addr = (uint8_t*) side_metadata_base + (addr >> 6); + intptr_t shift = (addr >> 3) & 0b111; + while(1) { + uint8_t old_val = *meta_addr; + uint8_t new_val = old_val | (1 << shift); + if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) { + break; + } + } +} + inline void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) { ImmixAllocator* allocator = &mutator->allocators.immix[MMTK_DEFAULT_IMMIX_ALLOCATOR]; return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 0); @@ -876,19 +889,6 @@ inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (uintptr_t)allocator->limit, size, align, offset, 1); } -inline void mmtk_set_side_metadata(void* side_metadata_base, void* obj) { - intptr_t addr = (intptr_t) obj; - uint8_t* meta_addr = (uint8_t*) side_metadata_base + (addr >> 6); - intptr_t shift = (addr >> 3) & 0b111; - while(1) { - uint8_t old_val = *meta_addr; - uint8_t new_val = old_val | (1 << shift); - if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) { - break; - } - } -} - inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) { if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) { mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj); @@ -1077,7 +1077,7 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT { jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); - jl_value_t *sym = (jl_sym_t*)jl_valueof(tag); + jl_value_t *sym = jl_valueof(tag); jl_ptls_t ptls = jl_current_task->ptls; jl_set_typetagof(sym, jl_symbol_tag, 0); mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, sym, sz); diff --git a/src/gc-stock.c b/src/gc-stock.c index a03ba07a0e14c..01ad8ea1a725a 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -3839,7 +3839,7 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT { jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); - jl_value_t *sym = (jl_sym_t*)jl_valueof(tag); + jl_value_t *sym = jl_valueof(tag); // set to old marked so that we won't look at it in the GC or write barrier. jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); return sym; diff --git a/src/llvm-late-gc-lowering-mmtk.cpp b/src/llvm-late-gc-lowering-mmtk.cpp index ca333305c631a..49994c63fbff4 100644 --- a/src/llvm-late-gc-lowering-mmtk.cpp +++ b/src/llvm-late-gc-lowering-mmtk.cpp @@ -134,7 +134,7 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F) auto v_as_ptr = builder.CreateIntToPtr(v_raw, smallAllocFunc->getReturnType()); // Post alloc - if (1) { + if (MMTK_NEEDS_VO_BIT) { auto intptr_ty = Type::getInt64Ty(target->getContext()); auto i8_ty = Type::getInt8Ty(F.getContext()); intptr_t metadata_base_address = reinterpret_cast(MMTK_SIDE_VO_BIT_BASE_ADDRESS); From 1a8cb88b9d5e1137a2857a66aad23d199f00927d Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 3 Dec 2024 06:26:19 +0000 Subject: [PATCH 4/4] Remove unused code. Add comments forjl_gc_permsymbol --- src/datatype.c | 10 ---------- src/gc-interface.h | 5 +++++ src/gc-mmtk.c | 2 +- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/datatype.c b/src/datatype.c index 35c9b033bbe94..3a2ebf2bb303a 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -290,11 +290,6 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, if ((void*)ret == HT_NOTFOUND) { if (!should_malloc) { char *perm_mem = (char *)jl_gc_perm_alloc(flddesc_sz, 0, 4, 0); -// #ifdef MMTK_GC -// // FIXME: Why do we need this? This is not a Julia object at all? -// jl_ptls_t ptls = jl_current_task->ptls; -// mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, jl_valueof(perm_mem), flddesc_sz); -// #endif assert(perm_mem); ret = (jl_datatype_layout_t *)perm_mem; memcpy(perm_mem, flddesc, flddesc_sz); @@ -978,11 +973,6 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name, jl_datatype_layout_t *layout = (jl_datatype_layout_t *) jl_gc_perm_alloc(sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t), 0, 4, 0); -// #ifdef MMTK_GC -// // FIXME: Why do we need this? jl_datatype_layout_t is not a Julia object? -// jl_ptls_t ptls = jl_current_task->ptls; -// mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, jl_valueof(layout), sizeof(jl_datatype_layout_t) + sizeof(jl_fielddescdyn_t)); -// #endif layout->size = large ? GC_MAX_SZCLASS+1 : 0; layout->nfields = 0; layout->alignment = sizeof(void *); diff --git a/src/gc-interface.h b/src/gc-interface.h index fb42795c62704..cc9ee947798fe 100644 --- a/src/gc-interface.h +++ b/src/gc-interface.h @@ -206,6 +206,11 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align, struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT; // permanently allocates a symbol (jl_sym_t). The object needs to be word aligned, // and tagged with jl_sym_tag. +// FIXME: Ideally we should merge this with jl_gc_permobj, as symbol is an object. +// Currently there are a few differences between the two functions, and refactoring is needed. +// 1. sz for this function includes the object header, and sz for jl_gc_permobj excludes the header size. +// 2. align for this function is word align, and align for jl_gc_permobj depends on the allocation size. +// 3. ty for this function is jl_symbol_tag << 4, and ty for jl_gc_permobj is a datatype pointer. struct _jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT; // This function notifies the GC about memory addresses that are set when loading the boot image. // The GC may use that information to, for instance, determine that all objects in that chunk of memory should diff --git a/src/gc-mmtk.c b/src/gc-mmtk.c index eaa755a656839..663995563d6af 100644 --- a/src/gc-mmtk.c +++ b/src/gc-mmtk.c @@ -1079,7 +1079,7 @@ jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0); jl_value_t *sym = jl_valueof(tag); jl_ptls_t ptls = jl_current_task->ptls; - jl_set_typetagof(sym, jl_symbol_tag, 0); + jl_set_typetagof(sym, jl_symbol_tag, 0); // We need to set symbol tag. The GC tag doesnt matter. mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, sym, sz); return sym; }