From 98a66ba3c0925ea21bfe051a191210eeae7df0f2 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Fri, 16 Jun 2023 10:10:34 +1200 Subject: [PATCH] Embed mutator in _jl_tls_states_t (#16) This PR embeds the MMTk mutator struct in `_jl_tls_states_t`, and also adds `jl_deinit_thread_heap` to allow a proper destruction of the mutator struct. --- src/gc.c | 5 +++++ src/julia.h | 4 ++-- src/julia_internal.h | 3 ++- src/julia_threads.h | 4 +--- src/llvm-final-gc-lowering.cpp | 12 +++++++++--- src/mmtk-gc.c | 19 ++++++++++++++----- src/symbol.c | 2 +- src/threading.c | 3 +++ 8 files changed, 37 insertions(+), 15 deletions(-) diff --git a/src/gc.c b/src/gc.c index ce80597a937f1..90eae32f0affc 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3501,6 +3501,11 @@ void jl_init_thread_heap(jl_ptls_t ptls) jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval); } +void jl_deinit_thread_heap(jl_ptls_t ptls) +{ + // Do nothing +} + // System-wide initializations void jl_gc_init(void) { diff --git a/src/julia.h b/src/julia.h index 7950eca3e0f1d..253105ef94386 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2391,7 +2391,7 @@ STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSA { jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; - mmtk_object_reference_write_post(ptls->mmtk_mutator_ptr, parent, ptr); + mmtk_object_reference_write_post(&ptls->mmtk_mutator, parent, ptr); } // Inlined fastpath @@ -2405,7 +2405,7 @@ STATIC_INLINE void mmtk_gc_wb_fast(const void *parent, const void *ptr) JL_NOTSA if (((byte_val >> shift) & 1) == 1) { jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; - mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, ptr); + mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, ptr); } } } diff --git a/src/julia_internal.h b/src/julia_internal.h index 5e5b0ebb76e41..76ed8f977dc7a 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -550,7 +550,7 @@ STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT o->header = tag | GC_OLD_MARKED; #ifdef MMTK_GC jl_ptls_t ptls = jl_current_task->ptls; - mmtk_post_alloc(ptls->mmtk_mutator_ptr, jl_valueof(o), allocsz, 1); + mmtk_post_alloc(&ptls->mmtk_mutator, jl_valueof(o), allocsz, 1); #endif return jl_valueof(o); } @@ -918,6 +918,7 @@ void jl_init_serializer(void); void jl_gc_init(void); void jl_init_uv(void); void jl_init_thread_heap(jl_ptls_t ptls) JL_NOTSAFEPOINT; +void jl_deinit_thread_heap(jl_ptls_t ptls) JL_NOTSAFEPOINT; void jl_init_int32_int64_cache(void); JL_DLLEXPORT void jl_init_options(void); diff --git a/src/julia_threads.h b/src/julia_threads.h index 46ad724b71aa0..f79d17d35cb64 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -282,9 +282,7 @@ typedef struct _jl_tls_states_t { ) #ifdef MMTK_GC - MMTkMutatorContext* mmtk_mutator_ptr; - void* cursor; - void* limit; + MMTkMutatorContext mmtk_mutator; #endif // some hidden state (usually just because we don't have the type's size declaration) diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index 6ad46f1eb01d4..48eb584b81893 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -280,17 +280,23 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F) #else // MMTK_GC auto pool_osize_i32 = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize); auto pool_osize = ConstantInt::get(Type::getInt64Ty(F.getContext()), osize); - auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, cursor)); - auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, limit)); + + // Assuming we use the first immix allocator. + // FIXME: We should get the allocator index and type from MMTk. + auto allocator_offset = offsetof(jl_tls_states_t, mmtk_mutator) + offsetof(MMTkMutatorContext, allocators) + offsetof(Allocators, immix); + + auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), allocator_offset + offsetof(ImmixAllocator, cursor)); + auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), allocator_offset + offsetof(ImmixAllocator, limit)); auto cursor_tls_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, cursor_pos); auto cursor_ptr = builder.CreateBitCast(cursor_tls_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "cursor_ptr"); auto cursor = builder.CreateLoad(Type::getInt64Ty(target->getContext()), cursor_ptr, "cursor"); - + // offset = 8 auto delta_offset = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), ConstantInt::get(Type::getInt64Ty(target->getContext()), 8)); auto delta_cursor = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), cursor); auto delta_op = builder.CreateNSWAdd(delta_offset, delta_cursor); + // alignment 16 (15 = 16 - 1) auto delta = builder.CreateAnd(delta_op, ConstantInt::get(Type::getInt64Ty(target->getContext()), 15), "delta"); auto result = builder.CreateNSWAdd(cursor, delta, "result"); diff --git a/src/mmtk-gc.c b/src/mmtk-gc.c index 5e868ef11c1d2..db3affd603cb2 100644 --- a/src/mmtk-gc.c +++ b/src/mmtk-gc.c @@ -266,8 +266,17 @@ void jl_init_thread_heap(jl_ptls_t ptls) memset(&ptls->gc_num, 0, sizeof(ptls->gc_num)); jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval); + // Create mutator MMTk_Mutator mmtk_mutator = mmtk_bind_mutator((void *)ptls, ptls->tid); - ptls->mmtk_mutator_ptr = ((MMTkMutatorContext*)mmtk_mutator); + // Copy the mutator to the thread local storage + memcpy(&ptls->mmtk_mutator, mmtk_mutator, sizeof(MMTkMutatorContext)); + // Call post_bind to maintain a list of active mutators and to reclaim the old mutator (which is no longer needed) + mmtk_post_bind_mutator(&ptls->mmtk_mutator, mmtk_mutator); +} + +void jl_deinit_thread_heap(jl_ptls_t ptls) +{ + mmtk_destroy_mutator(&ptls->mmtk_mutator); } // System-wide initialization @@ -506,7 +515,7 @@ void disable_collection(void) JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT { jl_ptls_t ptls = jl_current_task->ptls; - mmtk_memory_region_copy(ptls->mmtk_mutator_ptr, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n); + mmtk_memory_region_copy(&ptls->mmtk_mutator, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n); } // No inline write barrier -- only used for debugging @@ -524,20 +533,20 @@ JL_DLLEXPORT void jl_gc_wb1_slow(const void *parent) JL_NOTSAFEPOINT { jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; - mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, (const void*) 0); + mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, (const void*) 0); } JL_DLLEXPORT void jl_gc_wb2_slow(const void *parent, const void* ptr) JL_NOTSAFEPOINT { jl_task_t *ct = jl_current_task; jl_ptls_t ptls = ct->ptls; - mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, ptr); + mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, ptr); } void *jl_gc_perm_alloc_nolock(size_t sz, int zero, unsigned align, unsigned offset) { jl_ptls_t ptls = jl_current_task->ptls; - void* addr = mmtk_alloc(ptls->mmtk_mutator_ptr, sz, align, offset, 1); + void* addr = mmtk_alloc(&ptls->mmtk_mutator, sz, align, offset, 1); return addr; } diff --git a/src/symbol.c b/src/symbol.c index dcfa0b6086846..f1cd18cfb84cc 100644 --- a/src/symbol.c +++ b/src/symbol.c @@ -41,7 +41,7 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED); #ifdef MMTK_GC jl_ptls_t ptls = jl_current_task->ptls; - mmtk_post_alloc(ptls->mmtk_mutator_ptr, jl_valueof(tag), nb, 1); + mmtk_post_alloc(&ptls->mmtk_mutator, jl_valueof(tag), nb, 1); #endif jl_atomic_store_relaxed(&sym->left, NULL); jl_atomic_store_relaxed(&sym->right, NULL); diff --git a/src/threading.c b/src/threading.c index 51bdd6e8107da..d58528fa183be 100644 --- a/src/threading.c +++ b/src/threading.c @@ -478,6 +478,9 @@ static void jl_delete_thread(void *value) JL_NOTSAFEPOINT_ENTER #else pthread_mutex_unlock(&in_signal_lock); #endif + + jl_deinit_thread_heap(ptls); + // then park in safe-region (void)jl_gc_safe_enter(ptls); }