Embed mutator in _jl_tls_states_t

src/gc.c

@@ -3501,6 +3501,11 @@ void jl_init_thread_heap(jl_ptls_t ptls)
     jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval);
 }
 
+void jl_deinit_thread_heap(jl_ptls_t ptls)
+{
+    // Do nothing
+}
+
 // System-wide initializations
 void jl_gc_init(void)
 {

src/julia.h

@@ -2391,7 +2391,7 @@ STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSA
 {
     jl_task_t *ct = jl_current_task;
     jl_ptls_t ptls = ct->ptls;
-    mmtk_object_reference_write_post(ptls->mmtk_mutator_ptr, parent, ptr);
+    mmtk_object_reference_write_post(&ptls->mmtk_mutator, parent, ptr);
 }
 
 // Inlined fastpath
@@ -2405,7 +2405,7 @@ STATIC_INLINE void mmtk_gc_wb_fast(const void *parent, const void *ptr) JL_NOTSA
         if (((byte_val >> shift) & 1) == 1) {
             jl_task_t *ct = jl_current_task;
             jl_ptls_t ptls = ct->ptls;
-            mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, ptr);
+            mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, ptr);
         }
     }
 }

src/julia_internal.h

@@ -550,7 +550,7 @@ STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT
     o->header = tag | GC_OLD_MARKED;
 #ifdef MMTK_GC
     jl_ptls_t ptls = jl_current_task->ptls;
-    mmtk_post_alloc(ptls->mmtk_mutator_ptr, jl_valueof(o), allocsz, 1);
+    mmtk_post_alloc(&ptls->mmtk_mutator, jl_valueof(o), allocsz, 1);
 #endif
     return jl_valueof(o);
 }
@@ -918,6 +918,7 @@ void jl_init_serializer(void);
 void jl_gc_init(void);
 void jl_init_uv(void);
 void jl_init_thread_heap(jl_ptls_t ptls) JL_NOTSAFEPOINT;
+void jl_deinit_thread_heap(jl_ptls_t ptls) JL_NOTSAFEPOINT;
 void jl_init_int32_int64_cache(void);
 JL_DLLEXPORT void jl_init_options(void);
 

src/julia_threads.h

@@ -282,9 +282,7 @@ typedef struct _jl_tls_states_t {
     )
 
 #ifdef MMTK_GC
-    MMTkMutatorContext* mmtk_mutator_ptr;
-    void* cursor;
-    void* limit;
+    MMTkMutatorContext mmtk_mutator;
 #endif
 
     // some hidden state (usually just because we don't have the type's size declaration)

src/llvm-final-gc-lowering.cpp

@@ -280,17 +280,23 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
         #else // MMTK_GC
             auto pool_osize_i32 = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize);
             auto pool_osize = ConstantInt::get(Type::getInt64Ty(F.getContext()), osize);
-            auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, cursor));
-            auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()),  offsetof(jl_tls_states_t, limit));
+
+            // Assuming we use the first immix allocator.
+            // FIXME: We should get the allocator index and type from MMTk.
+            auto allocator_offset = offsetof(jl_tls_states_t, mmtk_mutator) + offsetof(MMTkMutatorContext, allocators) + offsetof(Allocators, immix);
+
+            auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), allocator_offset + offsetof(ImmixAllocator, cursor));
+            auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()),  allocator_offset + offsetof(ImmixAllocator, limit));
 
             auto cursor_tls_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, cursor_pos);
             auto cursor_ptr = builder.CreateBitCast(cursor_tls_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "cursor_ptr");
             auto cursor = builder.CreateLoad(Type::getInt64Ty(target->getContext()), cursor_ptr, "cursor");
 
-
+            // offset = 8
             auto delta_offset = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), ConstantInt::get(Type::getInt64Ty(target->getContext()), 8));
             auto delta_cursor = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), cursor);
             auto delta_op = builder.CreateNSWAdd(delta_offset, delta_cursor);
+            // alignment 16 (15 = 16 - 1)
             auto delta = builder.CreateAnd(delta_op, ConstantInt::get(Type::getInt64Ty(target->getContext()), 15), "delta");
             auto result = builder.CreateNSWAdd(cursor, delta, "result");
 

src/mmtk-gc.c

@@ -266,8 +266,17 @@ void jl_init_thread_heap(jl_ptls_t ptls)
     memset(&ptls->gc_num, 0, sizeof(ptls->gc_num));
     jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval);
 
+    // Create mutator
     MMTk_Mutator mmtk_mutator = mmtk_bind_mutator((void *)ptls, ptls->tid);
-    ptls->mmtk_mutator_ptr = ((MMTkMutatorContext*)mmtk_mutator);
+    // Copy the mutator to the thread local storage
+    memcpy(&ptls->mmtk_mutator, mmtk_mutator, sizeof(MMTkMutatorContext));
+    // Call post_bind to maintain a list of active mutators and to reclaim the old mutator (which is no longer needed)
+    mmtk_post_bind_mutator(&ptls->mmtk_mutator, mmtk_mutator);
+}
+
+void jl_deinit_thread_heap(jl_ptls_t ptls)
+{
+    mmtk_destroy_mutator(&ptls->mmtk_mutator);
 }
 
 // System-wide initialization
@@ -506,7 +515,7 @@ void disable_collection(void)
 JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT
 {
     jl_ptls_t ptls = jl_current_task->ptls;
-    mmtk_memory_region_copy(ptls->mmtk_mutator_ptr, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n);
+    mmtk_memory_region_copy(&ptls->mmtk_mutator, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n);
 }
 
 // No inline write barrier -- only used for debugging
@@ -524,20 +533,20 @@ JL_DLLEXPORT void jl_gc_wb1_slow(const void *parent) JL_NOTSAFEPOINT
 {
     jl_task_t *ct = jl_current_task;
     jl_ptls_t ptls = ct->ptls;
-    mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, (const void*) 0);
+    mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, (const void*) 0);
 }
 
 JL_DLLEXPORT void jl_gc_wb2_slow(const void *parent, const void* ptr) JL_NOTSAFEPOINT
 {
     jl_task_t *ct = jl_current_task;
     jl_ptls_t ptls = ct->ptls;
-    mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, ptr);
+    mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, ptr);
 }
 
 void *jl_gc_perm_alloc_nolock(size_t sz, int zero, unsigned align, unsigned offset)
 {
     jl_ptls_t ptls = jl_current_task->ptls;
-    void* addr = mmtk_alloc(ptls->mmtk_mutator_ptr, sz, align, offset, 1);
+    void* addr = mmtk_alloc(&ptls->mmtk_mutator, sz, align, offset, 1);
     return addr;
 }
 

src/symbol.c

@@ -41,7 +41,7 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT
     jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED);
 #ifdef MMTK_GC
     jl_ptls_t ptls = jl_current_task->ptls;
-    mmtk_post_alloc(ptls->mmtk_mutator_ptr, jl_valueof(tag), nb, 1);
+    mmtk_post_alloc(&ptls->mmtk_mutator, jl_valueof(tag), nb, 1);
 #endif
     jl_atomic_store_relaxed(&sym->left, NULL);
     jl_atomic_store_relaxed(&sym->right, NULL);

src/threading.c

@@ -478,6 +478,9 @@ static void jl_delete_thread(void *value) JL_NOTSAFEPOINT_ENTER
 #else
     pthread_mutex_unlock(&in_signal_lock);
 #endif
+
+    jl_deinit_thread_heap(ptls);
+
     // then park in safe-region
     (void)jl_gc_safe_enter(ptls);
 }