Support VO bit

src/gc-interface.h

@@ -204,7 +204,14 @@ JL_DLLEXPORT void *jl_gc_perm_alloc(size_t sz, int zero, unsigned align,
 // immortal region that is never swept. The second parameter specifies the type of the
 // object being allocated and will be used to set the object header.
 struct _jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT;
-
+// permanently allocates a symbol (jl_sym_t). The object needs to be word aligned,
+// and tagged with jl_sym_tag.
+// FIXME: Ideally we should merge this with jl_gc_permobj, as symbol is an object.
+// Currently there are a few differences between the two functions, and refactoring is needed.
+// 1. sz for this function includes the object header, and sz for jl_gc_permobj excludes the header size.
+// 2. align for this function is word align, and align for jl_gc_permobj depends on the allocation size.
+// 3. ty for this function is jl_symbol_tag << 4, and ty for jl_gc_permobj is a datatype pointer.
+struct _jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT;
 // This function notifies the GC about memory addresses that are set when loading the boot image.
 // The GC may use that information to, for instance, determine that all objects in that chunk of memory should
 // be treated as marked and belonged to the old generation in nursery collections.

src/gc-mmtk.c

@@ -860,18 +860,28 @@ inline void* bump_alloc_fast(MMTkMutatorContext* mutator, uintptr_t* cursor, uin
     }
 }
 
+inline void mmtk_set_side_metadata(const void* side_metadata_base, void* obj) {
+        intptr_t addr = (intptr_t) obj;
+        uint8_t* meta_addr = (uint8_t*) side_metadata_base + (addr >> 6);
+        intptr_t shift = (addr >> 3) & 0b111;
+        while(1) {
+            uint8_t old_val = *meta_addr;
+            uint8_t new_val = old_val | (1 << shift);
+            if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) {
+                break;
+            }
+        }
+}
+
 inline void* mmtk_immix_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) {
     ImmixAllocator* allocator = &mutator->allocators.immix[MMTK_DEFAULT_IMMIX_ALLOCATOR];
     return bump_alloc_fast(mutator, (uintptr_t*)&allocator->cursor, (intptr_t)allocator->limit, size, align, offset, 0);
 }
 
-inline void mmtk_immix_post_alloc_slow(MMTkMutatorContext* mutator, void* obj, size_t size) {
-    mmtk_post_alloc(mutator, obj, size, 0);
-}
-
 inline void mmtk_immix_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
-    // FIXME: for now, we do nothing
-    // but when supporting moving, this is where we set the valid object (VO) bit
+    if (MMTK_NEEDS_VO_BIT) {
+        mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj);
+    }
 }
 
 inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size, size_t align, size_t offset) {
@@ -881,16 +891,11 @@ inline void* mmtk_immortal_alloc_fast(MMTkMutatorContext* mutator, size_t size,
 
 inline void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
     if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) {
-        intptr_t addr = (intptr_t) obj;
-        uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6);
-        intptr_t shift = (addr >> 3) & 0b111;
-        while(1) {
-            uint8_t old_val = *meta_addr;
-            uint8_t new_val = old_val | (1 << shift);
-            if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) {
-                break;
-            }
-        }
+        mmtk_set_side_metadata(MMTK_SIDE_LOG_BIT_BASE_ADDRESS, obj);
+    }
+
+    if (MMTK_NEEDS_VO_BIT) {
+        mmtk_set_side_metadata(MMTK_SIDE_VO_BIT_BASE_ADDRESS, obj);
     }
 }
 
@@ -1069,6 +1074,16 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT
     return jl_valueof(o);
 }
 
+jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT
+{
+    jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0);
+    jl_value_t *sym = jl_valueof(tag);
+    jl_ptls_t ptls = jl_current_task->ptls;
+    jl_set_typetagof(sym, jl_symbol_tag, 0);    // We need to set symbol tag. The GC tag doesnt matter.
+    mmtk_immortal_post_alloc_fast(&ptls->gc_tls.mmtk_mutator, sym, sz);
+    return sym;
+}
+
 JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz)
 {
     jl_ptls_t ptls = jl_current_task->ptls;

src/gc-stock.c

@@ -3836,6 +3836,15 @@ jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT
     return jl_valueof(o);
 }
 
+jl_value_t *jl_gc_permsymbol(size_t sz) JL_NOTSAFEPOINT
+{
+    jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(sz, 0, sizeof(void*), 0);
+    jl_value_t *sym = jl_valueof(tag);
+    // set to old marked so that we won't look at it in the GC or write barrier.
+    jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED);
+    return sym;
+}
+
 JL_DLLEXPORT int jl_gc_enable_conservative_gc_support(void)
 {
     if (jl_is_initialized()) {

src/julia.h

@@ -2748,7 +2748,13 @@ extern void mmtk_object_reference_write_slow(void* mutator, const void* parent,
 #define MMTK_DEFAULT_IMMIX_ALLOCATOR (0)
 #define MMTK_IMMORTAL_BUMP_ALLOCATOR (0)
 
+// VO bit is required to support conservative stack scanning and moving.
+#define MMTK_NEEDS_VO_BIT (1)
+
+void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size);
+
 extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;
+extern const void* MMTK_SIDE_VO_BIT_BASE_ADDRESS;
 
 // Directly call into MMTk for write barrier (debugging only)
 STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSAFEPOINT

src/llvm-late-gc-lowering-mmtk.cpp

@@ -132,6 +132,37 @@ Value* LateLowerGCFrame::lowerGCAllocBytesLate(CallInst *target, Function &F)
 
                 auto v_raw = builder.CreateNSWAdd(result, ConstantInt::get(Type::getInt64Ty(target->getContext()), sizeof(jl_taggedvalue_t)));
                 auto v_as_ptr = builder.CreateIntToPtr(v_raw, smallAllocFunc->getReturnType());
+
+                // Post alloc
+                if (MMTK_NEEDS_VO_BIT) {
+                    auto intptr_ty = Type::getInt64Ty(target->getContext());
+                    auto i8_ty = Type::getInt8Ty(F.getContext());
+                    intptr_t metadata_base_address = reinterpret_cast<intptr_t>(MMTK_SIDE_VO_BIT_BASE_ADDRESS);
+                    auto metadata_base_val = ConstantInt::get(intptr_ty, metadata_base_address);
+                    auto metadata_base_ptr = ConstantExpr::getIntToPtr(metadata_base_val, PointerType::get(i8_ty, 0));
+
+                    // intptr_t addr = (intptr_t) v;
+                    auto addr = v_raw;
+
+                    // uint8_t* vo_meta_addr = (uint8_t*) (MMTK_SIDE_VO_BIT_BASE_ADDRESS) + (addr >> 6);
+                    auto shr = builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 6));
+                    auto metadata_ptr = builder.CreateGEP(i8_ty, metadata_base_ptr, shr);
+
+                    // intptr_t shift = (addr >> 3) & 0b111;
+                    auto shift = builder.CreateAnd(builder.CreateLShr(addr, ConstantInt::get(intptr_ty, 3)), ConstantInt::get(intptr_ty, 7));
+
+                    // uint8_t byte_val = *vo_meta_addr;
+                    auto byte_val = builder.CreateAlignedLoad(i8_ty, metadata_ptr, Align());
+
+                    // uint8_t new_val = byte_val | (1 << shift);
+                    auto shifted_val = builder.CreateShl(ConstantInt::get(intptr_ty, 1), shift);
+                    auto shifted_val_i8 = builder.CreateTruncOrBitCast(shifted_val, i8_ty);
+                    auto new_val = builder.CreateOr(byte_val, shifted_val_i8);
+
+                    // (*vo_meta_addr) = new_val;
+                    builder.CreateStore(new_val, metadata_ptr);
+                }
+
                 builder.CreateBr(next_instr->getParent());
 
                 phiNode->addIncoming(new_call, slowpath);

src/symbol.c

@@ -10,6 +10,7 @@
 #include "julia.h"
 #include "julia_internal.h"
 #include "julia_assert.h"
+#include "gc-interface.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -34,12 +35,8 @@ static size_t symbol_nbytes(size_t len) JL_NOTSAFEPOINT
 
 static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT
 {
-    jl_sym_t *sym;
     size_t nb = symbol_nbytes(len);
-    jl_taggedvalue_t *tag = (jl_taggedvalue_t*)jl_gc_perm_alloc(nb, 0, sizeof(void*), 0);
-    sym = (jl_sym_t*)jl_valueof(tag);
-    // set to old marked so that we won't look at it in the GC or write barrier.
-    jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED);
+    jl_sym_t *sym = (jl_sym_t*)jl_gc_permsymbol(nb);
     jl_atomic_store_relaxed(&sym->left, NULL);
     jl_atomic_store_relaxed(&sym->right, NULL);
     sym->hash = hash_symbol(str, len);