diff --git a/src/gc.c b/src/gc.c index 9b2895424395de..dbbfad589b9e35 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,9 +1,8 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" -#include "julia_assert.h" #include "julia_gcext.h" -#include "julia_internal.h" +#include "julia_assert.h" #ifdef __GLIBC__ #include // for malloc_trim #endif @@ -29,6 +28,7 @@ static jl_gc_callback_list_t *gc_cblist_notify_external_alloc; static jl_gc_callback_list_t *gc_cblist_notify_external_free; _Atomic(int32_t) nworkers_marking = 0; +extern uv_cond_t safepoint_cond; #define gc_invoke_callbacks(ty, list, args) \ do { \ @@ -1560,118 +1560,6 @@ static void gc_sweep_perm_alloc(void) // mark phase -STATIC_INLINE void gc_markqueue_push(jl_gc_markqueue_t *mq, void *v) JL_NOTSAFEPOINT -{ - if (jl_options.parallel_marking == 1) { - gc_markqueue_push1(mq, v); - } - else { - gc_markqueue_push2(mq, v); - } -} - -STATIC_INLINE void gc_chunkqueue_push(jl_gc_markqueue_t *mq, jl_gc_chunk_t *c) JL_NOTSAFEPOINT -{ - if (jl_options.parallel_marking == 1) { - gc_chunkqueue_push1(mq, c); - } - else { - gc_chunkqueue_push2(mq, c); - } -} - -STATIC_INLINE void *gc_markqueue_pop(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ - if (jl_options.parallel_marking == 1) { - return gc_markqueue_pop1(mq); - } - else { - return gc_markqueue_pop2(mq); - } -} - -STATIC_INLINE jl_gc_chunk_t gc_chunkqueue_pop(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ - if (jl_options.parallel_marking == 1) { - return gc_chunkqueue_pop1(mq); - } - else { - return gc_chunkqueue_pop2(mq); - } -} - -STATIC_INLINE void *gc_markqueue_steal_from(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - if (jl_options.parallel_marking == 1) { - return ws_queue_steal_from(&mq->q); - } -#endif - return gc_markqueue_pop1(mq); -} - -STATIC_INLINE jl_gc_chunk_t gc_chunkqueue_steal_from(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - if (jl_options.parallel_marking == 1) { - jl_gc_chunk_t c; - c.cid = empty_chunk; - idemp_ws_queue_t *cq = &mq->cq; - ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor); - ws_array_t *ary = jl_atomic_load_acquire(&cq->array); - if (anc.tail == 0) { - // Empty queue - return c; - } - c = ((jl_gc_chunk_t *)ary->buffer)[anc.tail - 1]; - ws_anchor_t anc2 = {anc.tail - 1, anc.tag}; - if (!jl_atomic_cmpswap(&cq->anchor, &anc, anc2)) { - // Steal failed - c.cid = empty_chunk; - } - return c; - } -#endif - return gc_chunkqueue_pop1(mq); -} - -STATIC_INLINE void gc_mark_entry_seq(jl_ptls_t ptls) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - if (jl_options.parallel_marking == 1) { - jl_atomic_fetch_add(&nworkers_marking, 1); - jl_atomic_exchange(&ptls->gc_state, JL_GC_STATE_PARALLEL); - } -#endif -} - -STATIC_INLINE void gc_mark_exit_seq(jl_ptls_t ptls) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - if (jl_options.parallel_marking == 1) { - jl_atomic_store_release(&ptls->gc_state, JL_GC_STATE_WAITING); - jl_atomic_fetch_add(&nworkers_marking, -1); - } -#endif -} - -void gc_wake_workers(jl_ptls_t ptls) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - if (jl_options.parallel_marking == 1) { - jl_fence(); - if (jl_n_threads > 1) { - jl_wake_libuv(); - uv_cond_broadcast(&safepoint_cond); - } - for (int i = 0; i < jl_n_threads; i++) { - if (i != ptls->tid) - uv_cond_signal(&jl_all_tls_states[i]->wake_signal); - } - } -#endif -} - JL_DLLEXPORT void jl_gc_queue_root(const jl_value_t *ptr) { jl_ptls_t ptls = jl_current_task->ptls; @@ -1793,8 +1681,111 @@ STATIC_INLINE void gc_mark_push_remset(jl_ptls_t ptls, jl_value_t *obj, } } +// Push gc work item `v` into `mq` +static void gc_markqueue_push(jl_gc_markqueue_t *mq, void *v) JL_NOTSAFEPOINT +{ +#ifndef GC_VERIFY + // Queue overflow + if (!ws_queue_push(&mq->q, v)) { + jl_safe_printf("GC internal error: queue overflow\n"); + abort(); + } +#else + if (__unlikely(mq->current == mq->end)) { + jl_safe_printf("GC internal error: queue overflow\n"); + abort(); + } + *mq->current = v; + mq->current++; +#endif +} + +// Pop gc work item from `mq` +static void *gc_markqueue_pop(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT +{ +#ifndef GC_VERIFY + return ws_queue_pop(&mq->q); +#else + jl_value_t *obj = NULL; + if (mq->current != mq->start) { + obj = *mq->current; + } + return obj; +#endif +} + +// Steal gc work item enqueued in `mq` +static void *gc_markqueue_steal_from(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT +{ +#ifndef GC_VERIFY + return ws_queue_steal_from(&mq->q); +#else + return NULL; +#endif +} + +// Chunk queue push/pop/steal functions are almost verbatim copied +// from `wsqueue.c`. Could be less repetitive with use of macros, +// at expense of debuggability + +// Push chunk `*c` into `mq` +static void gc_chunkqueue_push(jl_gc_markqueue_t *mq, jl_gc_chunk_t *c) JL_NOTSAFEPOINT +{ +#ifndef GC_VERIFY + idemp_ws_queue_t *cq = &mq->cq; + ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor); + ws_array_t *ary = jl_atomic_load_relaxed(&cq->array); + if (anc.tail == ary->capacity) { + jl_safe_printf("GC internal error: chunk-queue overflow"); + abort(); + } + ((jl_gc_chunk_t *)ary->buffer)[anc.tail] = *c; + anc.tail++; + anc.tag++; + jl_atomic_store_release(&cq->anchor, anc); +#endif +} + +// Pop chunk from `mq` +static jl_gc_chunk_t gc_chunkqueue_pop(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT +{ + jl_gc_chunk_t c = {.cid = empty_chunk}; +#ifndef GC_VERIFY + idemp_ws_queue_t *cq = &mq->cq; + ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor); + ws_array_t *ary = jl_atomic_load_acquire(&cq->array); + if (anc.tail == 0) + // Empty queue + return c; + anc.tail--; + c = ((jl_gc_chunk_t *)ary->buffer)[anc.tail]; + jl_atomic_store_release(&cq->anchor, anc); +#endif + return c; +} + +// Steal chunk enqueued in `mq` +static jl_gc_chunk_t gc_chunkqueue_steal_from(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT +{ + jl_gc_chunk_t c = {.cid = empty_chunk}; +#ifndef GC_VERIFY + idemp_ws_queue_t *cq = &mq->cq; + ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor); + ws_array_t *ary = jl_atomic_load_acquire(&cq->array); + if (anc.tail == 0) + // Empty queue + return c; + c = ((jl_gc_chunk_t *)ary->buffer)[anc.tail - 1]; + ws_anchor_t anc2 = {anc.tail - 1, anc.tag}; + if (!jl_atomic_cmpswap(&cq->anchor, &anc, anc2)) + // Steal failed + c.cid = empty_chunk; +#endif + return c; +} + // Enqueue an unmarked obj. last bit of `nptr` is set if `_obj` is young -STATIC_INLINE void gc_try_claim_and_push(jl_gc_markqueue_t *mq, void *_obj, +static void gc_try_claim_and_push(jl_gc_markqueue_t *mq, void *_obj, uintptr_t *nptr) JL_NOTSAFEPOINT { if (!_obj) @@ -1808,7 +1799,7 @@ STATIC_INLINE void gc_try_claim_and_push(jl_gc_markqueue_t *mq, void *_obj, } // Mark object with 8bit field descriptors -STATIC_INLINE jl_value_t *gc_mark_obj8(jl_ptls_t ptls, char *obj8_parent, uint8_t *obj8_begin, +static jl_value_t *gc_mark_obj8(jl_ptls_t ptls, char *obj8_parent, uint8_t *obj8_begin, uint8_t *obj8_end, uintptr_t nptr) JL_NOTSAFEPOINT { (void)jl_assume(obj8_begin < obj8_end); @@ -1836,7 +1827,7 @@ STATIC_INLINE jl_value_t *gc_mark_obj8(jl_ptls_t ptls, char *obj8_parent, uint8_ } // Mark object with 16bit field descriptors -STATIC_INLINE jl_value_t *gc_mark_obj16(jl_ptls_t ptls, char *obj16_parent, uint16_t *obj16_begin, +static jl_value_t *gc_mark_obj16(jl_ptls_t ptls, char *obj16_parent, uint16_t *obj16_begin, uint16_t *obj16_end, uintptr_t nptr) JL_NOTSAFEPOINT { (void)jl_assume(obj16_begin < obj16_end); @@ -1864,7 +1855,7 @@ STATIC_INLINE jl_value_t *gc_mark_obj16(jl_ptls_t ptls, char *obj16_parent, uint } // Mark object with 32bit field descriptors -STATIC_INLINE jl_value_t *gc_mark_obj32(jl_ptls_t ptls, char *obj32_parent, uint32_t *obj32_begin, +static jl_value_t *gc_mark_obj32(jl_ptls_t ptls, char *obj32_parent, uint32_t *obj32_begin, uint32_t *obj32_end, uintptr_t nptr) JL_NOTSAFEPOINT { (void)jl_assume(obj32_begin < obj32_end); @@ -1892,7 +1883,7 @@ STATIC_INLINE jl_value_t *gc_mark_obj32(jl_ptls_t ptls, char *obj32_parent, uint } // Mark object array -STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_value_t **obj_begin, +static void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_value_t **obj_begin, jl_value_t **obj_end, uint32_t step, uintptr_t nptr) JL_NOTSAFEPOINT { @@ -1920,7 +1911,7 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v } // Mark array with 8bit field descriptors -STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_value_t **ary8_begin, +static void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_value_t **ary8_begin, jl_value_t **ary8_end, uint8_t *elem_begin, uint8_t *elem_end, uintptr_t nptr) JL_NOTSAFEPOINT { @@ -1951,7 +1942,7 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va } // Mark array with 16bit field descriptors -STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, +static void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_value_t **ary16_begin, jl_value_t **ary16_end, uint16_t *elem_begin, uint16_t *elem_end, uintptr_t nptr) JL_NOTSAFEPOINT @@ -1983,7 +1974,7 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, } // Mark chunk of large array -STATIC_INLINE void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_chunk_t c) JL_NOTSAFEPOINT +void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_chunk_t c) JL_NOTSAFEPOINT { #ifndef GC_VERIFY switch (c.cid) { @@ -2022,7 +2013,7 @@ STATIC_INLINE void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_ch case finlist_chunk: { jl_value_t **fl_begin = c.begin; jl_value_t **fl_end = c.end; - gc_mark_finlist_(mq, fl_begin, fl_end); + _gc_mark_finlist(mq, fl_begin, fl_end); break; } default: { @@ -2035,7 +2026,7 @@ STATIC_INLINE void gc_mark_chunk(jl_ptls_t ptls, jl_gc_markqueue_t *mq, jl_gc_ch } // Mark gc frame -STATIC_INLINE void gc_mark_stack(jl_ptls_t ptls, jl_gcframe_t *s, uint32_t nroots, +static void gc_mark_stack(jl_ptls_t ptls, jl_gcframe_t *s, uint32_t nroots, uintptr_t offset, uintptr_t lb, uintptr_t ub) JL_NOTSAFEPOINT { jl_gc_markqueue_t *mq = &ptls->mark_queue; @@ -2069,7 +2060,7 @@ STATIC_INLINE void gc_mark_stack(jl_ptls_t ptls, jl_gcframe_t *s, uint32_t nroot } // Mark exception stack -STATIC_INLINE void gc_mark_excstack(jl_ptls_t ptls, jl_excstack_t *excstack, +static void gc_mark_excstack(jl_ptls_t ptls, jl_excstack_t *excstack, size_t itr) JL_NOTSAFEPOINT { jl_gc_markqueue_t *mq = &ptls->mark_queue; @@ -2098,7 +2089,7 @@ STATIC_INLINE void gc_mark_excstack(jl_ptls_t ptls, jl_excstack_t *excstack, } // Mark module binding -STATIC_INLINE void gc_mark_module_binding(jl_ptls_t ptls, jl_module_t *mb_parent, +static void gc_mark_module_binding(jl_ptls_t ptls, jl_module_t *mb_parent, jl_binding_t **mb_begin, jl_binding_t **mb_end, uintptr_t nptr, uint8_t bits) JL_NOTSAFEPOINT { @@ -2143,7 +2134,7 @@ STATIC_INLINE void gc_mark_module_binding(jl_ptls_t ptls, jl_module_t *mb_parent } } -void gc_mark_finlist_(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, jl_value_t **fl_end) +void _gc_mark_finlist(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, jl_value_t **fl_end) { jl_value_t *new_obj; #ifndef GC_VERIFY @@ -2177,7 +2168,7 @@ void gc_mark_finlist(jl_gc_markqueue_t *mq, arraylist_t *list, size_t start) return; jl_value_t **fl_begin = (jl_value_t **)list->items + start; jl_value_t **fl_end = (jl_value_t **)list->items + len; - gc_mark_finlist_(mq, fl_begin, fl_end); + _gc_mark_finlist(mq, fl_begin, fl_end); } JL_DLLEXPORT int jl_gc_mark_queue_obj(jl_ptls_t ptls, jl_value_t *obj) @@ -2198,7 +2189,7 @@ JL_DLLEXPORT void jl_gc_mark_queue_objarray(jl_ptls_t ptls, jl_value_t *parent, // Enqueue and mark all outgoing references from `new_obj` which have not been marked // yet. `meta_updated` is mostly used to make sure we don't update metadata twice for // objects which have been enqueued into the `remset` -STATIC_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_new_obj, +NOINLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void *_new_obj, int meta_updated) JL_NOTSAFEPOINT { jl_value_t *new_obj = (jl_value_t *)_new_obj; @@ -2465,6 +2456,21 @@ STATIC_INLINE void gc_mark_outrefs(jl_ptls_t ptls, jl_gc_markqueue_t *mq, void * } } + +// Wake-up workers to partake in parallel marking +static void gc_wake_workers(jl_ptls_t ptls) +{ + jl_fence(); + if (jl_n_threads > 1) { + jl_wake_libuv(); + uv_cond_broadcast(&safepoint_cond); + } + for (int i = 0; i < jl_n_threads; i++) { + if (i != ptls->tid) + uv_cond_signal(&jl_all_tls_states[i]->wake_signal); + } +} + // Used in `gc-debug` void gc_mark_loop_(jl_ptls_t ptls, jl_gc_markqueue_t *mq) { @@ -2534,30 +2540,32 @@ void gc_drain_all_queues(jl_ptls_t ptls, jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT } } -// Main mark loop. A stack (allocated on the heap) of `jl_value_t *` -// is used to keep track of processed items and a `chunk-stack` is used -// to store suffixes of large arrays that need processing. -// Maintaning these stacks (instead of native one) -// avoids stack overflow when marking deep objects and +// Main mark loop. Single stack (allocated on the heap) of `jl_value_t *` +// is used to keep track of processed items. Maintaning this stack (instead of +// native one) avoids stack overflow when marking deep objects and // makes it easier to implement parallel marking via work-stealing void gc_mark_loop(jl_ptls_t ptls) { - gc_mark_entry_seq(ptls); + jl_atomic_fetch_add(&nworkers_marking, 1); + uint8_t state0 = jl_atomic_exchange(&ptls->gc_state, JL_GC_STATE_PARALLEL); gc_mark_loop_(ptls, &ptls->mark_queue); gc_drain_own_chunkqueue(ptls, &ptls->mark_queue); gc_drain_all_queues(ptls, &ptls->mark_queue); - gc_mark_exit_seq(ptls); + jl_atomic_store_release(&ptls->gc_state, state0); + jl_atomic_fetch_add(&nworkers_marking, -1); } // Mark-loop wrapper. Call workers for parallel marking and mark STATIC_INLINE void gc_mark_loop_master(jl_ptls_t ptls) { - gc_mark_entry_seq(ptls); + jl_atomic_fetch_add(&nworkers_marking, 1); + uint8_t state0 = jl_atomic_exchange(&ptls->gc_state, JL_GC_STATE_PARALLEL); gc_wake_workers(ptls); gc_mark_loop_(ptls, &ptls->mark_queue); gc_drain_own_chunkqueue(ptls, &ptls->mark_queue); gc_drain_all_queues(ptls, &ptls->mark_queue); - gc_mark_exit_seq(ptls); + jl_atomic_store_release(&ptls->gc_state, state0); + jl_atomic_fetch_add(&nworkers_marking, -1); } static void gc_premark(jl_ptls_t ptls2) @@ -3144,22 +3152,16 @@ void jl_init_thread_heap(jl_ptls_t ptls) jl_gc_markqueue_t *mq = &ptls->mark_queue; #ifndef GC_VERIFY ws_queue_t *q = &mq->q; - ws_array_t *wsa = create_ws_array(mq_init_size, sizeof(void *)); jl_atomic_store_relaxed(&q->top, 0); jl_atomic_store_relaxed(&q->bottom, 0); jl_atomic_store_relaxed(&q->array, wsa); - mq->current = mq->start = (jl_value_t **)wsa->buffer; - mq->end = mq->start + mq_init_size; - size_t cq_init_size = (1 << 16); idemp_ws_queue_t *cq = &mq->cq; ws_anchor_t anc = {0, 0}; ws_array_t *wsa2 = create_ws_array(cq_init_size, sizeof(jl_gc_chunk_t)); jl_atomic_store_relaxed(&cq->anchor, anc); jl_atomic_store_relaxed(&cq->array, wsa2); - mq->current_chunk = mq->chunk_start = (jl_gc_chunk_t *)wsa2->buffer; - mq->chunk_end = mq->chunk_start + cq_init_size; #else mq->start = (jl_value_t **)malloc_s(mq_init_size * sizeof(jl_value_t *)); mq->current = mq->start; diff --git a/src/gc.h b/src/gc.h index 94d1bac5120142..6f262bb18264cc 100644 --- a/src/gc.h +++ b/src/gc.h @@ -16,9 +16,7 @@ #include "julia.h" #include "julia_threads.h" #include "julia_internal.h" -#include "options.h" #include "threading.h" -#include "wsqueue.h" #ifndef _OS_WINDOWS_ #include #if defined(_OS_DARWIN_) && !defined(MAP_ANONYMOUS) @@ -83,107 +81,26 @@ typedef struct { uint64_t total_mark_time; } jl_gc_num_t; -// GC marking +typedef enum { + empty_chunk, + objary_chunk, + ary8_chunk, + ary16_chunk, + finlist_chunk, +} gc_chunk_id_t; -// Push gc work item `v` into `mq` -STATIC_INLINE void gc_markqueue_push1(jl_gc_markqueue_t *mq, void *v) JL_NOTSAFEPOINT -{ - if (__unlikely(mq->current == mq->end)) { - jl_safe_printf("GC internal error: queue overflow\n"); - abort(); - } - *mq->current = (jl_value_t *)v; - mq->current++; -} -STATIC_INLINE void gc_markqueue_push2(jl_gc_markqueue_t *mq, void *v) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - if (!ws_queue_push(&mq->q, v)) { - jl_safe_printf("GC internal error: queue overflow\n"); - abort(); - } -#else - gc_markqueue_push1(mq, v); -#endif -} -// Pop gc work item from `mq` -STATIC_INLINE void *gc_markqueue_pop1(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ - jl_value_t *obj = NULL; - if (mq->current != mq->start) { - mq->current--; - obj = *mq->current; - } - return obj; -} -STATIC_INLINE void *gc_markqueue_pop2(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - return ws_queue_pop(&mq->q); -#else - return gc_markqueue_pop1(mq); -#endif -} -// Push chunk `*c` into `mq` -STATIC_INLINE void gc_chunkqueue_push1(jl_gc_markqueue_t *mq, jl_gc_chunk_t *c) JL_NOTSAFEPOINT -{ - if (__unlikely(mq->current_chunk == mq->chunk_end)) { - jl_safe_printf("GC internal error: queue overflow\n"); - abort(); - } - *mq->current_chunk = *c; - mq->current_chunk++; -} -STATIC_INLINE void gc_chunkqueue_push2(jl_gc_markqueue_t *mq, jl_gc_chunk_t *c) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - idemp_ws_queue_t *cq = &mq->cq; - ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor); - ws_array_t *ary = jl_atomic_load_relaxed(&cq->array); - if (anc.tail == ary->capacity) { - jl_safe_printf("GC internal error: chunk-queue overflow"); - abort(); - } - ((jl_gc_chunk_t *)ary->buffer)[anc.tail] = *c; - anc.tail++; - anc.tag++; - jl_atomic_store_release(&cq->anchor, anc); -#else - gc_chunkqueue_push1(mq, c); -#endif -} -// Pop chunk from `mq` -STATIC_INLINE jl_gc_chunk_t gc_chunkqueue_pop1(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ - jl_gc_chunk_t c; - c.cid = empty_chunk; - if (mq->current_chunk != mq->chunk_start) { - mq->current_chunk--; - c = *mq->current_chunk; - } - return c; -} -STATIC_INLINE jl_gc_chunk_t gc_chunkqueue_pop2(jl_gc_markqueue_t *mq) JL_NOTSAFEPOINT -{ -#ifndef GC_VERIFY - jl_gc_chunk_t c; - c.cid = empty_chunk; - idemp_ws_queue_t *cq = &mq->cq; - ws_anchor_t anc = jl_atomic_load_acquire(&cq->anchor); - ws_array_t *ary = jl_atomic_load_acquire(&cq->array); - if (anc.tail == 0) { - // Empty queue - return c; - } - anc.tail--; - c = ((jl_gc_chunk_t *)ary->buffer)[anc.tail]; - jl_atomic_store_release(&cq->anchor, anc); - return c; -#else - return gc_chunkqueue_pop1(mq); -#endif -} -extern uv_cond_t safepoint_cond; +typedef struct { + gc_chunk_id_t cid; + struct _jl_value_t *parent; + struct _jl_value_t **begin; + struct _jl_value_t **end; + void *elem_begin; + void *elem_end; + uint32_t step; + uintptr_t nptr; +} jl_gc_chunk_t; + +#define MAX_REFS_AT_ONCE (1 << 16) // layout for big (>2k) objects @@ -451,7 +368,7 @@ STATIC_INLINE void gc_big_object_link(bigval_t *hdr, bigval_t **list) JL_NOTSAFE } void gc_mark_queue_all_roots(jl_ptls_t ptls, jl_gc_markqueue_t *mq); -void gc_mark_finlist_(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, +void _gc_mark_finlist(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, jl_value_t **fl_end) JL_NOTSAFEPOINT; void gc_mark_finlist(jl_gc_markqueue_t *mq, arraylist_t *list, size_t start) JL_NOTSAFEPOINT; diff --git a/src/julia_threads.h b/src/julia_threads.h index 33b03e8429f9e9..b9a1c8f898fd19 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -170,36 +170,16 @@ typedef struct { arraylist_t free_stacks[JL_N_STACK_POOLS]; } jl_thread_heap_t; -typedef enum { - empty_chunk, - objary_chunk, - ary8_chunk, - ary16_chunk, - finlist_chunk, -} gc_chunk_id_t; - -typedef struct _jl_gc_chunk_t { - gc_chunk_id_t cid; - struct _jl_value_t *parent; - struct _jl_value_t **begin; - struct _jl_value_t **end; - void *elem_begin; - void *elem_end; - uint32_t step; - uintptr_t nptr; -} jl_gc_chunk_t; - -#define MAX_REFS_AT_ONCE (1 << 28) - typedef struct { +// Debugging infrastructure is limited to single threaded GC +#ifdef GC_VERIFY struct _jl_value_t **start; struct _jl_value_t **current; struct _jl_value_t **end; - struct _jl_gc_chunk_t *chunk_start; - struct _jl_gc_chunk_t *current_chunk; - struct _jl_gc_chunk_t *chunk_end; +#else ws_queue_t q; idemp_ws_queue_t cq; +#endif } jl_gc_markqueue_t; typedef struct {