diff --git a/src/gc-pages.c b/src/gc-pages.c index d579eb0cd4fbb..f07e223c98ddb 100644 --- a/src/gc-pages.c +++ b/src/gc-pages.c @@ -259,30 +259,15 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT return info.meta; } -// return a page to the freemap allocator -void jl_gc_free_page(void *p) JL_NOTSAFEPOINT +void jl_gc_pre_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT { - // update the allocmap and freemap to indicate this contains a free entry + void *p = pp->page_meta->data; + // update the allocmap struct jl_gc_metadata_ext info = page_metadata_ext(p); uint32_t msk; msk = (uint32_t)(1u << info.pagetable0_i); - assert(!(info.pagetable0->freemap[info.pagetable0_i32] & msk)); assert(info.pagetable0->allocmap[info.pagetable0_i32] & msk); info.pagetable0->allocmap[info.pagetable0_i32] &= ~msk; - info.pagetable0->freemap[info.pagetable0_i32] |= msk; - - msk = (uint32_t)(1u << info.pagetable1_i); - assert(info.pagetable1->allocmap0[info.pagetable1_i32] & msk); - if ((info.pagetable1->freemap0[info.pagetable1_i32] & msk) == 0) - info.pagetable1->freemap0[info.pagetable1_i32] |= msk; - - msk = (uint32_t)(1u << info.pagetable_i); - assert(memory_map.allocmap1[info.pagetable_i32] & msk); - if ((memory_map.freemap1[info.pagetable_i32] & msk) == 0) - memory_map.freemap1[info.pagetable_i32] |= msk; - - free(info.meta->ages); - info.meta->ages = NULL; // tell the OS we don't need these pages right now size_t decommit_size = GC_PAGE_SZ; @@ -296,10 +281,18 @@ void jl_gc_free_page(void *p) JL_NOTSAFEPOINT struct jl_gc_metadata_ext info = page_metadata_ext(otherp); msk = (uint32_t)(1u << info.pagetable0_i); if (info.pagetable0->allocmap[info.pagetable0_i32] & msk) - goto no_decommit; + return; otherp = (void*)((char*)otherp + GC_PAGE_SZ); } } + pp->should_decommit = 1; + pp->decommit_size = decommit_size; +} + +void jl_gc_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT +{ + void *p = pp->page_meta->data; + size_t decommit_size = pp->decommit_size; #ifdef _OS_WINDOWS_ VirtualFree(p, decommit_size, MEM_DECOMMIT); #elif defined(MADV_FREE) @@ -320,8 +313,32 @@ void jl_gc_free_page(void *p) JL_NOTSAFEPOINT * the page when it sweeps pools? */ msan_unpoison(p, decommit_size); +} + +// return a page to the freemap allocator +void jl_gc_post_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT +{ + void *p = pp->page_meta->data; + // update the freemap to indicate this contains a free entry + struct jl_gc_metadata_ext info = page_metadata_ext(p); + uint32_t msk; + msk = (uint32_t)(1u << info.pagetable0_i); + assert(!(info.pagetable0->freemap[info.pagetable0_i32] & msk)); + info.pagetable0->freemap[info.pagetable0_i32] |= msk; + + msk = (uint32_t)(1u << info.pagetable1_i); + assert(info.pagetable1->allocmap0[info.pagetable1_i32] & msk); + if ((info.pagetable1->freemap0[info.pagetable1_i32] & msk) == 0) + info.pagetable1->freemap0[info.pagetable1_i32] |= msk; + + msk = (uint32_t)(1u << info.pagetable_i); + assert(memory_map.allocmap1[info.pagetable_i32] & msk); + if ((memory_map.freemap1[info.pagetable_i32] & msk) == 0) + memory_map.freemap1[info.pagetable_i32] |= msk; + + free(info.meta->ages); + info.meta->ages = NULL; -no_decommit: // new pages are now available starting at max of lb and pagetable_i32 if (memory_map.lb > info.pagetable_i32) memory_map.lb = info.pagetable_i32; diff --git a/src/gc.c b/src/gc.c index 4925ed91ea179..4d9829de4763a 100644 --- a/src/gc.c +++ b/src/gc.c @@ -13,6 +13,10 @@ extern "C" { // Number of threads currently running the GC mark-loop _Atomic(int) gc_n_threads_marking; +// Bitmap used to synchronized parallel/concurrent sweeping +_Atomic(uint64_t) gc_sweep_mask; +// Flag to indicate whether concurrent sweeping of object pools is running +_Atomic(uint64_t) gc_concurrent_sweep_running; // `tid` of mutator thread that triggered GC _Atomic(int) gc_master_tid; // `tid` of first GC thread @@ -20,6 +24,8 @@ int gc_first_tid; // Mutex/cond used to synchronize sleep/wakeup of GC threads uv_mutex_t gc_threads_lock; uv_cond_t gc_threads_cond; +// Queue of packets corresponding to pages that need sweeping +gc_sweep_queue_t gc_sweep_queue; // Linked list of callback functions @@ -1391,9 +1397,26 @@ int jl_gc_classify_pools(size_t sz, int *osize) int64_t lazy_freed_pages = 0; -// Returns pointer to terminal pointer of list rooted at *pfl. -static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_taggedvalue_t **pfl, int sweep_full, int osize) JL_NOTSAFEPOINT +static void gc_sweep_queue_push(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT { + if (__unlikely(gc_sweep_queue.current == gc_sweep_queue.end)) { + // TODO: resize queue + jl_safe_printf("Sweep packet queue is full!\n"); + abort(); + } + memcpy(gc_sweep_queue.current, pp, sizeof(gc_sweep_packet_t)); + gc_sweep_queue.current++; +} + +static void sweep_packet(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT +{ + // Unpack sweep packet + jl_gc_pool_t *p = pp->pool; + jl_gc_pagemeta_t *pg = pp->page_meta; + jl_taggedvalue_t **pfl = &pp->freelist_front; + int osize = pp->osize; + uint8_t sweep_full = pp->sweep_full; + char *data = pg->data; uint32_t *ages = pg->ages; jl_taggedvalue_t *v = (jl_taggedvalue_t*)(data + GC_PAGE_OFFSET); @@ -1417,7 +1440,7 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t lazy_freed_pages++; } else { - jl_gc_free_page(data); + pp->should_free = 1; } nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize; goto done; @@ -1509,23 +1532,23 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t done: gc_time_count_page(freedall, pg_skpd); - gc_num.freed += (nfree - old_nfree) * osize; - return pfl; + jl_atomic_fetch_add((_Atomic(uint64_t) *)&gc_num.freed, (nfree - old_nfree) * osize); + pp->freelist_back = pfl; } -// the actual sweeping over all allocated pages in a memory pool -STATIC_INLINE void sweep_pool_page(jl_taggedvalue_t ***pfl, jl_gc_pagemeta_t *pg, int sweep_full) JL_NOTSAFEPOINT +STATIC_INLINE void sweep_sched_packet(jl_gc_pagemeta_t *pg, int sweep_full) JL_NOTSAFEPOINT { int p_n = pg->pool_n; int t_n = pg->thread_n; jl_ptls_t ptls2 = gc_all_tls_states[t_n]; jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; - pfl[t_n * JL_GC_N_POOLS + p_n] = sweep_page(p, pg, pfl[t_n * JL_GC_N_POOLS + p_n], sweep_full, osize); + gc_sweep_packet_t pckt = {p, pg, NULL, NULL, osize, sweep_full, 0}; + gc_sweep_queue_push(&pckt); } // sweep over a pagetable0 for all allocated pages -STATIC_INLINE int sweep_pool_pagetable0(jl_taggedvalue_t ***pfl, pagetable0_t *pagetable0, int sweep_full) JL_NOTSAFEPOINT +STATIC_INLINE int sweep_pool_pagetable0(pagetable0_t *pagetable0, int sweep_full) JL_NOTSAFEPOINT { unsigned ub = 0; unsigned alloc = 0; @@ -1541,7 +1564,7 @@ STATIC_INLINE int sweep_pool_pagetable0(jl_taggedvalue_t ***pfl, pagetable0_t *p j += next; line >>= next; jl_gc_pagemeta_t *pg = pagetable0->meta[pg_i * 32 + j]; - sweep_pool_page(pfl, pg, sweep_full); + sweep_sched_packet(pg, sweep_full); } } pagetable0->ub = ub; @@ -1549,7 +1572,7 @@ STATIC_INLINE int sweep_pool_pagetable0(jl_taggedvalue_t ***pfl, pagetable0_t *p } // sweep over pagetable1 for all pagetable0 that may contain allocated pages -STATIC_INLINE int sweep_pool_pagetable1(jl_taggedvalue_t ***pfl, pagetable1_t *pagetable1, int sweep_full) JL_NOTSAFEPOINT +STATIC_INLINE int sweep_pool_pagetable1(pagetable1_t *pagetable1, int sweep_full) JL_NOTSAFEPOINT { unsigned ub = 0; unsigned alloc = 0; @@ -1561,7 +1584,7 @@ STATIC_INLINE int sweep_pool_pagetable1(jl_taggedvalue_t ***pfl, pagetable1_t *p j += next; line >>= next; pagetable0_t *pagetable0 = pagetable1->meta0[pg_i * 32 + j]; - if (pagetable0 && !sweep_pool_pagetable0(pfl, pagetable0, sweep_full)) + if (pagetable0 && !sweep_pool_pagetable0(pagetable0, sweep_full)) pagetable1->allocmap0[pg_i] &= ~(1 << j); // no allocations found, remember that for next time } if (pagetable1->allocmap0[pg_i]) { @@ -1574,12 +1597,12 @@ STATIC_INLINE int sweep_pool_pagetable1(jl_taggedvalue_t ***pfl, pagetable1_t *p } // sweep over all memory for all pagetable1 that may contain allocated pages -static void sweep_pool_pagetable(jl_taggedvalue_t ***pfl, int sweep_full) JL_NOTSAFEPOINT +static void sweep_pool_pagetable(int sweep_full) JL_NOTSAFEPOINT { if (REGION2_PG_COUNT == 1) { // compile-time optimization pagetable1_t *pagetable1 = memory_map.meta1[0]; if (pagetable1 != NULL) - sweep_pool_pagetable1(pfl, pagetable1, sweep_full); + sweep_pool_pagetable1(pagetable1, sweep_full); return; } unsigned ub = 0; @@ -1591,7 +1614,7 @@ static void sweep_pool_pagetable(jl_taggedvalue_t ***pfl, int sweep_full) JL_NOT j += next; line >>= next; pagetable1_t *pagetable1 = memory_map.meta1[pg_i * 32 + j]; - if (pagetable1 && !sweep_pool_pagetable1(pfl, pagetable1, sweep_full)) + if (pagetable1 && !sweep_pool_pagetable1(pagetable1, sweep_full)) memory_map.allocmap1[pg_i] &= ~(1 << j); // no allocations found, remember that for next time } if (memory_map.allocmap1[pg_i]) { @@ -1624,6 +1647,24 @@ static void gc_pool_sync_nfree(jl_gc_pagemeta_t *pg, jl_taggedvalue_t *last) JL_ pg->nfree = nfree; } +void gc_sweep_pool_parallel(jl_ptls_t ptls, int master) JL_NOTSAFEPOINT +{ + size_t lb; + size_t ub; + size_t n_pckts = gc_sweep_queue.current - gc_sweep_queue.begin; + if (master) { + lb = 0; + ub = n_pckts / (jl_n_gcthreads + 1); + } + else { + lb = ((ptls->tid - gc_first_tid + 1) * n_pckts) / (jl_n_gcthreads + 1); + ub = ((ptls->tid - gc_first_tid + 2) * n_pckts) / (jl_n_gcthreads + 1); + } + for (gc_sweep_packet_t *pp = gc_sweep_queue.begin + lb; pp < gc_sweep_queue.begin + ub; pp++) { + sweep_packet(pp); + } +} + // setup the data-structures for a sweep over all memory pools static void gc_sweep_pool(int sweep_full) { @@ -1670,10 +1711,48 @@ static void gc_sweep_pool(int sweep_full) p->newpages = NULL; } } + // wait for concurrent sweep to finish + if (jl_n_gcthreads != 0) { + while (jl_atomic_load(&gc_concurrent_sweep_running)) { + jl_cpu_pause(); + } + } + gc_sweep_queue.current = gc_sweep_queue.begin; + + sweep_pool_pagetable(sweep_full); + + // wake up GC threads + uv_mutex_lock(&gc_threads_lock); + jl_atomic_store(&gc_sweep_mask, UINT64_MAX); + uv_cond_broadcast(&gc_threads_cond); + uv_mutex_unlock(&gc_threads_lock); // the actual sweeping - sweep_pool_pagetable(pfl, sweep_full); + gc_sweep_pool_parallel(NULL, 1); + for (int i = 1; i <= jl_n_gcthreads; i++) { + while (jl_atomic_load(&gc_sweep_mask) & (1ull << i)) { + jl_cpu_pause(); + } + } + // merge free lists and free empty pages that were not lazily sweeped + for (gc_sweep_packet_t *pp = gc_sweep_queue.begin; pp < gc_sweep_queue.current; pp++) { + int t_i = pp->page_meta->thread_n; + int i = pp->page_meta->pool_n; + if (pp->freelist_front != NULL) { + *pfl[t_i * JL_GC_N_POOLS + i] = pp->freelist_front; + pfl[t_i * JL_GC_N_POOLS + i] = pp->freelist_back; + } + else if (pp->should_free) { + if (jl_n_gcthreads == 0) { + jl_gc_pre_free_page(pp); + if (pp->should_decommit) { + jl_gc_free_page(pp); + } + jl_gc_post_free_page(pp); + } + } + } // null out terminal pointers of free lists for (int t_i = 0; t_i < n_threads; t_i++) { jl_ptls_t ptls2 = gc_all_tls_states[t_i]; @@ -1683,6 +1762,9 @@ static void gc_sweep_pool(int sweep_full) } } } + if (jl_n_gcthreads != 0) { + jl_atomic_store(&gc_concurrent_sweep_running, 1); + } gc_time_pool_end(sweep_full); } @@ -2867,7 +2949,7 @@ void gc_mark_loop_barrier(void) void gc_mark_clean_reclaim_sets(void) { - // Clean up `reclaim-sets` and reset `top/bottom` of queues + // Clean up `reclaim-sets` for (int i = 0; i < gc_n_threads; i++) { jl_ptls_t ptls2 = gc_all_tls_states[i]; arraylist_t *reclaim_set2 = &ptls2->mark_queue.reclaim_set; @@ -3599,6 +3681,11 @@ void jl_gc_init(void) arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); + // Init sweep packet queue + gc_sweep_queue.current = gc_sweep_queue.begin = + (gc_sweep_packet_t *)malloc_s(GC_SWEEP_QUEUE_INIT_SIZE * sizeof(gc_sweep_queue_t)); + gc_sweep_queue.end = gc_sweep_queue.begin + GC_SWEEP_QUEUE_INIT_SIZE; + gc_num.interval = default_collect_interval; last_long_collect_interval = default_collect_interval; gc_num.allocd = 0; diff --git a/src/gc.h b/src/gc.h index eb20dd0ac36f6..1bb6725d9be07 100644 --- a/src/gc.h +++ b/src/gc.h @@ -105,11 +105,30 @@ typedef struct _jl_gc_chunk_t { uintptr_t nptr; // (`nptr` & 0x1) if array has young element and (`nptr` & 0x2) if array owner is old } jl_gc_chunk_t; -#define GC_CHUNK_BATCH_SIZE (1 << 16) // maximum number of references that can be processed - // without creating a chunk +struct _jl_gc_pagemeta_t; +typedef struct { + jl_gc_pool_t *pool; + struct _jl_gc_pagemeta_t *page_meta; + jl_taggedvalue_t *freelist_front; + jl_taggedvalue_t **freelist_back; + int osize; + uint8_t sweep_full: 1; + uint8_t should_free: 1; + uint8_t should_decommit: 1; + size_t decommit_size; +} gc_sweep_packet_t; + +typedef struct { + gc_sweep_packet_t *begin; + gc_sweep_packet_t *current; + gc_sweep_packet_t *end; +} gc_sweep_queue_t; + +#define GC_CHUNK_BATCH_SIZE (1 << 16) // maximum number of references that can be processed without creating a chunk #define GC_PTR_QUEUE_INIT_SIZE (1 << 18) // initial size of queue of `jl_value_t *` #define GC_CHUNK_QUEUE_INIT_SIZE (1 << 14) // initial size of chunk-queue +#define GC_SWEEP_QUEUE_INIT_SIZE (1 << 22) // initial size of sweep queue // layout for big (>2k) objects @@ -145,7 +164,7 @@ typedef struct _mallocarray_t { } mallocarray_t; // pool page metadata -typedef struct { +typedef struct _jl_gc_pagemeta_t { // index of pool that owns this page uint8_t pool_n; // Whether any cell in the page is marked @@ -381,14 +400,16 @@ STATIC_INLINE void gc_big_object_link(bigval_t *hdr, bigval_t **list) JL_NOTSAFE extern uv_mutex_t gc_threads_lock; extern uv_cond_t gc_threads_cond; extern _Atomic(int) gc_n_threads_marking; +extern _Atomic(uint64_t) gc_sweep_mask; +extern _Atomic(uint64_t) gc_concurrent_sweep_running; +extern gc_sweep_queue_t gc_sweep_queue; void gc_mark_queue_all_roots(jl_ptls_t ptls, jl_gc_markqueue_t *mq); -void gc_mark_finlist_(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, - jl_value_t **fl_end) JL_NOTSAFEPOINT; -void gc_mark_finlist(jl_gc_markqueue_t *mq, arraylist_t *list, - size_t start) JL_NOTSAFEPOINT; +void gc_mark_finlist_(jl_gc_markqueue_t *mq, jl_value_t **fl_begin, jl_value_t **fl_end) JL_NOTSAFEPOINT; +void gc_mark_finlist(jl_gc_markqueue_t *mq, arraylist_t *list, size_t start) JL_NOTSAFEPOINT; void gc_mark_loop_serial_(jl_ptls_t ptls, jl_gc_markqueue_t *mq); void gc_mark_loop_serial(jl_ptls_t ptls); void gc_mark_loop_parallel(jl_ptls_t ptls, int master); +void gc_sweep_pool_parallel(jl_ptls_t ptls, int master) JL_NOTSAFEPOINT; void sweep_stack_pools(void); void jl_gc_debug_init(void); @@ -396,7 +417,9 @@ void jl_gc_debug_init(void); void jl_gc_init_page(void); NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT; -void jl_gc_free_page(void *p) JL_NOTSAFEPOINT; +void jl_gc_pre_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT; +void jl_gc_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT; +void jl_gc_post_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT; // GC debug diff --git a/src/partr.c b/src/partr.c index 403f911b1284f..92d84e20a653b 100644 --- a/src/partr.c +++ b/src/partr.c @@ -108,6 +108,8 @@ void jl_init_threadinginfra(void) void JL_NORETURN jl_finish_task(jl_task_t *t); +extern int gc_first_tid; + // gc thread function void jl_gc_threadfun(void *arg) { @@ -123,13 +125,47 @@ void jl_gc_threadfun(void *arg) // free the thread argument here free(targ); + uint64_t mask = 1ull << (ptls->tid - gc_first_tid + 1); + while (1) { uv_mutex_lock(&gc_threads_lock); - while (jl_atomic_load(&gc_n_threads_marking) == 0) { + while (jl_atomic_load(&gc_n_threads_marking) == 0 && (jl_atomic_load(&gc_sweep_mask) & mask) == 0) { uv_cond_wait(&gc_threads_cond, &gc_threads_lock); } uv_mutex_unlock(&gc_threads_lock); - gc_mark_loop_parallel(ptls, 0); + if ((jl_atomic_load(&gc_sweep_mask) & mask) == 0) { + gc_mark_loop_parallel(ptls, 0); + } + else { + gc_sweep_pool_parallel(ptls, 0); + jl_atomic_fetch_and(&gc_sweep_mask, ~mask); + if (ptls->tid == gc_first_tid) { + while (!jl_atomic_load(&gc_concurrent_sweep_running)) { + jl_cpu_pause(); + } + uv_mutex_lock(&gc_perm_lock); + for (gc_sweep_packet_t *pp = gc_sweep_queue.begin; pp < gc_sweep_queue.current; pp++) { + if (pp->should_free) { + jl_gc_pre_free_page(pp); + } + } + uv_mutex_unlock(&gc_perm_lock); + // `madvise` is expensive, so move it out of the critical section + for (gc_sweep_packet_t *pp = gc_sweep_queue.begin; pp < gc_sweep_queue.current; pp++) { + if (pp->should_decommit) { + jl_gc_free_page(pp); + } + } + uv_mutex_lock(&gc_perm_lock); + for (gc_sweep_packet_t *pp = gc_sweep_queue.begin; pp < gc_sweep_queue.current; pp++) { + if (pp->should_free) { + jl_gc_post_free_page(pp); + } + } + uv_mutex_unlock(&gc_perm_lock); + jl_atomic_store(&gc_concurrent_sweep_running, 0); + } + } } }