Skip to content

Commit

Permalink
make pr independent of 49644
Browse files Browse the repository at this point in the history
  • Loading branch information
@d-netto
d-netto committed May 15, 2023
1 parent 4ed4195 commit 74a2c11
Show file tree
Hide file tree
Showing 4 changed files with 210 additions and 47 deletions.
57 changes: 37 additions & 20 deletions src/gc-pages.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -259,30 +259,15 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT
return info.meta;
}

// return a page to the freemap allocator
void jl_gc_free_page(void *p) JL_NOTSAFEPOINT
void jl_gc_pre_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT
{
// update the allocmap and freemap to indicate this contains a free entry
void *p = pp->page_meta->data;
// update the allocmap
struct jl_gc_metadata_ext info = page_metadata_ext(p);
uint32_t msk;
msk = (uint32_t)(1u << info.pagetable0_i);
assert(!(info.pagetable0->freemap[info.pagetable0_i32] & msk));
assert(info.pagetable0->allocmap[info.pagetable0_i32] & msk);
info.pagetable0->allocmap[info.pagetable0_i32] &= ~msk;
info.pagetable0->freemap[info.pagetable0_i32] |= msk;

msk = (uint32_t)(1u << info.pagetable1_i);
assert(info.pagetable1->allocmap0[info.pagetable1_i32] & msk);
if ((info.pagetable1->freemap0[info.pagetable1_i32] & msk) == 0)
info.pagetable1->freemap0[info.pagetable1_i32] |= msk;

msk = (uint32_t)(1u << info.pagetable_i);
assert(memory_map.allocmap1[info.pagetable_i32] & msk);
if ((memory_map.freemap1[info.pagetable_i32] & msk) == 0)
memory_map.freemap1[info.pagetable_i32] |= msk;

free(info.meta->ages);
info.meta->ages = NULL;

// tell the OS we don't need these pages right now
size_t decommit_size = GC_PAGE_SZ;
Expand All @@ -296,10 +281,18 @@ void jl_gc_free_page(void *p) JL_NOTSAFEPOINT
struct jl_gc_metadata_ext info = page_metadata_ext(otherp);
msk = (uint32_t)(1u << info.pagetable0_i);
if (info.pagetable0->allocmap[info.pagetable0_i32] & msk)
goto no_decommit;
return;
otherp = (void*)((char*)otherp + GC_PAGE_SZ);
}
}
pp->should_decommit = 1;
pp->decommit_size = decommit_size;
}

void jl_gc_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT
{
void *p = pp->page_meta->data;
size_t decommit_size = pp->decommit_size;
#ifdef _OS_WINDOWS_
VirtualFree(p, decommit_size, MEM_DECOMMIT);
#elif defined(MADV_FREE)
Expand All @@ -320,8 +313,32 @@ void jl_gc_free_page(void *p) JL_NOTSAFEPOINT
* the page when it sweeps pools?
*/
msan_unpoison(p, decommit_size);
}

// return a page to the freemap allocator
void jl_gc_post_free_page(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT
{
void *p = pp->page_meta->data;
// update the freemap to indicate this contains a free entry
struct jl_gc_metadata_ext info = page_metadata_ext(p);
uint32_t msk;
msk = (uint32_t)(1u << info.pagetable0_i);
assert(!(info.pagetable0->freemap[info.pagetable0_i32] & msk));
info.pagetable0->freemap[info.pagetable0_i32] |= msk;

msk = (uint32_t)(1u << info.pagetable1_i);
assert(info.pagetable1->allocmap0[info.pagetable1_i32] & msk);
if ((info.pagetable1->freemap0[info.pagetable1_i32] & msk) == 0)
info.pagetable1->freemap0[info.pagetable1_i32] |= msk;

msk = (uint32_t)(1u << info.pagetable_i);
assert(memory_map.allocmap1[info.pagetable_i32] & msk);
if ((memory_map.freemap1[info.pagetable_i32] & msk) == 0)
memory_map.freemap1[info.pagetable_i32] |= msk;

free(info.meta->ages);
info.meta->ages = NULL;

no_decommit:
// new pages are now available starting at max of lb and pagetable_i32
if (memory_map.lb > info.pagetable_i32)
memory_map.lb = info.pagetable_i32;
Expand Down
121 changes: 104 additions & 17 deletions src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,19 @@ extern "C" {

// Number of threads currently running the GC mark-loop
_Atomic(int) gc_n_threads_marking;
// Bitmap used to synchronized parallel/concurrent sweeping
_Atomic(uint64_t) gc_sweep_mask;
// Flag to indicate whether concurrent sweeping of object pools is running
_Atomic(uint64_t) gc_concurrent_sweep_running;
// `tid` of mutator thread that triggered GC
_Atomic(int) gc_master_tid;
// `tid` of first GC thread
int gc_first_tid;
// Mutex/cond used to synchronize sleep/wakeup of GC threads
uv_mutex_t gc_threads_lock;
uv_cond_t gc_threads_cond;
// Queue of packets corresponding to pages that need sweeping
gc_sweep_queue_t gc_sweep_queue;

// Linked list of callback functions

Expand Down Expand Up @@ -1391,9 +1397,26 @@ int jl_gc_classify_pools(size_t sz, int *osize)

int64_t lazy_freed_pages = 0;

// Returns pointer to terminal pointer of list rooted at *pfl.
static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_taggedvalue_t **pfl, int sweep_full, int osize) JL_NOTSAFEPOINT
static void gc_sweep_queue_push(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT
{
if (__unlikely(gc_sweep_queue.current == gc_sweep_queue.end)) {
// TODO: resize queue
jl_safe_printf("Sweep packet queue is full!\n");
abort();
}
memcpy(gc_sweep_queue.current, pp, sizeof(gc_sweep_packet_t));
gc_sweep_queue.current++;
}

static void sweep_packet(gc_sweep_packet_t *pp) JL_NOTSAFEPOINT
{
// Unpack sweep packet
jl_gc_pool_t *p = pp->pool;
jl_gc_pagemeta_t *pg = pp->page_meta;
jl_taggedvalue_t **pfl = &pp->freelist_front;
int osize = pp->osize;
uint8_t sweep_full = pp->sweep_full;

char *data = pg->data;
uint32_t *ages = pg->ages;
jl_taggedvalue_t *v = (jl_taggedvalue_t*)(data + GC_PAGE_OFFSET);
Expand All @@ -1417,7 +1440,7 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t
lazy_freed_pages++;
}
else {
jl_gc_free_page(data);
pp->should_free = 1;
}
nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize;
goto done;
Expand Down Expand Up @@ -1509,23 +1532,23 @@ static jl_taggedvalue_t **sweep_page(jl_gc_pool_t *p, jl_gc_pagemeta_t *pg, jl_t

done:
gc_time_count_page(freedall, pg_skpd);
gc_num.freed += (nfree - old_nfree) * osize;
return pfl;
jl_atomic_fetch_add((_Atomic(uint64_t) *)&gc_num.freed, (nfree - old_nfree) * osize);
pp->freelist_back = pfl;
}

// the actual sweeping over all allocated pages in a memory pool
STATIC_INLINE void sweep_pool_page(jl_taggedvalue_t ***pfl, jl_gc_pagemeta_t *pg, int sweep_full) JL_NOTSAFEPOINT
STATIC_INLINE void sweep_sched_packet(jl_gc_pagemeta_t *pg, int sweep_full) JL_NOTSAFEPOINT
{
int p_n = pg->pool_n;
int t_n = pg->thread_n;
jl_ptls_t ptls2 = gc_all_tls_states[t_n];
jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n];
int osize = pg->osize;
pfl[t_n * JL_GC_N_POOLS + p_n] = sweep_page(p, pg, pfl[t_n * JL_GC_N_POOLS + p_n], sweep_full, osize);
gc_sweep_packet_t pckt = {p, pg, NULL, NULL, osize, sweep_full, 0};
gc_sweep_queue_push(&pckt);
}

// sweep over a pagetable0 for all allocated pages
STATIC_INLINE int sweep_pool_pagetable0(jl_taggedvalue_t ***pfl, pagetable0_t *pagetable0, int sweep_full) JL_NOTSAFEPOINT
STATIC_INLINE int sweep_pool_pagetable0(pagetable0_t *pagetable0, int sweep_full) JL_NOTSAFEPOINT
{
unsigned ub = 0;
unsigned alloc = 0;
Expand All @@ -1541,15 +1564,15 @@ STATIC_INLINE int sweep_pool_pagetable0(jl_taggedvalue_t ***pfl, pagetable0_t *p
j += next;
line >>= next;
jl_gc_pagemeta_t *pg = pagetable0->meta[pg_i * 32 + j];
sweep_pool_page(pfl, pg, sweep_full);
sweep_sched_packet(pg, sweep_full);
}
}
pagetable0->ub = ub;
return alloc;
}

// sweep over pagetable1 for all pagetable0 that may contain allocated pages
STATIC_INLINE int sweep_pool_pagetable1(jl_taggedvalue_t ***pfl, pagetable1_t *pagetable1, int sweep_full) JL_NOTSAFEPOINT
STATIC_INLINE int sweep_pool_pagetable1(pagetable1_t *pagetable1, int sweep_full) JL_NOTSAFEPOINT
{
unsigned ub = 0;
unsigned alloc = 0;
Expand All @@ -1561,7 +1584,7 @@ STATIC_INLINE int sweep_pool_pagetable1(jl_taggedvalue_t ***pfl, pagetable1_t *p
j += next;
line >>= next;
pagetable0_t *pagetable0 = pagetable1->meta0[pg_i * 32 + j];
if (pagetable0 && !sweep_pool_pagetable0(pfl, pagetable0, sweep_full))
if (pagetable0 && !sweep_pool_pagetable0(pagetable0, sweep_full))
pagetable1->allocmap0[pg_i] &= ~(1 << j); // no allocations found, remember that for next time
}
if (pagetable1->allocmap0[pg_i]) {
Expand All @@ -1574,12 +1597,12 @@ STATIC_INLINE int sweep_pool_pagetable1(jl_taggedvalue_t ***pfl, pagetable1_t *p
}

// sweep over all memory for all pagetable1 that may contain allocated pages
static void sweep_pool_pagetable(jl_taggedvalue_t ***pfl, int sweep_full) JL_NOTSAFEPOINT
static void sweep_pool_pagetable(int sweep_full) JL_NOTSAFEPOINT
{
if (REGION2_PG_COUNT == 1) { // compile-time optimization
pagetable1_t *pagetable1 = memory_map.meta1[0];
if (pagetable1 != NULL)
sweep_pool_pagetable1(pfl, pagetable1, sweep_full);
sweep_pool_pagetable1(pagetable1, sweep_full);
return;
}
unsigned ub = 0;
Expand All @@ -1591,7 +1614,7 @@ static void sweep_pool_pagetable(jl_taggedvalue_t ***pfl, int sweep_full) JL_NOT
j += next;
line >>= next;
pagetable1_t *pagetable1 = memory_map.meta1[pg_i * 32 + j];
if (pagetable1 && !sweep_pool_pagetable1(pfl, pagetable1, sweep_full))
if (pagetable1 && !sweep_pool_pagetable1(pagetable1, sweep_full))
memory_map.allocmap1[pg_i] &= ~(1 << j); // no allocations found, remember that for next time
}
if (memory_map.allocmap1[pg_i]) {
Expand Down Expand Up @@ -1624,6 +1647,24 @@ static void gc_pool_sync_nfree(jl_gc_pagemeta_t *pg, jl_taggedvalue_t *last) JL_
pg->nfree = nfree;
}

void gc_sweep_pool_parallel(jl_ptls_t ptls, int master) JL_NOTSAFEPOINT
{
size_t lb;
size_t ub;
size_t n_pckts = gc_sweep_queue.current - gc_sweep_queue.begin;
if (master) {
lb = 0;
ub = n_pckts / (jl_n_gcthreads + 1);
}
else {
lb = ((ptls->tid - gc_first_tid + 1) * n_pckts) / (jl_n_gcthreads + 1);
ub = ((ptls->tid - gc_first_tid + 2) * n_pckts) / (jl_n_gcthreads + 1);
}
for (gc_sweep_packet_t *pp = gc_sweep_queue.begin + lb; pp < gc_sweep_queue.begin + ub; pp++) {
sweep_packet(pp);
}
}

// setup the data-structures for a sweep over all memory pools
static void gc_sweep_pool(int sweep_full)
{
Expand Down Expand Up @@ -1670,10 +1711,48 @@ static void gc_sweep_pool(int sweep_full)
p->newpages = NULL;
}
}
// wait for concurrent sweep to finish
if (jl_n_gcthreads != 0) {
while (jl_atomic_load(&gc_concurrent_sweep_running)) {
jl_cpu_pause();
}
}
gc_sweep_queue.current = gc_sweep_queue.begin;

sweep_pool_pagetable(sweep_full);

// wake up GC threads
uv_mutex_lock(&gc_threads_lock);
jl_atomic_store(&gc_sweep_mask, UINT64_MAX);
uv_cond_broadcast(&gc_threads_cond);
uv_mutex_unlock(&gc_threads_lock);

// the actual sweeping
sweep_pool_pagetable(pfl, sweep_full);
gc_sweep_pool_parallel(NULL, 1);
for (int i = 1; i <= jl_n_gcthreads; i++) {
while (jl_atomic_load(&gc_sweep_mask) & (1ull << i)) {
jl_cpu_pause();
}
}

// merge free lists and free empty pages that were not lazily sweeped
for (gc_sweep_packet_t *pp = gc_sweep_queue.begin; pp < gc_sweep_queue.current; pp++) {
int t_i = pp->page_meta->thread_n;
int i = pp->page_meta->pool_n;
if (pp->freelist_front != NULL) {
*pfl[t_i * JL_GC_N_POOLS + i] = pp->freelist_front;
pfl[t_i * JL_GC_N_POOLS + i] = pp->freelist_back;
}
else if (pp->should_free) {
if (jl_n_gcthreads == 0) {
jl_gc_pre_free_page(pp);
if (pp->should_decommit) {
jl_gc_free_page(pp);
}
jl_gc_post_free_page(pp);
}
}
}
// null out terminal pointers of free lists
for (int t_i = 0; t_i < n_threads; t_i++) {
jl_ptls_t ptls2 = gc_all_tls_states[t_i];
Expand All @@ -1683,6 +1762,9 @@ static void gc_sweep_pool(int sweep_full)
}
}
}
if (jl_n_gcthreads != 0) {
jl_atomic_store(&gc_concurrent_sweep_running, 1);
}

gc_time_pool_end(sweep_full);
}
Expand Down Expand Up @@ -2867,7 +2949,7 @@ void gc_mark_loop_barrier(void)

void gc_mark_clean_reclaim_sets(void)
{
// Clean up `reclaim-sets` and reset `top/bottom` of queues
// Clean up `reclaim-sets`
for (int i = 0; i < gc_n_threads; i++) {
jl_ptls_t ptls2 = gc_all_tls_states[i];
arraylist_t *reclaim_set2 = &ptls2->mark_queue.reclaim_set;
Expand Down Expand Up @@ -3599,6 +3681,11 @@ void jl_gc_init(void)
arraylist_new(&finalizer_list_marked, 0);
arraylist_new(&to_finalize, 0);

// Init sweep packet queue
gc_sweep_queue.current = gc_sweep_queue.begin =
(gc_sweep_packet_t *)malloc_s(GC_SWEEP_QUEUE_INIT_SIZE * sizeof(gc_sweep_queue_t));
gc_sweep_queue.end = gc_sweep_queue.begin + GC_SWEEP_QUEUE_INIT_SIZE;

gc_num.interval = default_collect_interval;
last_long_collect_interval = default_collect_interval;
gc_num.allocd = 0;
Expand Down
Loading

0 comments on commit 74a2c11

Please sign in to comment.