From a62156a72c5c97582ac79eed6afe61a1d9ae4ed5 Mon Sep 17 00:00:00 2001 From: d-netto Date: Thu, 5 Oct 2023 14:07:59 -0300 Subject: [PATCH] revert membalancer heuristics --- NEWS.md | 3 - doc/src/devdocs/gc.md | 12 +- src/gc-debug.c | 19 +-- src/gc-pages.c | 4 - src/gc.c | 341 +++++++++++++------------------------- src/gc.h | 12 +- src/jl_exported_funcs.inc | 1 - src/julia_threads.h | 4 +- 8 files changed, 128 insertions(+), 268 deletions(-) diff --git a/NEWS.md b/NEWS.md index 4c5aebb8f19df..a3e61238c1e61 100644 --- a/NEWS.md +++ b/NEWS.md @@ -14,9 +14,6 @@ Language changes Compiler/Runtime improvements ----------------------------- -* Updated GC heuristics to count allocated pages instead of individual objects ([#50144]). -* A new `LazyLibrary` type is exported from `Libdl` for use in building chained lazy library - loads, primarily to be used within JLLs ([#50074]). Command-line option changes --------------------------- diff --git a/doc/src/devdocs/gc.md b/doc/src/devdocs/gc.md index 942535f426b34..c072912e77c3f 100644 --- a/doc/src/devdocs/gc.md +++ b/doc/src/devdocs/gc.md @@ -67,12 +67,6 @@ This scheme eliminates the need of explicitly keeping a flag to indicate a full ## Heuristics GC heuristics tune the GC by changing the size of the allocation interval between garbage collections. - -The GC heuristics measure how big the heap size is after a collection and set the next -collection according to the algorithm described by https://dl.acm.org/doi/10.1145/3563323, -in summary, it argues that the heap target should have a square root relationship with the live heap, and that it should also be scaled by how fast the GC is freeing objects and how fast the mutators are allocating. -The heuristics measure the heap size by counting the number of pages that are in use and the objects that use malloc. Previously we measured the heap size by counting -the alive objects, but that doesn't take into account fragmentation which could lead to bad decisions, that also meant that we used thread local information (allocations) to make -decisions about a process wide (when to GC), measuring pages means the decision is global. - -The GC will do full collections when the heap size reaches 80% of the maximum allowed size. +If a GC was unproductive, then we increase the size of the allocation interval to allow objects more time to die. +If a GC returns a lot of space we can shrink the interval. The goal is to find a steady state where we are +allocating just about the same amount as we are collecting. diff --git a/src/gc-debug.c b/src/gc-debug.c index 7145531828853..bd188e5b74e5a 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -1,10 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" -#include "julia.h" #include -#include -#include #include // re-include assert.h without NDEBUG, @@ -1220,25 +1217,15 @@ JL_DLLEXPORT void jl_enable_gc_logging(int enable) { gc_logging_enabled = enable; } -void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect, int64_t live_bytes) JL_NOTSAFEPOINT { +void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect) JL_NOTSAFEPOINT { if (!gc_logging_enabled) { return; } - jl_safe_printf("\nGC: pause %.2fms. collected %fMB. %s %s\n", - pause/1e6, freed/(double)(1<<20), + jl_safe_printf("GC: pause %.2fms. collected %fMB. %s %s\n", + pause/1e6, freed/1e6, full ? "full" : "incr", recollect ? "recollect" : "" ); - - jl_safe_printf("Heap stats: bytes_mapped %.2f MB, bytes_resident %.2f MB,\nheap_size %.2f MB, heap_target %.2f MB, Fragmentation %.3f\n", - jl_atomic_load_relaxed(&gc_heap_stats.bytes_mapped)/(double)(1<<20), - jl_atomic_load_relaxed(&gc_heap_stats.bytes_resident)/(double)(1<<20), - // live_bytes/(double)(1<<20), live byes tracking is not accurate. - jl_atomic_load_relaxed(&gc_heap_stats.heap_size)/(double)(1<<20), - jl_atomic_load_relaxed(&gc_heap_stats.heap_target)/(double)(1<<20), - (double)live_bytes/(double)jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - ); - // Should fragmentation use bytes_resident instead of heap_size? } #ifdef __cplusplus diff --git a/src/gc-pages.c b/src/gc-pages.c index 696f0831762be..0cff5971d73b0 100644 --- a/src/gc-pages.c +++ b/src/gc-pages.c @@ -52,8 +52,6 @@ char *jl_gc_try_alloc_pages_(int pg_cnt) JL_NOTSAFEPOINT // round data pointer up to the nearest gc_page_data-aligned // boundary if mmap didn't already do so. mem = (char*)gc_page_data(mem + GC_PAGE_SZ - 1); - jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_mapped, pages_sz); - jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, pages_sz); return mem; } @@ -117,7 +115,6 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT // try to get page from `pool_freed` meta = pop_lf_back(&global_page_pool_freed); if (meta != NULL) { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, GC_PAGE_SZ); gc_alloc_map_set(meta->data, GC_PAGE_ALLOCATED); goto exit; } @@ -191,7 +188,6 @@ void jl_gc_free_page(jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT madvise(p, decommit_size, MADV_DONTNEED); #endif msan_unpoison(p, decommit_size); - jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, -decommit_size); } #ifdef __cplusplus diff --git a/src/gc.c b/src/gc.c index 42a9daa01a747..526ab0a964179 100644 --- a/src/gc.c +++ b/src/gc.c @@ -47,8 +47,6 @@ static jl_gc_callback_list_t *gc_cblist_pre_gc; static jl_gc_callback_list_t *gc_cblist_post_gc; static jl_gc_callback_list_t *gc_cblist_notify_external_alloc; static jl_gc_callback_list_t *gc_cblist_notify_external_free; -static jl_gc_callback_list_t *gc_cblist_notify_gc_pressure; -typedef void (*jl_gc_cb_notify_gc_pressure_t)(void); #define gc_invoke_callbacks(ty, list, args) \ do { \ @@ -135,14 +133,6 @@ JL_DLLEXPORT void jl_gc_set_cb_notify_external_free(jl_gc_cb_notify_external_fre jl_gc_deregister_callback(&gc_cblist_notify_external_free, (jl_gc_cb_func_t)cb); } -JL_DLLEXPORT void jl_gc_set_cb_notify_gc_pressure(jl_gc_cb_notify_gc_pressure_t cb, int enable) -{ - if (enable) - jl_gc_register_callback(&gc_cblist_notify_gc_pressure, (jl_gc_cb_func_t)cb); - else - jl_gc_deregister_callback(&gc_cblist_notify_gc_pressure, (jl_gc_cb_func_t)cb); -} - // Protect all access to `finalizer_list_marked` and `to_finalize`. // For accessing `ptls->finalizers`, the lock is needed if a thread // is going to realloc the buffer (of its own list) or accessing the @@ -193,8 +183,6 @@ jl_gc_num_t gc_num = {0}; static size_t last_long_collect_interval; int gc_n_threads; jl_ptls_t* gc_all_tls_states; -gc_heapstatus_t gc_heap_stats = {0}; -int next_sweep_full = 0; const uint64_t _jl_buff_tag[3] = {0x4eadc0004eadc000ull, 0x4eadc0004eadc000ull, 0x4eadc0004eadc000ull}; // aka 0xHEADER00 JL_DLLEXPORT uintptr_t jl_get_buff_tag(void) { @@ -682,27 +670,19 @@ static int64_t last_gc_total_bytes = 0; #ifdef _P64 typedef uint64_t memsize_t; static const size_t default_collect_interval = 5600 * 1024 * sizeof(void*); +static const size_t max_collect_interval = 1250000000UL; static size_t total_mem; // We expose this to the user/ci as jl_gc_set_max_memory static memsize_t max_total_memory = (memsize_t) 2 * 1024 * 1024 * 1024 * 1024 * 1024; #else typedef uint32_t memsize_t; static const size_t default_collect_interval = 3200 * 1024 * sizeof(void*); +static const size_t max_collect_interval = 500000000UL; // Work really hard to stay within 2GB // Alternative is to risk running out of address space // on 32 bit architectures. -#define MAX32HEAP 1536 * 1024 * 1024 -static memsize_t max_total_memory = (memsize_t) MAX32HEAP; +static memsize_t max_total_memory = (memsize_t) 2 * 1024 * 1024 * 1024; #endif -// heuristic stuff for https://dl.acm.org/doi/10.1145/3563323 -static uint64_t old_pause_time = 0; -static uint64_t old_mut_time = 0; -static uint64_t old_heap_size = 0; -static uint64_t old_alloc_diff = 0; -static uint64_t old_freed_diff = 0; -static uint64_t gc_end_time = 0; -static int thrash_counter = 0; -static int thrashing = 0; // global variables for GC stats // Resetting the object to a young object, this is used when marking the @@ -755,7 +735,6 @@ static int64_t scanned_bytes; // young bytes scanned while marking static int64_t perm_scanned_bytes; // old bytes scanned while marking int prev_sweep_full = 1; int current_sweep_full = 0; -int under_pressure = 0; // Full collection heuristics static int64_t pool_live_bytes = 0; @@ -943,7 +922,7 @@ void gc_setmark_buf(jl_ptls_t ptls, void *o, uint8_t mark_mode, size_t minsz) JL STATIC_INLINE void maybe_collect(jl_ptls_t ptls) { - if (jl_atomic_load_relaxed(&gc_heap_stats.heap_size) >= jl_atomic_load_relaxed(&gc_heap_stats.heap_target) || jl_gc_debug_check_other()) { + if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) { jl_gc_collect(JL_GC_AUTO); } else { @@ -1032,13 +1011,6 @@ STATIC_INLINE jl_value_t *jl_gc_big_alloc_inner(jl_ptls_t ptls, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); jl_atomic_store_relaxed(&ptls->gc_num.bigalloc, jl_atomic_load_relaxed(&ptls->gc_num.bigalloc) + 1); - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + allocsz < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + allocsz); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + allocsz); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } #ifdef MEMDEBUG memset(v, 0xee, allocsz); #endif @@ -1091,8 +1063,6 @@ static bigval_t **sweep_big_list(int sweep_full, bigval_t **pv) JL_NOTSAFEPOINT if (nxt) nxt->prev = pv; gc_num.freed += v->sz&~3; - jl_atomic_store_relaxed(&gc_heap_stats.heap_size, - jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - (v->sz&~3)); #ifdef MEMDEBUG memset(v, 0xbb, v->sz&~3); #endif @@ -1152,13 +1122,6 @@ void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT jl_ptls_t ptls = jl_current_task->ptls; jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + sz < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + sz); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + sz); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } } static void combine_thread_gc_counts(jl_gc_num_t *dest) JL_NOTSAFEPOINT @@ -1171,16 +1134,12 @@ static void combine_thread_gc_counts(jl_gc_num_t *dest) JL_NOTSAFEPOINT jl_ptls_t ptls = gc_all_tls_states[i]; if (ptls) { dest->allocd += (jl_atomic_load_relaxed(&ptls->gc_num.allocd) + gc_num.interval); + dest->freed += jl_atomic_load_relaxed(&ptls->gc_num.freed); dest->malloc += jl_atomic_load_relaxed(&ptls->gc_num.malloc); dest->realloc += jl_atomic_load_relaxed(&ptls->gc_num.realloc); dest->poolalloc += jl_atomic_load_relaxed(&ptls->gc_num.poolalloc); dest->bigalloc += jl_atomic_load_relaxed(&ptls->gc_num.bigalloc); - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - uint64_t free_acc = jl_atomic_load_relaxed(&ptls->gc_num.free_acc); - dest->freed += jl_atomic_load_relaxed(&ptls->gc_num.free_acc); - jl_atomic_store_relaxed(&gc_heap_stats.heap_size, alloc_acc - free_acc + jl_atomic_load_relaxed(&gc_heap_stats.heap_size)); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, 0); + dest->freecall += jl_atomic_load_relaxed(&ptls->gc_num.freecall); } } } @@ -1237,8 +1196,6 @@ static void jl_gc_free_array(jl_array_t *a) JL_NOTSAFEPOINT jl_free_aligned(d); else free(d); - jl_atomic_store_relaxed(&gc_heap_stats.heap_size, - jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_array_nbytes(a)); gc_num.freed += jl_array_nbytes(a); gc_num.freecall++; } @@ -1313,7 +1270,6 @@ static NOINLINE jl_taggedvalue_t *gc_add_page(jl_gc_pool_t *p) JL_NOTSAFEPOINT set_page_metadata(pg); push_lf_back(&ptls->page_metadata_allocd, pg); jl_taggedvalue_t *fl = gc_reset_page(ptls, p, pg); - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, GC_PAGE_SZ); p->newpages = fl; return fl; } @@ -1511,7 +1467,6 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } else { gc_alloc_map_set(pg->data, GC_PAGE_LAZILY_FREED); - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -GC_PAGE_SZ); if (keep_as_local_buffer) { push_lf_back(buffered, pg); } @@ -3198,11 +3153,6 @@ JL_DLLEXPORT int64_t jl_gc_live_bytes(void) return live_bytes; } -double jl_gc_smooth(uint64_t old_val, uint64_t new_val, double factor) -{ - return factor * old_val + (1.0-factor) * new_val; -} - size_t jl_maxrss(void); // Only one thread should be running in this function @@ -3217,8 +3167,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) jl_gc_markqueue_t *mq = &ptls->mark_queue; uint64_t gc_start_time = jl_hrtime(); - uint64_t mutator_time = gc_start_time - gc_end_time; - uint64_t before_free_heap_size = jl_atomic_load_relaxed(&gc_heap_stats.heap_size); int64_t last_perm_scanned_bytes = perm_scanned_bytes; uint64_t start_mark_time = jl_hrtime(); JL_PROBE_GC_MARK_BEGIN(); @@ -3309,14 +3257,19 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) uint64_t mark_time = end_mark_time - start_mark_time; gc_num.mark_time = mark_time; gc_num.total_mark_time += mark_time; + int64_t allocd = gc_num.allocd; gc_settime_postmark_end(); // marking is over // Flush everything in mark cache gc_sync_all_caches_nolock(ptls); + int64_t live_sz_ub = live_bytes + allocd; + int64_t live_sz_est = scanned_bytes + perm_scanned_bytes; + int64_t estimate_freed = live_sz_ub - live_sz_est; gc_verify(ptls); + gc_stats_all_pool(); gc_stats_big_obj(); objprofile_printall(); @@ -3325,17 +3278,42 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) if (!prev_sweep_full) promoted_bytes += perm_scanned_bytes - last_perm_scanned_bytes; // 5. next collection decision - int remset_nptr = 0; - int sweep_full = next_sweep_full; - int recollect = 0; + int not_freed_enough = (collection == JL_GC_AUTO) && estimate_freed < (7*(allocd/10)); + int nptr = 0; assert(gc_n_threads); for (int i = 0; i < gc_n_threads; i++) { jl_ptls_t ptls2 = gc_all_tls_states[i]; if (ptls2 != NULL) - remset_nptr += ptls2->heap.remset_nptr; + nptr += ptls2->heap.remset_nptr; } - (void)remset_nptr; //Use this information for something? + // many pointers in the intergen frontier => "quick" mark is not quick + int large_frontier = nptr*sizeof(void*) >= default_collect_interval; + int sweep_full = 0; + int recollect = 0; + + // update heuristics only if this GC was automatically triggered + if (collection == JL_GC_AUTO) { + if (large_frontier) { + sweep_full = 1; + gc_num.interval = last_long_collect_interval; + } + if (not_freed_enough || large_frontier) { + gc_num.interval = gc_num.interval * 2; + } + + size_t maxmem = 0; +#ifdef _P64 + // on a big memory machine, increase max_collect_interval to totalmem / nthreads / 2 + maxmem = total_mem / (gc_n_threads - jl_n_gcthreads) / 2; +#endif + if (maxmem < max_collect_interval) + maxmem = max_collect_interval; + if (gc_num.interval > maxmem) { + sweep_full = 1; + gc_num.interval = maxmem; + } + } // If the live data outgrows the suggested max_total_memory // we keep going with minimum intervals and full gcs until @@ -3352,6 +3330,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) // on the first collection after sweep_full, and the current scan perm_scanned_bytes = 0; promoted_bytes = 0; + last_long_collect_interval = gc_num.interval; } scanned_bytes = 0; pool_live_bytes = 0; @@ -3378,10 +3357,9 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) if (sweep_full) gc_sweep_perm_alloc(); } - JL_PROBE_GC_SWEEP_END(); - gc_end_time = jl_hrtime(); + uint64_t gc_end_time = jl_hrtime(); uint64_t pause = gc_end_time - gc_start_time; uint64_t sweep_time = gc_end_time - start_sweep_time; gc_num.total_sweep_time += sweep_time; @@ -3393,56 +3371,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) gc_num.last_incremental_sweep = gc_end_time; } - size_t heap_size = jl_atomic_load_relaxed(&gc_heap_stats.heap_size); - double target_allocs = 0.0; - double min_interval = default_collect_interval; - if (collection == JL_GC_AUTO) { - uint64_t alloc_diff = before_free_heap_size - old_heap_size; - uint64_t freed_diff = before_free_heap_size - heap_size; - double alloc_smooth_factor = 0.95; - double collect_smooth_factor = 0.5; - double tuning_factor = 0.03; - double alloc_mem = jl_gc_smooth(old_alloc_diff, alloc_diff, alloc_smooth_factor); - double alloc_time = jl_gc_smooth(old_mut_time, mutator_time + sweep_time, alloc_smooth_factor); // Charge sweeping to the mutator - double gc_mem = jl_gc_smooth(old_freed_diff, freed_diff, collect_smooth_factor); - double gc_time = jl_gc_smooth(old_pause_time, pause - sweep_time, collect_smooth_factor); - old_alloc_diff = alloc_diff; - old_mut_time = mutator_time; - old_freed_diff = freed_diff; - old_pause_time = pause; - old_heap_size = heap_size; // TODO: Update these values dynamically instead of just during the GC - if (gc_time > alloc_time * 95 && !(thrash_counter < 4)) - thrash_counter += 1; - else if (thrash_counter > 0) - thrash_counter -= 1; - if (alloc_mem != 0 && alloc_time != 0 && gc_mem != 0 && gc_time != 0 ) { - double alloc_rate = alloc_mem/alloc_time; - double gc_rate = gc_mem/gc_time; - target_allocs = sqrt(((double)heap_size/min_interval * alloc_rate)/(gc_rate * tuning_factor)); // work on multiples of min interval - } - } - if (thrashing == 0 && thrash_counter >= 3) - thrashing = 1; - else if (thrashing == 1 && thrash_counter <= 2) - thrashing = 0; // maybe we should report this to the user or error out? - - int bad_result = (target_allocs*min_interval + heap_size) > 2 * jl_atomic_load_relaxed(&gc_heap_stats.heap_target); // Don't follow through on a bad decision - if (target_allocs == 0.0 || thrashing || bad_result) // If we are thrashing go back to default - target_allocs = 2*sqrt((double)heap_size/min_interval); - uint64_t target_heap = (uint64_t)target_allocs*min_interval + heap_size; - if (target_heap > max_total_memory && !thrashing) // Allow it to go over if we are thrashing if we die we die - target_heap = max_total_memory; - else if (target_heap < default_collect_interval) - target_heap = default_collect_interval; - jl_atomic_store_relaxed(&gc_heap_stats.heap_target, target_heap); - - double old_ratio = (double)promoted_bytes/(double)heap_size; - if (heap_size > max_total_memory * 0.8 || old_ratio > 0.15) - next_sweep_full = 1; - else - next_sweep_full = 0; - if (heap_size > max_total_memory * 0.8 || thrashing) - under_pressure = 1; // sweeping is over // 7. if it is a quick sweep, put back the remembered objects in queued state // so that we don't trigger the barrier again on them. @@ -3474,19 +3402,55 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) } #endif - _report_gc_finished(pause, gc_num.freed, sweep_full, recollect, live_bytes); - uint64_t max_memory = last_live_bytes + gc_num.allocd; - if (max_memory > gc_num.max_memory) { - gc_num.max_memory = max_memory; - } + _report_gc_finished(pause, gc_num.freed, sweep_full, recollect); + gc_final_pause_end(gc_start_time, gc_end_time); gc_time_sweep_pause(gc_end_time, allocd, live_bytes, estimate_freed, sweep_full); gc_num.full_sweep += sweep_full; + uint64_t max_memory = last_live_bytes + gc_num.allocd; + if (max_memory > gc_num.max_memory) { + gc_num.max_memory = max_memory; + } + last_live_bytes = live_bytes; + // Can't call inc_live_bytes here because we already added allocd + // to the graph earlier live_bytes += -gc_num.freed + gc_num.allocd; jl_timing_counter_dec(JL_TIMING_COUNTER_HeapSize, gc_num.freed); + if (collection == JL_GC_AUTO) { + //If we aren't freeing enough or are seeing lots and lots of pointers let it increase faster + if (!not_freed_enough || large_frontier) { + int64_t tot = 2 * (live_bytes + gc_num.allocd) / 3; + if (gc_num.interval > tot) { + gc_num.interval = tot; + last_long_collect_interval = tot; + } + // If the current interval is larger than half the live data decrease the interval + } + else { + int64_t half = (live_bytes / 2); + if (gc_num.interval > half) + gc_num.interval = half; + } + + // But never go below default + if (gc_num.interval < default_collect_interval) gc_num.interval = default_collect_interval; + } + + if (gc_num.interval + live_bytes > max_total_memory) { + if (live_bytes < max_total_memory) { + gc_num.interval = max_total_memory - live_bytes; + last_long_collect_interval = max_total_memory - live_bytes; + } + else { + // We can't stay under our goal so let's go back to + // the minimum interval and hope things get better + gc_num.interval = default_collect_interval; + } + } + gc_time_summary(sweep_full, t_start, gc_end_time, gc_num.freed, live_bytes, gc_num.interval, pause, gc_num.time_to_safepoint, @@ -3594,10 +3558,6 @@ JL_DLLEXPORT void jl_gc_collect(jl_gc_collection_t collection) gc_invoke_callbacks(jl_gc_cb_post_gc_t, gc_cblist_post_gc, (collection)); - if (under_pressure) - gc_invoke_callbacks(jl_gc_cb_notify_gc_pressure_t, - gc_cblist_notify_gc_pressure, ()); - under_pressure = 0; #ifdef _OS_WINDOWS_ SetLastError(last_error); #endif @@ -3685,7 +3645,7 @@ void jl_gc_init(void) arraylist_new(&finalizer_list_marked, 0); arraylist_new(&to_finalize, 0); - jl_atomic_store_relaxed(&gc_heap_stats.heap_target, default_collect_interval); + gc_num.interval = default_collect_interval; last_long_collect_interval = default_collect_interval; gc_num.allocd = 0; @@ -3696,7 +3656,13 @@ void jl_gc_init(void) total_mem = uv_get_total_memory(); uint64_t constrained_mem = uv_get_constrained_memory(); if (constrained_mem > 0 && constrained_mem < total_mem) - jl_gc_set_max_memory(constrained_mem - 250*1024*1024); // LLVM + other libraries need some amount of memory + total_mem = constrained_mem; + double percent; + if (total_mem < 128e9) + percent = total_mem * 2.34375e-12 + 0.6; // 60% at 0 gigs and 90% at 128 to not + else // overcommit too much on memory contrained devices + percent = 0.9; + max_total_memory = total_mem * percent; #endif if (jl_options.heap_size_hint) jl_gc_set_max_memory(jl_options.heap_size_hint - 250*1024*1024); @@ -3708,11 +3674,7 @@ JL_DLLEXPORT void jl_gc_set_max_memory(uint64_t max_mem) { if (max_mem > 0 && max_mem < (uint64_t)1 << (sizeof(memsize_t) * 8 - 1)) { - #ifdef _P64 max_total_memory = max_mem; - #else - max_total_memory = max_mem < MAX32HEAP ? max_mem : MAX32HEAP; - #endif } } @@ -3741,13 +3703,6 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + sz < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + sz); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + sz); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } } return data; } @@ -3764,13 +3719,6 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + nm*sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + sz < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + sz * nm); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + sz * nm); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } } return data; } @@ -3782,13 +3730,10 @@ JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz) free(p); if (pgcstack != NULL && ct->world_age) { jl_ptls_t ptls = ct->ptls; - uint64_t free_acc = jl_atomic_load_relaxed(&ptls->gc_num.free_acc); - if (free_acc + sz < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, free_acc + sz); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -(free_acc + sz)); - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, 0); - } + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + sz); + jl_atomic_store_relaxed(&ptls->gc_num.freecall, + jl_atomic_load_relaxed(&ptls->gc_num.freecall) + 1); } } @@ -3800,32 +3745,14 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size if (data != NULL && pgcstack != NULL && ct->world_age) { jl_ptls_t ptls = ct->ptls; maybe_collect(ptls); - if (!(sz < old)) + if (sz < old) + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + (old - sz)); + else jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (sz - old)); jl_atomic_store_relaxed(&ptls->gc_num.realloc, jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); - - int64_t diff = sz - old; - if (diff < 0) { - diff = -diff; - uint64_t free_acc = jl_atomic_load_relaxed(&ptls->gc_num.free_acc); - if (free_acc + diff < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, free_acc + diff); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -(free_acc + diff)); - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, 0); - } - } - else { - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + diff < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + diff); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + diff); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } - } } return data; } @@ -3896,7 +3823,10 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) size_t allocsz = LLT_ALIGN(sz, JL_CACHE_BYTE_ALIGNMENT); if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); - + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); + jl_atomic_store_relaxed(&ptls->gc_num.malloc, + jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -3904,18 +3834,6 @@ JL_DLLEXPORT void *jl_gc_managed_malloc(size_t sz) void *b = malloc_cache_align(allocsz); if (b == NULL) jl_throw(jl_memory_exception); - - jl_atomic_store_relaxed(&ptls->gc_num.allocd, - jl_atomic_load_relaxed(&ptls->gc_num.allocd) + allocsz); - jl_atomic_store_relaxed(&ptls->gc_num.malloc, - jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + allocsz < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + allocsz); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + allocsz); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } #ifdef _OS_WINDOWS_ SetLastError(last_error); #endif @@ -3930,11 +3848,24 @@ static void *gc_managed_realloc_(jl_ptls_t ptls, void *d, size_t sz, size_t olds { if (can_collect) maybe_collect(ptls); - int is_old_marked = jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED; + size_t allocsz = LLT_ALIGN(sz, JL_CACHE_BYTE_ALIGNMENT); if (allocsz < sz) // overflow in adding offs, size was "negative" jl_throw(jl_memory_exception); + if (jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED) { + ptls->gc_cache.perm_scanned_bytes += allocsz - oldsz; + live_bytes += allocsz - oldsz; + } + else if (allocsz < oldsz) + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + (oldsz - allocsz)); + else + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (allocsz - oldsz)); + jl_atomic_store_relaxed(&ptls->gc_num.realloc, + jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); + int last_errno = errno; #ifdef _OS_WINDOWS_ DWORD last_error = GetLastError(); @@ -3950,40 +3881,7 @@ static void *gc_managed_realloc_(jl_ptls_t ptls, void *d, size_t sz, size_t olds SetLastError(last_error); #endif errno = last_errno; - // gc_managed_realloc_ is currently used exclusively for resizing array buffers. - if (is_old_marked) { - ptls->gc_cache.perm_scanned_bytes += allocsz - oldsz; - inc_live_bytes(allocsz - oldsz); - } - else if (!(allocsz < oldsz)) - jl_atomic_store_relaxed(&ptls->gc_num.allocd, - jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (allocsz - oldsz)); - jl_atomic_store_relaxed(&ptls->gc_num.realloc, - jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); - - int64_t diff = allocsz - oldsz; - if (diff < 0) { - diff = -diff; - uint64_t free_acc = jl_atomic_load_relaxed(&ptls->gc_num.free_acc); - if (free_acc + diff < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, free_acc + diff); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -(free_acc + diff)); - jl_atomic_store_relaxed(&ptls->gc_num.free_acc, 0); - } - } - else { - uint64_t alloc_acc = jl_atomic_load_relaxed(&ptls->gc_num.alloc_acc); - if (alloc_acc + diff < 16*1024) - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, alloc_acc + diff); - else { - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, alloc_acc + diff); - jl_atomic_store_relaxed(&ptls->gc_num.alloc_acc, 0); - } - } - if (allocsz > oldsz) { - maybe_record_alloc_to_profile((jl_value_t*)b, allocsz - oldsz, (jl_datatype_t*)jl_buff_tag); - } + maybe_record_alloc_to_profile((jl_value_t*)b, sz, jl_gc_unknown_type_tag); return b; } @@ -4056,7 +3954,6 @@ static void *gc_perm_alloc_large(size_t sz, int zero, unsigned align, unsigned o #ifdef _OS_WINDOWS_ SetLastError(last_error); #endif - jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size,sz); errno = last_errno; jl_may_leak(base); assert(align > 0); diff --git a/src/gc.h b/src/gc.h index 7985a3ddd43ef..b094f64a0a013 100644 --- a/src/gc.h +++ b/src/gc.h @@ -10,7 +10,6 @@ #define JL_GC_H #include -#include #include #include #include @@ -263,13 +262,6 @@ typedef struct { pagetable1_t *meta1[REGION2_PG_COUNT]; } pagetable_t; -typedef struct { - _Atomic(size_t) bytes_mapped; - _Atomic(size_t) bytes_resident; - _Atomic(size_t) heap_size; - _Atomic(size_t) heap_target; -} gc_heapstatus_t; - #define GC_PAGE_UNMAPPED 0 #define GC_PAGE_ALLOCATED 1 #define GC_PAGE_LAZILY_FREED 2 @@ -387,7 +379,6 @@ extern int64_t buffered_pages; extern int gc_first_tid; extern int gc_n_threads; extern jl_ptls_t* gc_all_tls_states; -extern gc_heapstatus_t gc_heap_stats; STATIC_INLINE bigval_t *bigval_header(jl_taggedvalue_t *o) JL_NOTSAFEPOINT { @@ -654,8 +645,7 @@ void gc_count_pool(void); size_t jl_array_nbytes(jl_array_t *a) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_enable_gc_logging(int enable); -JL_DLLEXPORT uint32_t jl_get_num_stack_mappings(void); -void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect, int64_t live_bytes) JL_NOTSAFEPOINT; +void _report_gc_finished(uint64_t pause, uint64_t freed, int full, int recollect) JL_NOTSAFEPOINT; #ifdef __cplusplus } diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index c503eeb4407b3..153b2f9d9c9d9 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -193,7 +193,6 @@ XX(jl_gc_schedule_foreign_sweepfunc) \ XX(jl_gc_set_cb_notify_external_alloc) \ XX(jl_gc_set_cb_notify_external_free) \ - XX(jl_gc_set_cb_notify_gc_pressure) \ XX(jl_gc_set_cb_post_gc) \ XX(jl_gc_set_cb_pre_gc) \ XX(jl_gc_set_cb_root_scanner) \ diff --git a/src/julia_threads.h b/src/julia_threads.h index b8276682ee359..5fc52bad20b11 100644 --- a/src/julia_threads.h +++ b/src/julia_threads.h @@ -130,12 +130,12 @@ typedef struct { typedef struct { _Atomic(int64_t) allocd; + _Atomic(int64_t) freed; _Atomic(uint64_t) malloc; _Atomic(uint64_t) realloc; _Atomic(uint64_t) poolalloc; _Atomic(uint64_t) bigalloc; - _Atomic(int64_t) free_acc; - _Atomic(uint64_t) alloc_acc; + _Atomic(uint64_t) freecall; } jl_thread_gc_num_t; typedef struct {