diff --git a/src/Makefile b/src/Makefile index 747090a8dcd83..c78df75bdceb5 100644 --- a/src/Makefile +++ b/src/Makefile @@ -44,7 +44,7 @@ SRCS := \ jltypes gf typemap smallintset ast builtins module interpreter symbol \ dlload sys init task array genericmemory staticdata toplevel jl_uv datatype \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ - threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \ + threading scheduler stackwalk gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall diff --git a/src/gc-page-profiler.c b/src/gc-page-profiler.c new file mode 100644 index 0000000000000..ebb39abe5defb --- /dev/null +++ b/src/gc-page-profiler.c @@ -0,0 +1,237 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#include "gc-page-profiler.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// whether page profiling is enabled +int page_profile_enabled; +// number of pages written +size_t page_profile_pages_written; +// stream to write page profile to +ios_t *page_profile_stream; +// mutex for page profile +uv_mutex_t page_profile_lock; + +#define GC_SERIALIZER_EMPTY ((const char *)0x1) +#define GC_SERIALIZER_GARBAGE ((const char *)0x2) + +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT +{ + gc_page_profiler_serializer_t serializer; + serializer.length = 0; + serializer.capacity = GC_PAGE_SZ; + if (__unlikely(page_profile_enabled)) { + serializer.buffer = (char const **)calloc_s(serializer.capacity); + } + else { + serializer.buffer = NULL; + } + return serializer; +} + +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, + jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + memset(serializer->buffer, 0, serializer->capacity); + serializer->length = 0; + serializer->data = (char *)pg->data; + serializer->osize = pg->osize; + } +} + +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + free(serializer->buffer); +} + +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, + const char *str) JL_NOTSAFEPOINT +{ + serializer->buffer[serializer->length++] = str; +} + +void gc_enable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 1; +} + +void gc_disable_page_profile(void) JL_NOTSAFEPOINT +{ + page_profile_enabled = 0; +} + +void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_EMPTY); + } +} + +void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + gc_page_serializer_write(serializer, GC_SERIALIZER_GARBAGE); + } +} + +void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, + jl_taggedvalue_t *v) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + const char *name = jl_typeof_str(jl_valueof(v)); + gc_page_serializer_write(serializer, name); + } +} + +// inspired by the function from `src/gc-heap-snapshot.cpp` +void gc_page_profile_print_escape_json(ios_t *stream, const char *s, + size_t len) JL_NOTSAFEPOINT +{ + { + for (size_t i = 0; i < len; i++) { + const char *c = &s[i]; + switch (*c) { + case '\\': ios_write(stream, "\\\\", 2); break; + case '\b': ios_write(stream, "\\b", 2); break; + case '\f': ios_write(stream, "\\f", 2); break; + case '\n': ios_write(stream, "\\n", 2); break; + case '\r': ios_write(stream, "\\r", 2); break; + case '\t': ios_write(stream, "\\t", 2); break; + default: + if (('\x00' <= *c) & (*c <= '\x1f')) { + ios_printf(stream, "\\u%04x", (int)*c); + } + else { + ios_putc(*c, stream); + } + } + } + } +} + +void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "{"); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"address\": \"%p\",", serializer->data); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"object_size\": %d,", serializer->osize); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"objects\": ["); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "]"); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "}"); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + } +} + +void gc_page_profile_write_comma(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + if (page_profile_pages_written > 0) { + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, ","); + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + } + } +} + +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) + JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + // write to file + uv_mutex_lock(&page_profile_lock); + gc_page_profile_write_comma(serializer); + gc_page_profile_write_preamble(serializer); + char str[GC_TYPE_STR_MAXLEN]; + for (size_t i = 0; i < serializer->length; i++) { + memset(str, 0, GC_TYPE_STR_MAXLEN); + if (serializer->buffer[i] == GC_SERIALIZER_EMPTY) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"empty\","); + } + else if (serializer->buffer[i] == GC_SERIALIZER_GARBAGE) { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"garbage\","); + } + else { + snprintf(str, GC_TYPE_STR_MAXLEN, "\"%s\",", serializer->buffer[i]); + } + if (i == serializer->length - 1) { + str[strlen(str) - 1] = '\0'; + } + gc_page_profile_print_escape_json(page_profile_stream, str, strlen(str)); + } + gc_page_profile_write_epilogue(serializer); + page_profile_pages_written++; + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_preamble(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "{"); + gc_page_profile_print_escape_json(stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "\"pages\": ["); + gc_page_profile_print_escape_json(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +void gc_page_profile_write_json_epilogue(ios_t *stream) JL_NOTSAFEPOINT +{ + if (__unlikely(page_profile_enabled)) { + uv_mutex_lock(&page_profile_lock); + char str[GC_TYPE_STR_MAXLEN]; + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "]"); + gc_page_profile_print_escape_json(stream, str, strlen(str)); + memset(str, 0, GC_TYPE_STR_MAXLEN); + snprintf(str, GC_TYPE_STR_MAXLEN, "}"); + gc_page_profile_print_escape_json(stream, str, strlen(str)); + uv_mutex_unlock(&page_profile_lock); + } +} + +JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream) +{ + gc_enable_page_profile(); + page_profile_stream = stream; + gc_page_profile_write_json_preamble(stream); + jl_gc_collect(JL_GC_FULL); + gc_page_profile_write_json_epilogue(stream); + gc_disable_page_profile(); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/gc-page-profiler.h b/src/gc-page-profiler.h new file mode 100644 index 0000000000000..0307f1670daa7 --- /dev/null +++ b/src/gc-page-profiler.h @@ -0,0 +1,44 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + +#ifndef GC_PAGE_PROFILER_H +#define GC_PAGE_PROFILER_H + +#include "gc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define GC_TYPE_STR_MAXLEN (512) + +typedef struct { + size_t length; + size_t capacity; + char const **buffer; + char *data; + int osize; +} gc_page_profiler_serializer_t; + +// mutex for page profile +extern uv_mutex_t page_profile_lock; + +// Serializer functions +gc_page_profiler_serializer_t gc_page_serializer_create(void) JL_NOTSAFEPOINT; +void gc_page_serializer_init(gc_page_profiler_serializer_t *serializer, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT; +void gc_page_serializer_destroy(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_serializer_write(gc_page_profiler_serializer_t *serializer, const char *str) JL_NOTSAFEPOINT; +// Page profile functions +void gc_page_profile_write_preamble(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_epilogue(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_empty_page(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_garbage(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; +void gc_page_profile_write_live_obj(gc_page_profiler_serializer_t *serializer, jl_taggedvalue_t *v) JL_NOTSAFEPOINT; +void gc_enable_page_profile(void) JL_NOTSAFEPOINT; +void gc_disable_page_profile(void) JL_NOTSAFEPOINT; +void gc_page_profile_write_to_file(gc_page_profiler_serializer_t *serializer) JL_NOTSAFEPOINT; + +#ifdef __cplusplus +} +#endif + +#endif // GC_PAGE_PROFILER_H diff --git a/src/gc.c b/src/gc.c index 04004f267d01c..29a47a30bce21 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1,6 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "gc.h" +#include "gc-page-profiler.h" #include "julia.h" #include "julia_gcext.h" #include "julia_assert.h" @@ -1427,7 +1428,7 @@ STATIC_INLINE void gc_dump_page_utilization_data(void) JL_NOTSAFEPOINT int64_t buffered_pages = 0; // Returns pointer to terminal pointer of list rooted at *pfl. -static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, +static void gc_sweep_page(gc_page_profiler_serializer_t *s, jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered, jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT { char *data = pg->data; @@ -1439,6 +1440,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } size_t old_nfree = pg->nfree; size_t nfree; + gc_page_serializer_init(s, pg); int re_use_page = 1; int keep_as_local_buffer = 0; @@ -1458,6 +1460,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } #endif nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize; + gc_page_profile_write_empty_page(s); goto done; } // For quick sweep, we might be able to skip the page if the page doesn't @@ -1467,6 +1470,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag if (!prev_sweep_full || pg->prev_nold == pg->nold) { freedall = 0; nfree = pg->nfree; + gc_page_profile_write_empty_page(s); goto done; } } @@ -1484,12 +1488,14 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag int bits = v->bits.gc; // if an object is past `lim_newpages` then we can guarantee it's garbage if (!gc_marked(bits) || (char*)v >= lim_newpages) { + gc_page_profile_write_garbage(s); *pfl = v; pfl = &v->next; pfl_begin = (pfl_begin != NULL) ? pfl_begin : pfl; pg_nfree++; } else { // marked young or old + gc_page_profile_write_live_obj(s, v); if (current_sweep_full || bits == GC_MARKED) { // old enough bits = v->bits.gc = GC_OLD; // promote } @@ -1533,6 +1539,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag push_lf_back(&global_page_pool_lazily_freed, pg); } } + gc_page_profile_write_to_file(s); gc_update_page_fragmentation_data(pg); gc_time_count_page(freedall, pg_skpd); jl_ptls_t ptls = gc_all_tls_states[pg->thread_n]; @@ -1541,7 +1548,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag } // the actual sweeping over all allocated pages in a memory pool -STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, +STATIC_INLINE void gc_sweep_pool_page(gc_page_profiler_serializer_t *s, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed, jl_gc_pagemeta_t *pg) JL_NOTSAFEPOINT { int p_n = pg->pool_n; @@ -1549,7 +1556,7 @@ STATIC_INLINE void gc_sweep_pool_page(jl_gc_page_stack_t *allocd, jl_gc_page_sta jl_ptls_t ptls2 = gc_all_tls_states[t_n]; jl_gc_pool_t *p = &ptls2->heap.norm_pools[p_n]; int osize = pg->osize; - gc_sweep_page(p, allocd, lazily_freed, pg, osize); + gc_sweep_page(s, p, allocd, lazily_freed, pg, osize); } // sweep over all memory that is being used and not in a pool @@ -1586,11 +1593,20 @@ void gc_sweep_wake_all(void) uv_mutex_unlock(&gc_threads_lock); } +void gc_sweep_wait_for_all(void) +{ + jl_atomic_store(&gc_allocd_scratch, NULL); + while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { + jl_cpu_pause(); + } +} + void gc_sweep_pool_parallel(void) { jl_atomic_fetch_add(&gc_n_threads_sweeping, 1); jl_gc_page_stack_t *allocd_scratch = jl_atomic_load(&gc_allocd_scratch); if (allocd_scratch != NULL) { + gc_page_profiler_serializer_t serializer = gc_page_serializer_create(); while (1) { int found_pg = 0; for (int t_i = 0; t_i < gc_n_threads; t_i++) { @@ -1603,25 +1619,18 @@ void gc_sweep_pool_parallel(void) if (pg == NULL) { continue; } - gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg); + gc_sweep_pool_page(&serializer, allocd, &ptls2->page_metadata_buffered, pg); found_pg = 1; } if (!found_pg) { break; } } + gc_page_serializer_destroy(&serializer); } jl_atomic_fetch_add(&gc_n_threads_sweeping, -1); } -void gc_sweep_wait_for_all(void) -{ - jl_atomic_store(&gc_allocd_scratch, NULL); - while (jl_atomic_load_relaxed(&gc_n_threads_sweeping) != 0) { - jl_cpu_pause(); - } -} - void gc_free_pages(void) { while (1) { @@ -3863,6 +3872,7 @@ void jl_gc_init(void) { JL_MUTEX_INIT(&heapsnapshot_lock, "heapsnapshot_lock"); JL_MUTEX_INIT(&finalizers_lock, "finalizers_lock"); + uv_mutex_init(&page_profile_lock); uv_mutex_init(&gc_cache_lock); uv_mutex_init(&gc_perm_lock); uv_mutex_init(&gc_threads_lock); diff --git a/stdlib/Profile/src/Profile.jl b/stdlib/Profile/src/Profile.jl index 206d2957a91e5..2a5736ba92a87 100644 --- a/stdlib/Profile/src/Profile.jl +++ b/stdlib/Profile/src/Profile.jl @@ -16,6 +16,7 @@ public @profile, callers, init, take_heap_snapshot, + take_page_profile, clear_malloc_data, Allocs @@ -1267,6 +1268,19 @@ function take_heap_snapshot(all_one::Bool=false; dir::Union{Nothing,S}=nothing) return take_heap_snapshot(fpath, all_one) end +""" + Profile.take_page_profile(io::IOStream) + Profile.take_page_profile(filepath::String) +""" +function take_page_profile(io::IOStream) + Base.@_lock_ios(io, ccall(:jl_gc_take_page_profile, Cvoid, (Ptr{Cvoid},), io.handle)) +end +function take_page_profile(filepath::String) + open(filepath, "w") do io + take_page_profile(io) + end + return filepath +end include("Allocs.jl") include("precompile.jl")