From d426343418aab6148706860bd1678ac309dc12c0 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 29 Mar 2023 21:05:13 +0100 Subject: [PATCH] Store object age in a bitmap Closes [Feature #19729] Previously 2 bits of the flags on each RVALUE are reserved to store the number of GC cycles that each object has survived. This commit introduces a new bit array on the heap page, called age_bits, to store that information instead. This patch still reserves one of the age bits in the flags (the old FL_PROMOTED0 bit, now renamed FL_PROMOTED). This is set to 0 for young objects and 1 for old objects, and is used as a performance optimisation for the write barrier. Fetching the age_bits from the heap page and doing the required math to calculate if the object was old or not would slow down the write barrier. So we keep this bit synced in the flags for fast access. --- class.c | 1 - gc.c | 134 ++++++++++++------------ include/ruby/internal/core/rtypeddata.h | 5 +- include/ruby/internal/fl_type.h | 51 +++------ proc.c | 2 +- yjit/src/cruby_bindings.inc.rs | 5 +- 6 files changed, 86 insertions(+), 112 deletions(-) diff --git a/class.c b/class.c index de9bb3f822ddb1..e0222d47342571 100644 --- a/class.c +++ b/class.c @@ -232,7 +232,6 @@ class_alloc(VALUE flags, VALUE klass) size_t alloc_size = sizeof(struct RClass) + sizeof(rb_classext_t); flags &= T_MASK; - flags |= FL_PROMOTED1 /* start from age == 2 */; if (RGENGC_WB_PROTECTED_CLASS) flags |= FL_WB_PROTECTED; NEWOBJ_OF(obj, struct RClass, klass, flags, alloc_size, 0); diff --git a/gc.c b/gc.c index 976f72a1ea5b8d..a6c5eacbdd5366 100644 --- a/gc.c +++ b/gc.c @@ -939,6 +939,9 @@ static const bool HEAP_PAGE_ALLOC_USE_MMAP = false; static bool heap_page_alloc_use_mmap; #endif +#define RVALUE_AGE_BIT_COUNT 2 +#define RVALUE_AGE_BIT_MASK (((bits_t)1 << RVALUE_AGE_BIT_COUNT) - 1) + struct heap_page { short slot_size; short total_slots; @@ -968,6 +971,7 @@ struct heap_page { /* If set, the object is not movable */ bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT]; + bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT]; }; /* @@ -1011,6 +1015,35 @@ asan_unlock_freelist(struct heap_page *page) #define GC_SWEEP_PAGES_FREEABLE_PER_STEP 3 +#define RVALUE_AGE_BITMAP_INDEX(n) (NUM_IN_PAGE(n) / (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) +#define RVALUE_AGE_BITMAP_OFFSET(n) ((NUM_IN_PAGE(n) % (BITS_BITLENGTH / RVALUE_AGE_BIT_COUNT)) * RVALUE_AGE_BIT_COUNT) + +#define RVALUE_OLD_AGE 3 + +static int +RVALUE_AGE_GET(VALUE obj) +{ + bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; + return (int)(age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] >> RVALUE_AGE_BITMAP_OFFSET(obj)) & RVALUE_AGE_BIT_MASK; +} + +static void +RVALUE_AGE_SET(VALUE obj, int age) +{ + RUBY_ASSERT(age <= RVALUE_OLD_AGE); + bits_t *age_bits = GET_HEAP_PAGE(obj)->age_bits; + // clear the bits + age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] &= ~(RVALUE_AGE_BIT_MASK << (RVALUE_AGE_BITMAP_OFFSET(obj))); + // shift the correct value in + age_bits[RVALUE_AGE_BITMAP_INDEX(obj)] |= ((bits_t)age << RVALUE_AGE_BITMAP_OFFSET(obj)); + if (age == RVALUE_OLD_AGE) { + RB_FL_SET_RAW(obj, RUBY_FL_PROMOTED); + } + else { + RB_FL_UNSET_RAW(obj, RUBY_FL_PROMOTED); + } +} + /* Aliases */ #define rb_objspace (*rb_objspace_of(GET_VM())) #define rb_objspace_of(vm) ((vm)->objspace) @@ -1485,8 +1518,6 @@ asan_poison_object_restore(VALUE obj, void *ptr) #define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj)) #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj)) -#define RVALUE_OLD_AGE 3 -#define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */ static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj); static int rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj); @@ -1494,12 +1525,6 @@ static int rgengc_remember(rb_objspace_t *objspace, VALUE obj); static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap); static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap); -static inline int -RVALUE_FLAGS_AGE(VALUE flags) -{ - return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT); -} - static int check_rvalue_consistency_force(const VALUE obj, int terminate) { @@ -1539,7 +1564,7 @@ check_rvalue_consistency_force(const VALUE obj, int terminate) const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0; const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0; const int remembered_bit = MARKED_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj) != 0; - const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags); + const int age = RVALUE_AGE_GET((VALUE)obj); if (GET_HEAP_PAGE(obj)->flags.in_tomb) { fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj)); @@ -1682,29 +1707,16 @@ RVALUE_UNCOLLECTIBLE(VALUE obj) return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0; } -static inline int -RVALUE_OLD_P_RAW(VALUE obj) -{ - const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1; - return (RBASIC(obj)->flags & promoted) == promoted; -} - static inline int RVALUE_OLD_P(VALUE obj) { + GC_ASSERT(!RB_SPECIAL_CONST_P(obj)); check_rvalue_consistency(obj); - return RVALUE_OLD_P_RAW(obj); + // Because this will only ever be called on GC controlled objects, + // we can use the faster _RAW function here + return RB_OBJ_PROMOTED_RAW(obj); } -#if RGENGC_CHECK_MODE || GC_DEBUG -static inline int -RVALUE_AGE(VALUE obj) -{ - check_rvalue_consistency(obj); - return RVALUE_FLAGS_AGE(RBASIC(obj)->flags); -} -#endif - static inline void RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj) { @@ -1725,39 +1737,39 @@ RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj) RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj); } -static inline VALUE -RVALUE_FLAGS_AGE_SET(VALUE flags, int age) -{ - flags &= ~(FL_PROMOTED0 | FL_PROMOTED1); - flags |= (age << RVALUE_AGE_SHIFT); - return flags; -} - /* set age to age+1 */ static inline void RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj) { - VALUE flags = RBASIC(obj)->flags; - int age = RVALUE_FLAGS_AGE(flags); + int age = RVALUE_AGE_GET((VALUE)obj); if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) { rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj)); } age++; - RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age); + RVALUE_AGE_SET(obj, age); if (age == RVALUE_OLD_AGE) { RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj); } + check_rvalue_consistency(obj); } static inline void -RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj) +RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj) { - RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0); - CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj); + check_rvalue_consistency(obj); + GC_ASSERT(!RVALUE_OLD_P(obj)); + RVALUE_AGE_SET(obj, RVALUE_OLD_AGE - 1); + check_rvalue_consistency(obj); +} + +static inline void +RVALUE_AGE_RESET(VALUE obj) +{ + RVALUE_AGE_SET(obj, 0); } static inline void @@ -1770,7 +1782,8 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj) CLEAR_IN_BITMAP(GET_HEAP_PAGE(obj)->remembered_bits, obj); } - RVALUE_DEMOTE_RAW(objspace, obj); + CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj); + RVALUE_AGE_RESET(obj); if (RVALUE_MARKED(obj)) { objspace->rgengc.old_objects--; @@ -1779,22 +1792,6 @@ RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj) check_rvalue_consistency(obj); } -static inline void -RVALUE_AGE_RESET_RAW(VALUE obj) -{ - RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0); -} - -static inline void -RVALUE_AGE_RESET(VALUE obj) -{ - check_rvalue_consistency(obj); - GC_ASSERT(!RVALUE_OLD_P(obj)); - - RVALUE_AGE_RESET_RAW(obj); - check_rvalue_consistency(obj); -} - static inline int RVALUE_BLACK_P(VALUE obj) { @@ -1962,6 +1959,8 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj page->freelist = p; asan_lock_freelist(page); + RVALUE_AGE_RESET(obj); + if (RGENGC_CHECK_MODE && /* obj should belong to page */ !(page->start <= (uintptr_t)obj && @@ -2505,6 +2504,11 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, p->as.basic.flags = flags; *((VALUE *)&p->as.basic.klass) = klass; + int t = flags & RUBY_T_MASK; + if (t == T_CLASS || t == T_MODULE || t == T_ICLASS) { + RVALUE_AGE_SET_CANDIDATE(objspace, obj); + } + #if RACTOR_CHECK_MODE rb_ractor_setup_belonging(obj); #endif @@ -2521,12 +2525,6 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, GC_ASSERT(RVALUE_OLD_P(obj) == FALSE); GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE); - if (flags & FL_PROMOTED1) { - if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj)); - } - else { - if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj)); - } if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj)); } RB_VM_LOCK_LEAVE_NO_BARRIER(); @@ -9017,7 +9015,7 @@ rb_copy_wb_protected_attribute(VALUE dest, VALUE obj) if (RVALUE_WB_UNPROTECTED(obj) && !RVALUE_WB_UNPROTECTED(dest)) { if (!RVALUE_OLD_P(dest)) { MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(dest), dest); - RVALUE_AGE_RESET_RAW(dest); + RVALUE_AGE_RESET(dest); } else { RVALUE_DEMOTE(objspace, dest); @@ -9789,6 +9787,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s int marked; int wb_unprotected; int uncollectible; + int age; RVALUE *dest = (RVALUE *)free; RVALUE *src = (RVALUE *)scan; @@ -9804,6 +9803,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s wb_unprotected = RVALUE_WB_UNPROTECTED((VALUE)src); uncollectible = RVALUE_UNCOLLECTIBLE((VALUE)src); bool remembered = RVALUE_REMEMBERED((VALUE)src); + age = RVALUE_AGE_GET((VALUE)src); /* Clear bits for eventual T_MOVED */ CLEAR_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)src), (VALUE)src); @@ -9846,6 +9846,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s } memset(src, 0, src_slot_size); + RVALUE_AGE_RESET((VALUE)src); /* Set bits for object in new location */ if (remembered) { @@ -9876,6 +9877,7 @@ gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, s CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS((VALUE)dest), (VALUE)dest); } + RVALUE_AGE_SET((VALUE)dest, age); /* Assign forwarding address */ src->as.moved.flags = T_MOVED; src->as.moved.dummy = Qundef; @@ -13439,7 +13441,7 @@ rb_raw_obj_info_common(char *const buff, const size_t buff_size, const VALUE obj } } else { - const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags); + const int age = RVALUE_AGE_GET(obj); if (is_pointer_to_heap(&rb_objspace, (void *)obj)) { APPEND_F("%p [%d%s%s%s%s%s%s] %s ", @@ -13776,7 +13778,7 @@ rb_gcdebug_print_obj_condition(VALUE obj) fprintf(stderr, "marked? : %s\n", MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj) ? "true" : "false"); fprintf(stderr, "pinned? : %s\n", MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), obj) ? "true" : "false"); - fprintf(stderr, "age? : %d\n", RVALUE_AGE(obj)); + fprintf(stderr, "age? : %d\n", RVALUE_AGE_GET(obj)); fprintf(stderr, "old? : %s\n", RVALUE_OLD_P(obj) ? "true" : "false"); fprintf(stderr, "WB-protected?: %s\n", RVALUE_WB_UNPROTECTED(obj) ? "false" : "true"); fprintf(stderr, "remembered? : %s\n", RVALUE_REMEMBERED(obj) ? "true" : "false"); diff --git a/include/ruby/internal/core/rtypeddata.h b/include/ruby/internal/core/rtypeddata.h index 1abf9653055fe9..c7904746fd0115 100644 --- a/include/ruby/internal/core/rtypeddata.h +++ b/include/ruby/internal/core/rtypeddata.h @@ -173,10 +173,9 @@ rbimpl_typeddata_flags { RUBY_TYPED_WB_PROTECTED = RUBY_FL_WB_PROTECTED, /* THIS FLAG DEPENDS ON Ruby version */ /** - * This flag is mysterious. It seems nobody is currently using it. The - * intention of this flag is also unclear. We need further investigations. + * This flag no longer in use */ - RUBY_TYPED_PROMOTED1 = RUBY_FL_PROMOTED1, /* THIS FLAG DEPENDS ON Ruby version */ + RUBY_TYPED_UNUSED = RUBY_FL_UNUSED6, /** * This flag determines whether marking and compaction should be carried out diff --git a/include/ruby/internal/fl_type.h b/include/ruby/internal/fl_type.h index d59c258a3f877f..f7f5abdd9b1e35 100644 --- a/include/ruby/internal/fl_type.h +++ b/include/ruby/internal/fl_type.h @@ -57,8 +57,7 @@ #define FL_SINGLETON RBIMPL_CAST((VALUE)RUBY_FL_SINGLETON) /**< @old{RUBY_FL_SINGLETON} */ #define FL_WB_PROTECTED RBIMPL_CAST((VALUE)RUBY_FL_WB_PROTECTED) /**< @old{RUBY_FL_WB_PROTECTED} */ -#define FL_PROMOTED0 RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED0) /**< @old{RUBY_FL_PROMOTED0} */ -#define FL_PROMOTED1 RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED1) /**< @old{RUBY_FL_PROMOTED1} */ +#define FL_PROMOTED RBIMPL_CAST((VALUE)RUBY_FL_PROMOTED) /**< @old{RUBY_FL_PROMOTED} */ #define FL_FINALIZE RBIMPL_CAST((VALUE)RUBY_FL_FINALIZE) /**< @old{RUBY_FL_FINALIZE} */ #define FL_TAINT RBIMPL_CAST((VALUE)RUBY_FL_TAINT) /**< @old{RUBY_FL_TAINT} */ #define FL_SHAREABLE RBIMPL_CAST((VALUE)RUBY_FL_SHAREABLE) /**< @old{RUBY_FL_SHAREABLE} */ @@ -200,12 +199,15 @@ ruby_fl_type { RUBY_FL_WB_PROTECTED = (1<<5), /** - * This flag has something to do with our garbage collector. These days - * ruby objects are "generational". There are those who are young and - * those who are old. Young objects are prone to die; monitored relatively - * extensively by the garbage collector. OTOH old objects tend to live - * longer. They are relatively rarely considered. This flag is set when a - * object experienced promotion i.e. survived a garbage collection. + * Ruby objects are "generational". There are young objects & old objects. + * Young objects are prone to die & monitored relatively extensively by the + * garbage collector. Old objects tend to live longer & are monitored less + * frequently. When an object survives a GC, its age is incremented. When + * age is equal to RVALUE_OLD_AGE, the object becomes Old. This flag is set + * when an object becomes old, and is used by the write barrier to check if + * an old object should be considered for marking more frequently - as old + * objects that have references added between major GCs need to be remarked + * to prevent the referred object being mistakenly swept. * * @internal * @@ -213,41 +215,14 @@ ruby_fl_type { * 3rd parties. It must be an implementation detail that they should never * know. Might better be hidden. */ - RUBY_FL_PROMOTED0 = (1<<5), + RUBY_FL_PROMOTED = (1<<5), /** - * This flag has something to do with our garbage collector. These days - * ruby objects are "generational". There are those who are young and - * those who are old. Young objects are prone to die; monitored relatively - * extensively by the garbage collector. OTOH old objects tend to live - * longer. They are relatively rarely considered. This flag is set when a - * object experienced two promotions i.e. survived garbage collections - * twice. + * This flag is no longer in use * * @internal - * - * But honestly, @shyouhei doesn't think this flag should be visible from - * 3rd parties. It must be an implementation detail that they should never - * know. Might better be hidden. - */ - RUBY_FL_PROMOTED1 = (1<<6), - - /** - * This flag has something to do with our garbage collector. These days - * ruby objects are "generational". There are those who are young and - * those who are old. Young objects are prone to die; monitored relatively - * extensively by the garbage collector. OTOH old objects tend to live - * longer. They are relatively rarely considered. This flag is set when a - * object experienced promotions i.e. survived more than one garbage - * collections. - * - * @internal - * - * But honestly, @shyouhei doesn't think this flag should be visible from - * 3rd parties. It must be an implementation detail that they should never - * know. Might better be hidden. */ - RUBY_FL_PROMOTED = RUBY_FL_PROMOTED0 | RUBY_FL_PROMOTED1, + RUBY_FL_UNUSED6 = (1<<6), /** * This flag has something to do with finalisers. A ruby object can have diff --git a/proc.c b/proc.c index 17c837d1647baf..08474506ac4df0 100644 --- a/proc.c +++ b/proc.c @@ -70,7 +70,7 @@ CLONESETUP(VALUE clone, VALUE obj) RBIMPL_ASSERT_OR_ASSUME(! RB_SPECIAL_CONST_P(obj)); RBIMPL_ASSERT_OR_ASSUME(! RB_SPECIAL_CONST_P(clone)); - const VALUE flags = RUBY_FL_PROMOTED0 | RUBY_FL_PROMOTED1 | RUBY_FL_FINALIZE; + const VALUE flags = RUBY_FL_PROMOTED | RUBY_FL_FINALIZE; rb_obj_setup(clone, rb_singleton_class_clone(obj), RB_FL_TEST_RAW(obj, ~flags)); rb_singleton_class_attached(RBASIC_CLASS(clone), clone); diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 553f9cdbead7f5..bbf1578a45efe6 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -192,9 +192,8 @@ pub type ruby_value_type = u32; pub const RUBY_FL_USHIFT: ruby_fl_ushift = 12; pub type ruby_fl_ushift = u32; pub const RUBY_FL_WB_PROTECTED: ruby_fl_type = 32; -pub const RUBY_FL_PROMOTED0: ruby_fl_type = 32; -pub const RUBY_FL_PROMOTED1: ruby_fl_type = 64; -pub const RUBY_FL_PROMOTED: ruby_fl_type = 96; +pub const RUBY_FL_PROMOTED: ruby_fl_type = 32; +pub const RUBY_FL_UNUSED6: ruby_fl_type = 64; pub const RUBY_FL_FINALIZE: ruby_fl_type = 128; pub const RUBY_FL_TAINT: ruby_fl_type = 0; pub const RUBY_FL_SHAREABLE: ruby_fl_type = 256;