GC: Fix survival rates for LOH/POH

src/coreclr/gc/gc.cpp

@@ -2657,13 +2657,12 @@ size_t      gc_heap::end_loh_size = 0;
 size_t      gc_heap::bgc_begin_poh_size = 0;
 size_t      gc_heap::end_poh_size = 0;
 
-#ifdef BGC_SERVO_TUNING
 uint64_t    gc_heap::loh_a_no_bgc = 0;
-
 uint64_t    gc_heap::loh_a_bgc_marking = 0;
-
 uint64_t    gc_heap::loh_a_bgc_planning = 0;
-
+uint64_t    gc_heap::poh_a_bgc_marking = 0;
+uint64_t    gc_heap::poh_a_bgc_planning = 0;
+#ifdef BGC_SERVO_TUNING
 size_t      gc_heap::bgc_maxgen_end_fl_size = 0;
 #endif //BGC_SERVO_TUNING
 
@@ -2794,9 +2793,9 @@ FinalizerWorkItem* gc_heap::finalizer_work;
 BOOL gc_heap::proceed_with_gc_p = FALSE;
 GCSpinLock gc_heap::gc_lock;
 
-#ifdef BGC_SERVO_TUNING
+#ifdef BACKGROUND_GC
 uint64_t gc_heap::total_loh_a_last_bgc = 0;
-#endif //BGC_SERVO_TUNING
+#endif //BACKGROUND_GC
 
 #ifdef USE_REGIONS
 region_free_list gc_heap::global_regions_to_decommit[count_free_region_kinds];
@@ -15039,10 +15038,12 @@ gc_heap::init_gc_heap (int h_number)
     make_mark_stack(arr);
 
 #ifdef BACKGROUND_GC
-#ifdef BGC_SERVO_TUNING
     loh_a_no_bgc = 0;
     loh_a_bgc_marking = 0;
     loh_a_bgc_planning = 0;
+    poh_a_bgc_marking = 0;
+    poh_a_bgc_planning = 0;
+#ifdef BGC_SERVO_TUNING
     bgc_maxgen_end_fl_size = 0;
 #endif //BGC_SERVO_TUNING
     freeable_soh_segment = 0;
@@ -18424,6 +18425,43 @@ bool gc_heap::should_retry_other_heap (int gen_number, size_t size)
     }
 }
 
+void gc_heap::bgc_record_uoh_allocation(int gen_number, size_t size)
+{
+    assert(gen_number == loh_generation || gen_number == poh_generation);
+
+    if (gc_heap::background_running_p())
+    {
+        background_uoh_alloc_count++;
+
+        if (current_c_gc_state == c_gc_state_planning)
+        {
+            if (gen_number == loh_generation)
+            {
+                loh_a_bgc_planning += size;
+            }
+            else
+            {
+                poh_a_bgc_planning += size;
+            }
+        }
+        else
+        {
+            if (gen_number == loh_generation)
+            {
+                loh_a_bgc_marking += size;
+            }
+            else
+            {
+                poh_a_bgc_marking += size;
+            }
+        }
+    }
+    else
+    {
+        loh_a_no_bgc += size;
+    }
+}
+
 allocation_state gc_heap::allocate_uoh (int gen_number,
                                           size_t size,
                                           alloc_context* acontext,
@@ -18446,26 +18484,12 @@ allocation_state gc_heap::allocate_uoh (int gen_number,
 #endif //RECORD_LOH_STATE
 
 #ifdef BACKGROUND_GC
+    bgc_record_uoh_allocation(gen_number, size);
+
     if (gc_heap::background_running_p())
     {
-#ifdef BGC_SERVO_TUNING
-        bool planning_p = (current_c_gc_state == c_gc_state_planning);
-#endif //BGC_SERVO_TUNING
-
-        background_uoh_alloc_count++;
         //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
         {
-#ifdef BGC_SERVO_TUNING
-            if (planning_p)
-            {
-                loh_a_bgc_planning += size;
-            }
-            else
-            {
-                loh_a_bgc_marking += size;
-            }
-#endif //BGC_SERVO_TUNING
-
             int spin_for_allocation = (gen_number == loh_generation) ?
                 bgc_loh_allocate_spin() :
                 bgc_poh_allocate_spin();
@@ -18491,12 +18515,6 @@ allocation_state gc_heap::allocate_uoh (int gen_number,
             }
         }
     }
-#ifdef BGC_SERVO_TUNING
-    else
-    {
-        loh_a_no_bgc += size;
-    }
-#endif //BGC_SERVO_TUNING
 #endif //BACKGROUND_GC
 
     gc_reason gr = reason_oos_loh;
@@ -45809,9 +45827,6 @@ void gc_heap::background_sweep()
     concurrent_print_time_delta ("Sw");
     dprintf (2, ("---- (GC%zu)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
 
-    //block concurrent allocation for large objects
-    dprintf (3, ("lh state: planning"));
-
     for (int i = 0; i <= max_generation; i++)
     {
         generation* gen_to_reset = generation_of (i);
@@ -45860,6 +45875,9 @@ void gc_heap::background_sweep()
     sweep_ro_segments();
 #endif //FEATURE_BASICFREEZE
 
+    dprintf (3, ("lh state: planning"));
+
+    // Multiple threads may reach here.  This conditional partially avoids multiple volatile writes.
     if (current_c_gc_state != c_gc_state_planning)
     {
         current_c_gc_state = c_gc_state_planning;
@@ -45890,9 +45908,7 @@ void gc_heap::background_sweep()
 
     if (heap_number == 0)
     {
-#ifdef BGC_SERVO_TUNING
         get_and_reset_loh_alloc_info();
-#endif //BGC_SERVO_TUNING
         uint64_t suspended_end_ts = GetHighPrecisionTimeStamp();
         last_bgc_info[last_bgc_info_index].pause_durations[1] = (size_t)(suspended_end_ts - suspended_start_time);
         total_suspended_time += last_bgc_info[last_bgc_info_index].pause_durations[1];
@@ -46221,6 +46237,7 @@ void gc_heap::background_sweep()
             concurrent_print_time_delta ("Swe SOH");
             FIRE_EVENT(BGC1stSweepEnd, 0);
 
+            //block concurrent allocation for large objects
             enter_spin_lock (&more_space_lock_uoh);
             add_saved_spinlock_info (true, me_acquire, mt_bgc_uoh_sweep, msl_entered);
 
@@ -46276,6 +46293,14 @@ void gc_heap::background_sweep()
     // be accurate.
     compute_new_dynamic_data (max_generation);
 
+    // We also need to adjust size_before for UOH allocations that occurred during sweeping.
+    gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
+    assert(loh_a_bgc_marking == 0);
+    assert(poh_a_bgc_marking == 0);
+    assert(loh_a_no_bgc == 0);
+    current_gc_data_per_heap->gen_data[loh_generation].size_before += loh_a_bgc_planning;
+    current_gc_data_per_heap->gen_data[poh_generation].size_before += poh_a_bgc_planning;
+
 #ifdef DOUBLY_LINKED_FL
     current_bgc_state = bgc_not_in_process;
 
@@ -50238,12 +50263,10 @@ void gc_heap::check_and_adjust_bgc_tuning (int gen_number, size_t physical_size,
         }
     }
 }
+#endif //BGC_SERVO_TUNING
 
 void gc_heap::get_and_reset_loh_alloc_info()
 {
-    if (!bgc_tuning::enable_fl_tuning)
-        return;
-
     total_loh_a_last_bgc = 0;
 
     uint64_t total_loh_a_no_bgc = 0;
@@ -50257,12 +50280,22 @@ void gc_heap::get_and_reset_loh_alloc_info()
     {
         gc_heap* hp = pGenGCHeap;
 #endif //MULTIPLE_HEAPS
+
+        // We need to adjust size_before for UOH allocations that occurred during marking
+        // before we lose the values here.
+        gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
+        // loh/poh_a_bgc_planning should be the same as they were when init_records set size_before.
+        current_gc_data_per_heap->gen_data[loh_generation].size_before += hp->loh_a_bgc_marking;
+        current_gc_data_per_heap->gen_data[poh_generation].size_before += hp->poh_a_bgc_marking;
+
         total_loh_a_no_bgc += hp->loh_a_no_bgc;
         hp->loh_a_no_bgc = 0;
-        total_loh_a_bgc_marking += hp->loh_a_bgc_marking;
+        total_loh_a_bgc_marking += hp->loh_a_bgc_marking + hp->poh_a_bgc_marking;
         hp->loh_a_bgc_marking = 0;
-        total_loh_a_bgc_planning += hp->loh_a_bgc_planning;
+        hp->poh_a_bgc_marking = 0;
+        total_loh_a_bgc_planning += hp->loh_a_bgc_planning + hp->poh_a_bgc_planning;
         hp->loh_a_bgc_planning = 0;
+        hp->poh_a_bgc_planning = 0;
     }
     dprintf (2, ("LOH alloc: outside bgc: %zd; bm: %zd; bp: %zd",
         total_loh_a_no_bgc,
@@ -50271,7 +50304,6 @@ void gc_heap::get_and_reset_loh_alloc_info()
 
     total_loh_a_last_bgc = total_loh_a_no_bgc + total_loh_a_bgc_marking + total_loh_a_bgc_planning;
 }
-#endif //BGC_SERVO_TUNING
 
 bool gc_heap::is_pm_ratio_exceeded()
 {

src/coreclr/gc/gcpriv.h

@@ -1752,6 +1752,8 @@ class gc_heap
 
     PER_HEAP_ISOLATED_METHOD void add_to_history();
 
+    PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info();
+
 #ifdef BGC_SERVO_TUNING
     // Currently BGC servo tuning is an experimental feature.
     class bgc_tuning
@@ -1997,7 +1999,6 @@ class gc_heap
     };
 
     PER_HEAP_ISOLATED_METHOD void check_and_adjust_bgc_tuning (int gen_number, size_t physical_size, ptrdiff_t virtual_fl_size);
-    PER_HEAP_ISOLATED_METHOD void get_and_reset_loh_alloc_info();
 #endif //BGC_SERVO_TUNING
 
 #ifndef USE_REGIONS
@@ -2230,6 +2231,8 @@ class gc_heap
     PER_HEAP_METHOD BOOL bgc_loh_allocate_spin();
 
     PER_HEAP_METHOD BOOL bgc_poh_allocate_spin();
+
+    PER_HEAP_METHOD void bgc_record_uoh_allocation(int gen_number, size_t size);
 #endif //BACKGROUND_GC
 
     PER_HEAP_METHOD void add_saved_spinlock_info (
@@ -3436,6 +3439,13 @@ class gc_heap
 
     PER_HEAP_FIELD_SINGLE_GC uint8_t* next_sweep_obj;
     PER_HEAP_FIELD_SINGLE_GC uint8_t* current_sweep_pos;
+
+    PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_no_bgc;
+    PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_marking;
+    PER_HEAP_FIELD_SINGLE_GC uint64_t loh_a_bgc_planning;
+    PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_marking;
+    PER_HEAP_FIELD_SINGLE_GC uint64_t poh_a_bgc_planning;
+
 #ifdef DOUBLY_LINKED_FL
     PER_HEAP_FIELD_SINGLE_GC heap_segment* current_sweep_seg;
 #endif //DOUBLY_LINKED_FL
@@ -3461,9 +3471,6 @@ class gc_heap
 #endif //SNOOP_STATS
 
 #ifdef BGC_SERVO_TUNING
-    PER_HEAP_FIELD_SINGLE_GC uint64_t   loh_a_no_bgc;
-    PER_HEAP_FIELD_SINGLE_GC uint64_t   loh_a_bgc_marking;
-    PER_HEAP_FIELD_SINGLE_GC uint64_t   loh_a_bgc_planning;
     PER_HEAP_FIELD_SINGLE_GC size_t     bgc_maxgen_end_fl_size;
 #endif //BGC_SERVO_TUNING
 #endif //BACKGROUND_GC
@@ -4097,11 +4104,9 @@ class gc_heap
 
     PER_HEAP_ISOLATED_FIELD_SINGLE_GC GCEvent bgc_start_event;
 
-#ifdef BGC_SERVO_TUNING
     // Total allocated last BGC's plan + between last and this bgc +
     // this bgc's mark
     PER_HEAP_ISOLATED_FIELD_SINGLE_GC uint64_t   total_loh_a_last_bgc;
-#endif //BGC_SERVO_TUNING
 #endif //BACKGROUND_GC
 
 #ifdef USE_REGIONS