diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 5c5903050fdca..68a783e9b6ccd 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -823,6 +823,11 @@ class t_join join_struct.r_join_lock = n_th; } + int get_num_threads() + { + return join_struct.n_threads; + } + void destroy () { dprintf (JOIN_LOG, ("Destroying join structure")); @@ -887,6 +892,8 @@ class t_join // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent() if (color == join_struct.lock_color.LoadWithoutBarrier()) { + dprintf (9999, ("---h%d %d j%d %d - respin!!! (c:%d-%d)", + gch->heap_number, join_id, join_struct.n_threads, color, join_struct.lock_color.LoadWithoutBarrier())); goto respin; } @@ -1117,6 +1124,25 @@ t_join bgc_t_join; } \ } +#define spin_and_wait(count_to_spin, expr) \ +{ \ + while (!expr) \ + { \ + for (int j = 0; j < count_to_spin; j++) \ + { \ + if (expr) \ + { \ + break; \ + } \ + YieldProcessor (); \ + } \ + if (!(expr)) \ + { \ + GCToOSInterface::YieldThread (0); \ + } \ + } \ +} + #ifdef BACKGROUND_GC #define max_pending_allocs 64 @@ -2318,9 +2344,6 @@ sorted_table* gc_heap::seg_table; #ifdef MULTIPLE_HEAPS GCEvent gc_heap::ee_suspend_event; -#ifdef DYNAMIC_HEAP_COUNT -GCEvent gc_heap::gc_idle_thread_event; -#endif //DYNAMIC_HEAP_COUNT size_t gc_heap::min_gen0_balance_delta = 0; size_t gc_heap::min_balance_threshold = 0; #endif //MULTIPLE_HEAPS @@ -2328,6 +2351,9 @@ size_t gc_heap::min_balance_threshold = 0; VOLATILE(BOOL) gc_heap::gc_started; #ifdef MULTIPLE_HEAPS +#ifdef STRESS_DYNAMIC_HEAP_COUNT +int gc_heap::heaps_in_this_gc = 0; +#endif //STRESS_DYNAMIC_HEAP_COUNT GCEvent gc_heap::gc_start_event; bool gc_heap::gc_thread_no_affinitize_p = false; uintptr_t process_mask = 0; @@ -6967,12 +6993,6 @@ BOOL gc_heap::create_thread_support (int number_of_heaps) { goto cleanup; } -#ifdef DYNAMIC_HEAP_COUNT - if (!gc_idle_thread_event.CreateOSManualEventNoThrow (FALSE)) - { - goto cleanup; - } -#endif //DYNAMIC_HEAP_COUNT if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE)) { goto cleanup; @@ -7051,9 +7071,30 @@ void gc_heap::gc_thread_function () continue; } +#ifdef DYNAMIC_HEAP_COUNT + // wait till the threads that should have gone idle at least reached the place where they are about to wait on the idle event. + if ((gc_heap::dynamic_adaptation_mode == dynamic_adaptation_to_application_sizes) && + (n_heaps != dynamic_heap_count_data.last_n_heaps)) + { + int spin_count = 1024; + int idle_thread_count = n_max_heaps - n_heaps; + dprintf (9999, ("heap count changed %d->%d, idle should be %d and is %d", dynamic_heap_count_data.last_n_heaps, n_heaps, + idle_thread_count, VolatileLoadWithoutBarrier (&dynamic_heap_count_data.idle_thread_count))); + if (idle_thread_count != dynamic_heap_count_data.idle_thread_count) + { + spin_and_wait (spin_count, (idle_thread_count == dynamic_heap_count_data.idle_thread_count)); + dprintf (9999, ("heap count changed %d->%d, now idle is %d", dynamic_heap_count_data.last_n_heaps, n_heaps, + VolatileLoadWithoutBarrier (&dynamic_heap_count_data.idle_thread_count))); + } + + dynamic_heap_count_data.last_n_heaps = n_heaps; + } +#endif //DYNAMIC_HEAP_COUNT suspended_start_time = GetHighPrecisionTimeStamp(); BEGIN_TIMING(suspend_ee_during_log); + dprintf (9999, ("h0 suspending EE in GC!")); GCToEEInterface::SuspendEE(SUSPEND_FOR_GC); + dprintf (9999, ("h0 suspended EE in GC!")); END_TIMING(suspend_ee_during_log); proceed_with_gc_p = TRUE; @@ -7067,46 +7108,74 @@ void gc_heap::gc_thread_function () { settings.init_mechanisms(); #ifdef DYNAMIC_HEAP_COUNT - // make sure the other gc threads cannot see this as a request to change heap count - // see explanation below about the cases when we return from gc_start_event.Wait - assert (dynamic_heap_count_data.new_n_heaps == n_heaps); + if (gc_heap::dynamic_adaptation_mode == dynamic_adaptation_to_application_sizes) + { + // make sure the other gc threads cannot see this as a request to change heap count + // see explanation below about the cases when we return from gc_start_event.Wait + assert (dynamic_heap_count_data.new_n_heaps == n_heaps); + } #endif //DYNAMIC_HEAP_COUNT + dprintf (9999, ("GC thread %d setting_gc_start_in_gc(h%d)", heap_number, n_heaps)); gc_start_event.Set(); } dprintf (3, (ThreadStressLog::gcServerThread0StartMsg(), heap_number)); } else { + dprintf (9999, ("GC thread %d waiting_for_gc_start(%d)(gc%Id)", heap_number, n_heaps, VolatileLoadWithoutBarrier(&settings.gc_index))); gc_start_event.Wait(INFINITE, FALSE); #ifdef DYNAMIC_HEAP_COUNT - // we have a couple different cases to handle here when we come back from the wait: - // 1. We are starting a GC. Signaled by dynamic_heap_count_data.new_n_heaps == n_heaps - // a) We are starting a GC, but this thread is idle. Signaled by n_heaps <= heap_number - // b) We are starting a GC, and this thread is participating. Signaled by heap_number < n_heaps - // 2. We are changing heap count. Signaled by dynamic_heap_count_data.new_n_heaps != n_heaps - // a) We are changing heap count, but this thread is idle. Signaled by n_heaps <= heap_number. - // b) We are changing heap count, and this thread is participating. Signaled by heap_number < n_heaps. - - // check for 1.a) and 2.a) cases above - if (n_heaps <= heap_number) - { - dprintf (2, ("GC thread %d idle", heap_number)); - - // make sure GC is complete so we know the gc_idle_thread_event has been reset - g_theGCHeap->WaitUntilGCComplete(); + dprintf (9999, ("GC thread %d waiting_done_gc_start(%d-%d)(i: %d)(gc%Id)", + heap_number, n_heaps, dynamic_heap_count_data.new_n_heaps, dynamic_heap_count_data.init_only_p, VolatileLoadWithoutBarrier (&settings.gc_index))); + + if ((gc_heap::dynamic_adaptation_mode == dynamic_adaptation_to_application_sizes) && + (dynamic_heap_count_data.new_n_heaps != n_heaps)) + { + // The reason why we need to do this is - + // + for threads that were participating, we need them to do work for change_heap_count + // + for threads that were not participating but will need to participate, we need to make sure they are woken now instead of + // randomly sometime later. + int old_n_heaps = n_heaps; + int new_n_heaps = dynamic_heap_count_data.new_n_heaps; + int num_threads_to_wake = max (new_n_heaps, old_n_heaps); + if (heap_number < num_threads_to_wake) + { + dprintf (9999, ("h%d < %d, calling change", heap_number, num_threads_to_wake)); + change_heap_count (dynamic_heap_count_data.new_n_heaps); + if (new_n_heaps < old_n_heaps) + { + dprintf (9999, ("h%d after change", heap_number)); + // at the end of change_heap_count we've changed join's heap count to the new one if it's smaller. So we need to make sure + // only that many threads will participate in the following GCs. + if (heap_number < new_n_heaps) + { + dprintf (9999, ("h%d < %d participating (dec)", heap_number, new_n_heaps)); + } + else + { + Interlocked::Increment (&dynamic_heap_count_data.idle_thread_count); + dprintf (9999, ("GC thread %d wait_on_idle(%d < %d)(gc%Id), total idle %d", heap_number, old_n_heaps, new_n_heaps, + VolatileLoadWithoutBarrier (&settings.gc_index), VolatileLoadWithoutBarrier (&dynamic_heap_count_data.idle_thread_count))); + gc_idle_thread_event.Wait (INFINITE, FALSE); + dprintf (9999, ("GC thread %d waking_from_idle(%d)(gc%Id) after doing change", heap_number, n_heaps, VolatileLoadWithoutBarrier (&settings.gc_index))); + } + } + else + { + dprintf (9999, ("h%d < %d participating (inc)", heap_number, new_n_heaps)); + } + } + else + { + Interlocked::Increment (&dynamic_heap_count_data.idle_thread_count); + dprintf (9999, ("GC thread %d wait_on_idle(< max %d)(gc%Id), total idle %d", heap_number, num_threads_to_wake, + VolatileLoadWithoutBarrier (&settings.gc_index), VolatileLoadWithoutBarrier (&dynamic_heap_count_data.idle_thread_count))); + gc_idle_thread_event.Wait (INFINITE, FALSE); + dprintf (9999, ("GC thread %d waking_from_idle(%d)(gc%Id)", heap_number, n_heaps, VolatileLoadWithoutBarrier (&settings.gc_index))); + } - // now wait on the gc_idle_thread_event - gc_idle_thread_event.Wait(INFINITE, FALSE); - dprintf (2, ("GC thread %d waking from idle", heap_number)); - continue; - } - // case 2.b) above: is this a request to change heap count? - if (dynamic_heap_count_data.new_n_heaps != n_heaps) - { - change_heap_count (dynamic_heap_count_data.new_n_heaps); continue; } - // case 1.b) above: we're starting a GC. #endif //DYNAMIC_HEAP_COUNT dprintf (3, (ThreadStressLog::gcServerThreadNStartMsg(), heap_number)); } @@ -12513,6 +12582,16 @@ void gc_heap::rearrange_uoh_segments() freeable_uoh_segment = 0; } +void gc_heap::delay_free_segments() +{ + rearrange_uoh_segments(); +#ifdef BACKGROUND_GC + background_delay_delete_uoh_segments(); + if (!gc_heap::background_running_p()) + rearrange_small_heap_segments(); +#endif //BACKGROUND_GC +} + #ifndef USE_REGIONS void gc_heap::rearrange_heap_segments(BOOL compacting) { @@ -14846,6 +14925,25 @@ gc_heap::init_gc_heap (int h_number) gc_done_event_lock = -1; gc_done_event_set = false; +#ifdef DYNAMIC_HEAP_COUNT + if (h_number != 0) + { + if (!gc_idle_thread_event.CreateAutoEventNoThrow (FALSE)) + { + return 0; + } + +#ifdef BACKGROUND_GC + if (!bgc_idle_thread_event.CreateAutoEventNoThrow (FALSE)) + { + return 0; + } +#endif //BACKGROUND_GC + + dprintf (9999, ("creating idle events for h%d", h_number)); + } +#endif //DYNAMIC_HEAP_COUNT + if (!init_dynamic_data()) { return 0; @@ -16024,7 +16122,6 @@ void min_fl_list_info::thread_item_no_prev (uint8_t* item) tail = item; } -// This is only implemented for gen2 right now!!!! // the min_fl_list array is arranged as chunks of n_heaps min_fl_list_info, the 1st chunk corresponds to the 1st bucket, // and so on. void allocator::rethread_items (size_t* num_total_fl_items, size_t* num_total_fl_items_rethreaded, gc_heap* current_heap, @@ -17392,6 +17489,7 @@ BOOL gc_heap::a_fit_free_list_uoh_p (size_t size, gen_number, align_const); dd_new_allocation (dynamic_data_of (gen_number)) -= limit; + size_t saved_free_list_size = free_list_size; #ifdef FEATURE_LOH_COMPACTION if (loh_pad) { @@ -17420,7 +17518,7 @@ BOOL gc_heap::a_fit_free_list_uoh_p (size_t size, { generation_free_obj_space (gen) += remain_size; } - generation_free_list_space (gen) -= free_list_size; + generation_free_list_space (gen) -= saved_free_list_size; assert ((ptrdiff_t)generation_free_list_space (gen) >= 0); generation_free_list_allocated (gen) += limit; @@ -22744,7 +22842,12 @@ void gc_heap::merge_fl_from_other_heaps (int gen_idx, int to_n_heaps, int from_n assert (free_list_space_decrease <= generation_free_list_space (gen)); generation_free_list_space (gen) -= free_list_space_decrease; - assert (free_list_space_decrease <= dd_fragmentation (dd)); + // TODO - I'm seeing for gen2 this is free_list_space_decrease can be a bit larger than frag. + // Need to fix this later. + if (gen_idx != max_generation) + { + assert (free_list_space_decrease <= dd_fragmentation (dd)); + } size_t free_list_space_increase = 0; for (int from_hn = 0; from_hn < from_n_heaps; from_hn++) @@ -23719,9 +23822,6 @@ void gc_heap::garbage_collect (int n) #ifdef MULTIPLE_HEAPS gc_start_event.Reset(); -#ifdef DYNAMIC_HEAP_COUNT - gc_idle_thread_event.Reset(); -#endif //DYNAMIC_HEAP_COUNT gc_t_join.restart(); #endif //MULTIPLE_HEAPS } @@ -23743,6 +23843,9 @@ void gc_heap::garbage_collect (int n) #endif // STRESS_HEAP #ifdef MULTIPLE_HEAPS +#ifdef STRESS_DYNAMIC_HEAP_COUNT + Interlocked::Increment (&heaps_in_this_gc); +#endif //STRESS_DYNAMIC_HEAP_COUNT //align all heaps on the max generation to condemn dprintf (3, ("Joining for max generation to condemn")); condemned_generation_num = generation_to_condemn (n, @@ -23758,30 +23861,31 @@ void gc_heap::garbage_collect (int n) #endif //FEATURE_BASICFREEZE #ifdef MULTIPLE_HEAPS +#ifdef STRESS_DYNAMIC_HEAP_COUNT + dprintf (9999, ("%d heaps, join sees %d, actually joined %d, %d idle threads (%d)", + n_heaps, gc_t_join.get_num_threads (), heaps_in_this_gc, + VolatileLoadWithoutBarrier(&dynamic_heap_count_data.idle_thread_count), (n_max_heaps - n_heaps))); + if (heaps_in_this_gc != n_heaps) + { + dprintf (9999, ("should have %d heaps but actually have %d!!", n_heaps, heaps_in_this_gc)); + GCToOSInterface::DebugBreak (); + } + + heaps_in_this_gc = 0; +#endif //STRESS_DYNAMIC_HEAP_COUNT + for (int i = 0; i < n_heaps; i++) { gc_heap* hp = g_heaps[i]; // check for card table growth if (g_gc_card_table != hp->card_table) hp->copy_brick_card_table(); - - hp->rearrange_uoh_segments(); -#ifdef BACKGROUND_GC - hp->background_delay_delete_uoh_segments(); - if (!gc_heap::background_running_p()) - hp->rearrange_small_heap_segments(); -#endif //BACKGROUND_GC + hp->delay_free_segments(); } #else //MULTIPLE_HEAPS if (g_gc_card_table != card_table) copy_brick_card_table(); - - rearrange_uoh_segments(); -#ifdef BACKGROUND_GC - background_delay_delete_uoh_segments(); - if (!gc_heap::background_running_p()) - rearrange_small_heap_segments(); -#endif //BACKGROUND_GC + delay_free_segments(); #endif //MULTIPLE_HEAPS BOOL should_evaluate_elevation = TRUE; @@ -23868,10 +23972,8 @@ void gc_heap::garbage_collect (int n) do_pre_gc(); #ifdef MULTIPLE_HEAPS + dprintf (9999, ("in GC, resetting gc_start")); gc_start_event.Reset(); -#ifdef DYNAMIC_HEAP_COUNT - gc_idle_thread_event.Reset(); -#endif //DYNAMIC_HEAP_COUNT dprintf(3, ("Starting all gc threads for gc")); gc_t_join.restart(); #endif //MULTIPLE_HEAPS @@ -24327,7 +24429,7 @@ void gc_heap::equalize_promoted_bytes(int condemned_gen_number) // hope is to achieve better work balancing in relocate and compact phases // this is also used when the heap count changes to balance regions between heaps int highest_gen_number = ((condemned_gen_number == max_generation) ? - (total_generation_count - 1) : condemned_gen_number); + (total_generation_count - 1) : condemned_gen_number); int stop_gen_idx = get_stop_generation_index (condemned_gen_number); for (int gen_idx = highest_gen_number; gen_idx >= stop_gen_idx; gen_idx--) @@ -25089,7 +25191,7 @@ void gc_heap::check_heap_count () sample.allocating_thread_count = allocating_thread_count; sample.heap_size = heap_size; - dprintf (6666, ("sample %d: soh_msl_wait_time: %zd, uoh_msl_wait_time: %zd, elapsed_between_gcs: %zd, gc_elapsed_time: %d, heap_size: %zd MB", + dprintf (9999, ("sample %d: soh_msl_wait_time: %zd, uoh_msl_wait_time: %zd, elapsed_between_gcs: %zd, gc_elapsed_time: %d, heap_size: %zd MB", dynamic_heap_count_data.sample_index, sample.soh_msl_wait_time, sample.uoh_msl_wait_time, @@ -25106,7 +25208,9 @@ void gc_heap::check_heap_count () sample.elapsed_between_gcs ); - if (settings.gc_index < prev_change_heap_count_gc_index + 3) + dprintf (9999, ("current GC %Id, prev %Id", VolatileLoadWithoutBarrier (&settings.gc_index), prev_change_heap_count_gc_index)); + + if (settings.gc_index < (prev_change_heap_count_gc_index + 3)) { // reconsider the decision every few gcs return; @@ -25116,6 +25220,7 @@ void gc_heap::check_heap_count () { // can't have background gc running while we change the number of heaps // so it's useless to compute a new number of heaps here + dprintf (9999, ("BGC in progress, don't change")); } else { @@ -25130,7 +25235,7 @@ void gc_heap::check_heap_count () percent_overhead[i] = 0; else if (percent_overhead[i] > 100) percent_overhead[i] = 100; - dprintf (6666, ("sample %d: percent_overhead: %d%%", i, (int)percent_overhead[i])); + dprintf (9999, ("sample %d: percent_overhead: %.3f%%", i, percent_overhead[i])); } // compute the median of the percent overhead samples #define compare_and_swap(i, j) \ @@ -25164,7 +25269,7 @@ void gc_heap::check_heap_count () smoothed_median_percent_overhead = median_percent_overhead; } - dprintf (6666, ("median overhead: %d%% smoothed median overhead: %d%%", (int)(median_percent_overhead*1000), (int)(smoothed_median_percent_overhead*1000))); + dprintf (9999, ("median overhead: %.3f%% smoothed median overhead: %.3f%%", median_percent_overhead, smoothed_median_percent_overhead)); // estimate the space cost of adding a heap as the min gen0 size size_t heap_space_cost_per_heap = dd_min_size (hp0_dd0); @@ -25195,6 +25300,9 @@ void gc_heap::check_heap_count () // estimate the potential space saving of going down a step float space_cost_decrease_per_step_down = percent_heap_space_cost_per_heap * step_down; + dprintf (9999, ("up: %d down: %d, ou %.3f, od %.3f, su %.3f, sd %.3f", step_up, step_down, + overhead_reduction_per_step_up, overhead_increase_per_step_down, space_cost_increase_per_step_up, space_cost_decrease_per_step_down)); + #ifdef STRESS_DYNAMIC_HEAP_COUNT // quick hack for initial testing int new_n_heaps = (int)gc_rand::get_rand (n_max_heaps - 1) + 1; @@ -25207,6 +25315,7 @@ void gc_heap::check_heap_count () // but not down to zero, obviously... new_n_heaps = max (new_n_heaps, 1); } + dprintf (9999, ("stress %d -> %d", n_heaps, new_n_heaps)); #else //STRESS_DYNAMIC_HEAP_COUNT int new_n_heaps = n_heaps; if (median_percent_overhead > 10.0f) @@ -25232,7 +25341,7 @@ void gc_heap::check_heap_count () new_n_heaps -= step_down; } - dprintf (6666, ("or: %d, si: %d, sd: %d, oi: %d => %d -> %d", + dprintf (9999, ("or: %d, si: %d, sd: %d, oi: %d => %d -> %d", (int)overhead_reduction_per_step_up, (int)space_cost_increase_per_step_up, (int)space_cost_decrease_per_step_down, @@ -25268,8 +25377,10 @@ void gc_heap::check_heap_count () if (new_n_heaps != n_heaps) { + dprintf (9999, ("h0 suspending EE in check")); // can't have threads allocating while we change the number of heaps GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP); + dprintf (9999, ("h0 suspended EE in check")); if (gc_heap::background_running_p()) { @@ -25281,10 +25392,19 @@ void gc_heap::check_heap_count () } } + if (dynamic_heap_count_data.new_n_heaps != n_heaps) + { + if (!prepare_to_change_heap_count (dynamic_heap_count_data.new_n_heaps)) + { + // we don't have sufficient resources - reset the new heap count + dynamic_heap_count_data.new_n_heaps = n_heaps; + } + } + if (dynamic_heap_count_data.new_n_heaps == n_heaps) { // heap count stays the same, no work to do - dprintf (6666, ("heap count stays the same, no work to do %d == %d", dynamic_heap_count_data.new_n_heaps, n_heaps)); + dprintf (9999, ("heap count stays the same, no work to do %d == %d", dynamic_heap_count_data.new_n_heaps, n_heaps)); // come back after 3 GCs to reconsider prev_change_heap_count_gc_index = settings.gc_index; @@ -25292,10 +25412,35 @@ void gc_heap::check_heap_count () return; } + int new_n_heaps = dynamic_heap_count_data.new_n_heaps; + if (GCScan::GetGcRuntimeStructuresValid()) { + // At this point we are guaranteed to be able to change the heap count to the new one. + // Change the heap count for joins here because we will need to join new_n_heaps threads together. + dprintf (9999, ("changing join hp %d->%d", n_heaps, new_n_heaps)); + int max_threads_to_wake = max (n_heaps, new_n_heaps); + gc_t_join.update_n_threads (max_threads_to_wake); + // make sure the other gc threads cannot see this as a request to GC assert (dynamic_heap_count_data.new_n_heaps != n_heaps); + + if (n_heaps < new_n_heaps) + { + int saved_idle_thread_count = dynamic_heap_count_data.idle_thread_count; + Interlocked::ExchangeAdd (&dynamic_heap_count_data.idle_thread_count, (n_heaps - new_n_heaps)); + dprintf (9999, ("GC thread %d setting idle events for h%d-h%d, total idle %d -> %d", heap_number, n_heaps, (new_n_heaps - 1), + saved_idle_thread_count, VolatileLoadWithoutBarrier (&dynamic_heap_count_data.idle_thread_count))); + + for (int heap_idx = n_heaps; heap_idx < new_n_heaps; heap_idx++) + { + g_heaps[heap_idx]->gc_idle_thread_event.Set(); +#ifdef BACKGROUND_GC + g_heaps[heap_idx]->bgc_idle_thread_event.Set(); +#endif //BACKGROUND_GC + } + } + gc_start_event.Set(); } @@ -25304,17 +25449,17 @@ void gc_heap::check_heap_count () change_heap_count (dynamic_heap_count_data.new_n_heaps); GCToEEInterface::RestartEE(TRUE); + dprintf (9999, ("h0 restarted EE")); prev_change_heap_count_gc_index = settings.gc_index; // we made changes to the heap count that will change the overhead, // so change the smoothed overhead to reflect that - int new_n_heaps = n_heaps; - dynamic_heap_count_data.smoothed_median_percent_overhead = dynamic_heap_count_data.smoothed_median_percent_overhead/new_n_heaps*old_n_heaps; + dynamic_heap_count_data.smoothed_median_percent_overhead = dynamic_heap_count_data.smoothed_median_percent_overhead/n_heaps*old_n_heaps; } bool gc_heap::prepare_to_change_heap_count (int new_n_heaps) { - dprintf (6666, ("trying to change heap count %d -> %d", n_heaps, new_n_heaps)); + dprintf (9999, ("trying to change heap count %d -> %d", n_heaps, new_n_heaps)); // use this variable for clarity - n_heaps will change during the transition int old_n_heaps = n_heaps; @@ -25357,6 +25502,17 @@ bool gc_heap::prepare_to_change_heap_count (int new_n_heaps) } } + // Before we look at whether we have sufficient regions we should return regions that should be deleted to free + // so we don't lose them when we decommission heaps. We could do this for only heaps that we are about + // to decomission. But it's better to do this for all heaps because we don't need to worry about adding them to the + // heaps remain (freeable uoh/soh regions) and we get rid of regions with the heap_segment_flags_uoh_delete flag + // because background_delay_delete_uoh_segments makes the assumption it can't be the start region. + for (int i = 0; i < old_n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + hp->delay_free_segments (); + } + // if we want to increase the number of heaps, we have to make sure we can give // each heap a region for each generation. If we cannot do that, we have to give up ptrdiff_t region_count_in_gen[total_generation_count]; @@ -25437,39 +25593,34 @@ bool gc_heap::prepare_to_change_heap_count (int new_n_heaps) bool gc_heap::change_heap_count (int new_n_heaps) { + dprintf (9999, ("BEG heap%d changing %d->%d", heap_number, n_heaps, new_n_heaps)); + // use this variable for clarity - n_heaps will change during the transition int old_n_heaps = n_heaps; + bool init_only_p = dynamic_heap_count_data.init_only_p; - if (heap_number == 0) { - if (!prepare_to_change_heap_count (new_n_heaps)) - { - // we don't have sufficient resources - reset the new heap count - dynamic_heap_count_data.new_n_heaps = n_heaps; - } - } - - if (GCScan::GetGcRuntimeStructuresValid()) - { - // join for sufficient resources decision gc_t_join.join (this, gc_join_merge_temp_fl); if (gc_t_join.joined ()) { + // BGC is not running, we can safely change its join's heap count. +#ifdef BACKGROUND_GC + bgc_t_join.update_n_threads (new_n_heaps); +#endif //BACKGROUND_GC + + dynamic_heap_count_data.init_only_p = false; + dprintf (9999, ("in change h%d resetting gc_start, update bgc join to %d heaps", heap_number, new_n_heaps)); gc_start_event.Reset(); gc_t_join.restart (); } } - // gc_heap::n_heaps may have changed by now, compare to the snapshot *before* the join - if (dynamic_heap_count_data.new_n_heaps == old_n_heaps) - { - dprintf (6666, ("failed to change heap count, no work to do %d == %d", dynamic_heap_count_data.new_n_heaps, old_n_heaps)); - return false; - } + assert (dynamic_heap_count_data.new_n_heaps != old_n_heaps); + + dprintf (9999, ("Waiting h0 heap%d changing %d->%d", heap_number, n_heaps, new_n_heaps)); if (heap_number == 0) { - // after having checked for sufficient resources, we are now committed to actually change the heap count dprintf (3, ("switching heap count from %d to %d heaps", old_n_heaps, new_n_heaps)); // spread finalization data out to heaps coming into service @@ -25497,7 +25648,7 @@ bool gc_heap::change_heap_count (int new_n_heaps) { gc_heap* hp = g_heaps[i]; - if (GCScan::GetGcRuntimeStructuresValid()) + if (!init_only_p) { hp->fix_allocation_contexts (TRUE); } @@ -25606,7 +25757,7 @@ bool gc_heap::change_heap_count (int new_n_heaps) hpd->free_regions[kind].transfer_regions(&hp->free_regions[kind]); } } - // update number of heaps + dprintf (9999, ("h%d changing %d->%d", heap_number, n_heaps, new_n_heaps)); n_heaps = new_n_heaps; // even out the regions over the current number of heaps @@ -25647,7 +25798,9 @@ bool gc_heap::change_heap_count (int new_n_heaps) } } - if (GCScan::GetGcRuntimeStructuresValid()) + dprintf (3, ("individual heap%d changing %d->%d", heap_number, n_heaps, new_n_heaps)); + + if (!init_only_p) { // join for rethreading the free lists gc_t_join.join (this, gc_join_merge_temp_fl); @@ -25659,7 +25812,11 @@ bool gc_heap::change_heap_count (int new_n_heaps) // rethread the free lists for (int gen_idx = 0; gen_idx < total_generation_count; gen_idx++) { - rethread_fl_items (gen_idx); + if (heap_number < old_n_heaps) + { + dprintf (3, ("h%d calling per heap work!", heap_number)); + rethread_fl_items (gen_idx); + } // join for merging the free lists gc_t_join.join (this, gc_join_merge_temp_fl); @@ -25676,12 +25833,6 @@ bool gc_heap::change_heap_count (int new_n_heaps) if (heap_number == 0) { - // udate the number of heaps in the joins - gc_t_join.update_n_threads(new_n_heaps); - #ifdef BACKGROUND_GC - bgc_t_join.update_n_threads(new_n_heaps); - #endif //BACKGROUND_GC - // compute the total budget per generation over the old heaps // and figure out what the new budget per heap is ptrdiff_t budget_per_heap[total_generation_count]; @@ -25741,16 +25892,25 @@ bool gc_heap::change_heap_count (int new_n_heaps) hp->decommission_heap(); } - if (GCScan::GetGcRuntimeStructuresValid()) + if (!init_only_p) { // make sure no allocation contexts point to idle heaps fix_allocation_contexts_heaps(); } - if (old_n_heaps < new_n_heaps) + dynamic_heap_count_data.last_n_heaps = old_n_heaps; + } + + // join the last time to change the heap count again if needed. + if (new_n_heaps < old_n_heaps) + { + gc_t_join.join (this, gc_join_merge_temp_fl); + if (gc_t_join.joined ()) { - // wake up threads for the new heaps - gc_idle_thread_event.Set(); + dprintf (9999, ("now changing the join heap count to the smaller one %d", new_n_heaps)); + gc_t_join.update_n_threads (new_n_heaps); + + gc_t_join.restart (); } } @@ -38902,9 +39062,9 @@ void gc_heap::bgc_thread_function() { // this is the case where we have more background GC threads than heaps // - wait until we're told to continue... - dprintf (3, ("BGC thread %d idle", heap_number)); - gc_idle_thread_event.Wait(INFINITE, FALSE); - dprintf (3, ("BGC thread %d waking from idle", heap_number)); + dprintf (9999, ("BGC thread %d idle (%d heaps) (gc%Id)", heap_number, n_heaps, VolatileLoadWithoutBarrier (&settings.gc_index))); + bgc_idle_thread_event.Wait(INFINITE, FALSE); + dprintf (9999, ("BGC thread %d waking from idle (%d heaps) (gc%Id)", heap_number, n_heaps, VolatileLoadWithoutBarrier (&settings.gc_index))); continue; } #endif //DYNAMIC_HEAP_COUNT @@ -47952,6 +48112,7 @@ HRESULT GCHeap::Initialize() uint32_t nhp = 1; uint32_t nhp_from_config = 0; + uint32_t max_nhp_from_config = (uint32_t)GCConfig::GetMaxHeapCount(); #ifndef MULTIPLE_HEAPS GCConfig::SetServerGC(false); @@ -48146,6 +48307,10 @@ HRESULT GCHeap::Initialize() #ifdef MULTIPLE_HEAPS assert (nhp <= g_num_processors); + if (max_nhp_from_config) + { + nhp = min (nhp, max_nhp_from_config); + } gc_heap::n_max_heaps = nhp; gc_heap::n_heaps = nhp; hr = gc_heap::initialize_gc (seg_size, large_seg_size, pin_seg_size, nhp); @@ -48296,9 +48461,32 @@ HRESULT GCHeap::Initialize() { // start with only 1 heap gc_heap::smoothed_desired_total[0] /= gc_heap::n_heaps; - gc_heap::g_heaps[0]->change_heap_count (1); + int initial_n_heaps = 1; + dprintf (9999, ("gc_heap::n_heaps is %d, initial %d", gc_heap::n_heaps, initial_n_heaps)); + + { + if (!gc_heap::prepare_to_change_heap_count (initial_n_heaps)) + { + // we don't have sufficient resources. + return E_FAIL; + } + + gc_heap::dynamic_heap_count_data.new_n_heaps = initial_n_heaps; + gc_heap::dynamic_heap_count_data.idle_thread_count = 0; + gc_heap::dynamic_heap_count_data.init_only_p = true; + + int max_threads_to_wake = max (gc_heap::n_heaps, initial_n_heaps); + gc_t_join.update_n_threads (max_threads_to_wake); + gc_heap::gc_start_event.Set (); + } + + gc_heap::g_heaps[0]->change_heap_count (initial_n_heaps); + gc_heap::gc_start_event.Reset (); + + // This needs to be different from our initial heap count so we can make sure we wait for + // the idle threads correctly in gc_thread_function. + gc_heap::dynamic_heap_count_data.last_n_heaps = 0; } - gc_heap::dynamic_heap_count_data.new_n_heaps = gc_heap::n_heaps; #endif //DYNAMIC_HEAP_COUNT GCScan::GcRuntimeStructuresValid (TRUE); @@ -49461,13 +49649,14 @@ void gc_heap::do_pre_gc() #ifdef TRACE_GC size_t total_allocated_since_last_gc = get_total_allocated_since_last_gc(); #ifdef BACKGROUND_GC - dprintf (1, (ThreadStressLog::gcDetailedStartMsg(), + //dprintf (1, (ThreadStressLog::gcDetailedStartMsg(), + dprintf (9999, ("*GC* %d(gen0:%d)(%d)(alloc: %zd)(%s)(%d)(h:%d)", VolatileLoad(&settings.gc_index), dd_collection_count (hp->dynamic_data_of (0)), settings.condemned_generation, total_allocated_since_last_gc, (settings.concurrent ? "BGC" : (gc_heap::background_running_p() ? "FGC" : "NGC")), - settings.b_state)); + settings.b_state, n_heaps)); #else dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %zd)", VolatileLoad(&settings.gc_index), @@ -49882,7 +50071,8 @@ void gc_heap::do_post_gc() } #endif //BGC_SERVO_TUNING - dprintf (1, (ThreadStressLog::gcDetailedEndMsg(), + //dprintf (1, (ThreadStressLog::gcDetailedEndMsg(), + dprintf (9999, ("*EGC* %zd(gen0:%zd)(%zd)(%d)(%s)(%s)(%s)(ml: %d->%d)", VolatileLoad(&settings.gc_index), dd_collection_count(hp->dynamic_data_of(0)), (size_t)(GetHighPrecisionTimeStamp() / 1000), diff --git a/src/coreclr/gc/gcconfig.h b/src/coreclr/gc/gcconfig.h index 72786778d5a97..aeded6bc97f17 100644 --- a/src/coreclr/gc/gcconfig.h +++ b/src/coreclr/gc/gcconfig.h @@ -83,6 +83,7 @@ class GCConfigStringHolder INT_CONFIG (BGCSpinCount, "BGCSpinCount", NULL, 140, "Specifies the bgc spin count") \ INT_CONFIG (BGCSpin, "BGCSpin", NULL, 2, "Specifies the bgc spin time") \ INT_CONFIG (HeapCount, "GCHeapCount", "System.GC.HeapCount", 0, "Specifies the number of server GC heaps") \ + INT_CONFIG (MaxHeapCount, "GCMaxHeapCount", "System.GC.MaxHeapCount", 0, "Specifies the max number of server GC heaps to adjust to") \ INT_CONFIG (Gen0Size, "GCgen0size", NULL, 0, "Specifies the smallest gen0 budget") \ INT_CONFIG (SegmentSize, "GCSegmentSize", NULL, 0, "Specifies the managed heap segment size") \ INT_CONFIG (LatencyMode, "GCLatencyMode", NULL, -1, "Specifies the GC latency mode - batch, interactive or low latency (note that the same " \ diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index da0085ce19610..207e8d9b8fd49 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -2421,6 +2421,7 @@ class gc_heap #ifndef USE_REGIONS PER_HEAP_METHOD void rearrange_heap_segments(BOOL compacting); #endif //!USE_REGIONS + PER_HEAP_METHOD void delay_free_segments(); PER_HEAP_ISOLATED_METHOD void distribute_free_regions(); #ifdef BACKGROUND_GC PER_HEAP_ISOLATED_METHOD void reset_write_watch_for_gc_heap(void* base_address, size_t region_size); @@ -2599,7 +2600,7 @@ class gc_heap // check if we should change the heap count PER_HEAP_METHOD void check_heap_count(); - PER_HEAP_METHOD bool prepare_to_change_heap_count (int new_n_heaps); + PER_HEAP_ISOLATED_METHOD bool prepare_to_change_heap_count (int new_n_heaps); PER_HEAP_METHOD bool change_heap_count (int new_n_heaps); #endif //DYNAMIC_HEAP_COUNT #endif //USE_REGIONS @@ -3777,6 +3778,13 @@ class gc_heap PER_HEAP_FIELD_MAINTAINED mark* loh_pinned_queue; #endif //FEATURE_LOH_COMPACTION +#ifdef DYNAMIC_HEAP_COUNT + PER_HEAP_FIELD_MAINTAINED GCEvent gc_idle_thread_event; +#ifdef BACKGROUND_GC + PER_HEAP_FIELD_MAINTAINED GCEvent bgc_idle_thread_event; +#endif //BACKGROUND_GC +#endif //DYNAMIC_HEAP_COUNT + /******************************************/ // PER_HEAP_FIELD_MAINTAINED_ALLOC fields // /******************************************/ @@ -4083,7 +4091,6 @@ class gc_heap // These 2 fields' values do not change but are set/unset per GC PER_HEAP_ISOLATED_FIELD_SINGLE_GC GCEvent gc_start_event; PER_HEAP_ISOLATED_FIELD_SINGLE_GC GCEvent ee_suspend_event; - PER_HEAP_ISOLATED_FIELD_SINGLE_GC GCEvent gc_idle_thread_event; // Also updated on the heap#0 GC thread because that's where we are actually doing the decommit. PER_HEAP_ISOLATED_FIELD_SINGLE_GC BOOL gradual_decommit_in_progress_p; @@ -4162,6 +4169,10 @@ class gc_heap PER_HEAP_ISOLATED_FIELD_SINGLE_GC uint8_t* gc_high; // high end of the highest region being condemned #endif //USE_REGIONS +#ifdef STRESS_DYNAMIC_HEAP_COUNT + PER_HEAP_ISOLATED_FIELD_SINGLE_GC int heaps_in_this_gc; +#endif //STRESS_DYNAMIC_HEAP_COUNT + /**************************************************/ // PER_HEAP_ISOLATED_FIELD_SINGLE_GC_ALLOC fields // /**************************************************/ @@ -4286,6 +4297,11 @@ class gc_heap float space_cost_decrease_per_step_down;// percentage effect on space of decreasing heap count int new_n_heaps; + // the heap count we changed from + int last_n_heaps; + // don't start a GC till we see (n_max_heaps - new_n_heaps) number of threads idling + VOLATILE(int32_t) idle_thread_count; + bool init_only_p; #ifdef STRESS_DYNAMIC_HEAP_COUNT int lowest_heap_with_msl_uoh; #endif //STRESS_DYNAMIC_HEAP_COUNT