diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index bf7d5a07c12b21..00fee432b6a343 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -1595,6 +1595,33 @@ static void leave_spin_lock (GCSpinLock * spin_lock) #endif //_DEBUG +// simple spin locks for use by the GC itself where we don't want to wait for GC to finish +static void enter_simple_spin_lock (GCSpinLock* spin_lock) +{ + while (true) + { + if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0) + break; + + while (spin_lock->lock >= 0) + { + YieldProcessor(); // indicate to the processor that we are spinning + } + } +#ifdef _DEBUG + spin_lock->holding_thread = GCToEEInterface::GetThread(); +#endif //_DEBUG +} + +static void leave_simple_spin_lock (GCSpinLock* spin_lock) +{ +#ifdef _DEBUG + spin_lock->holding_thread = (Thread*)-1; +#endif //_DEBUG + spin_lock->lock = -1; +} + + bool gc_heap::enable_preemptive () { return GCToEEInterface::EnablePreemptiveGC(); @@ -2723,7 +2750,9 @@ uint64_t gc_heap::total_loh_a_last_bgc = 0; size_t gc_heap::eph_gen_starts_size = 0; #if defined(USE_REGIONS) region_free_list gc_heap::global_regions_to_decommit[count_free_region_kinds]; -region_free_list gc_heap::global_free_huge_regions; +#ifdef MULTIPLE_HEAPS +region_free_list gc_heap::global_free_regions[count_free_region_kinds]; +#endif //MULTIPLE_HEAPS #else heap_segment* gc_heap::segment_standby_list; #endif //USE_REGIONS @@ -3801,27 +3830,12 @@ uint8_t* region_allocator::allocate_end (uint32_t num_units, allocate_direction void region_allocator::enter_spin_lock() { - while (true) - { - if (Interlocked::CompareExchange(®ion_allocator_lock.lock, 0, -1) < 0) - break; - - while (region_allocator_lock.lock >= 0) - { - YieldProcessor(); // indicate to the processor that we are spinning - } - } -#ifdef _DEBUG - region_allocator_lock.holding_thread = GCToEEInterface::GetThread(); -#endif //_DEBUG + enter_simple_spin_lock (®ion_allocator_lock); } void region_allocator::leave_spin_lock() { -#ifdef _DEBUG - region_allocator_lock.holding_thread = (Thread*)-1; -#endif //_DEBUG - region_allocator_lock.lock = -1; + leave_simple_spin_lock (®ion_allocator_lock); } uint8_t* region_allocator::allocate (uint32_t num_units, allocate_direction direction, region_allocator_callback_fn fn) @@ -3852,8 +3866,11 @@ uint8_t* region_allocator::allocate (uint32_t num_units, allocate_direction dire uint32_t current_val = *(current_index - ((direction == -1) ? 1 : 0)); uint32_t current_num_units = get_num_units (current_val); bool free_p = is_unit_memory_free (current_val); + +#ifdef _DEBUG dprintf (REGIONS_LOG, ("ALLOC[%s: %zd]%d->%d", (free_p ? "F" : "B"), (size_t)current_num_units, (int)(current_index - region_map_left_start), (int)(current_index + current_num_units - region_map_left_start))); +#endif //_DEBUG if (free_p) { @@ -11366,6 +11383,8 @@ void gc_heap::return_free_region (heap_segment* region) } } +static GCSpinLock global_free_regions_spin_lock; + // USE_REGIONS TODO: SOH should be able to get a large region and split it up into basic regions // if needed. // USE_REGIONS TODO: In Server GC we should allow to get a free region from another heap. @@ -11373,34 +11392,71 @@ heap_segment* gc_heap::get_free_region (int gen_number, size_t size) { heap_segment* region = 0; + const size_t LARGE_REGION_SIZE = global_region_allocator.get_large_region_alignment(); + + free_region_kind kind; if (gen_number <= max_generation) { assert (size == 0); - region = free_regions[basic_free_region].unlink_region_front(); + kind = basic_free_region; + } + else if (size == LARGE_REGION_SIZE) + { + kind = large_free_region; } else { - const size_t LARGE_REGION_SIZE = global_region_allocator.get_large_region_alignment(); + kind = huge_free_region; + } - assert (size >= LARGE_REGION_SIZE); - if (size == LARGE_REGION_SIZE) - { - // get it from the local list of large free regions if possible - region = free_regions[large_free_region].unlink_region_front(); - } - else + if (kind != huge_free_region) + { + region = free_regions[kind].unlink_region_front(); + } + else + { + region = free_regions[kind].unlink_smallest_region (size); + } + +#ifdef MULTIPLE_HEAPS + if (region == nullptr) + { + // check global free region list for this kind of region + if (global_free_regions[kind].get_num_free_regions_no_verify() > 0) { - // get it from the local list of huge free regions if possible - region = free_regions[huge_free_region].unlink_smallest_region (size); - if (region == nullptr) + enter_simple_spin_lock (&global_free_regions_spin_lock); + + if (kind != huge_free_region) + { + region = global_free_regions[kind].unlink_region_front(); + } + else { ASSERT_HOLDING_SPIN_LOCK(&gc_lock); - // get it from the global list of huge free regions - region = global_free_huge_regions.unlink_smallest_region (size); + region = global_free_regions[kind].unlink_smallest_region (size); } + + leave_simple_spin_lock (&global_free_regions_spin_lock); + } + +#ifdef TRACE_GC + const char* kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; + if (region != nullptr) + { + dprintf (REGIONS_LOG, ("got %s region %zx (%zx) (committed: %zd) from global free list", + kind_name[kind], + region, + heap_segment_mem (region), + get_region_committed_size (region))); + } + else + { + dprintf (REGIONS_LOG, ("no %s regions available from global free list", kind_name[kind])); } +#endif // TRACE_GC } +#endif //MULTIPLE_HEAPS if (region) { @@ -12525,6 +12581,11 @@ size_t region_free_list::get_num_free_regions() return num_free_regions; } +size_t region_free_list::get_num_free_regions_no_verify() +{ + return num_free_regions; +} + void region_free_list::add_region (heap_segment* region, region_free_list to_free_list[count_free_region_kinds]) { free_region_kind kind = get_region_kind (region); @@ -12680,6 +12741,232 @@ void region_free_list::sort_by_committed_and_age() } tail_free_region = prev; } + +#ifdef MULTIPLE_HEAPS +void gc_heap::distribute_committed_in_free_regions(free_region_kind kind, size_t region_size, + size_t heap_budget_in_region_units[MAX_SUPPORTED_CPUS][2]) +{ + const ptrdiff_t MAX_PTR_DIFF = (ptrdiff_t)(~((size_t)0) >> 1); + const ptrdiff_t MIN_PTR_DIFF = ~MAX_PTR_DIFF; + +#ifdef TRACE_GC + const char* kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; +#endif // TRACE_GC + + ptrdiff_t min_committed; + ptrdiff_t max_committed; + + while (true) + { + // figure out which are the heaps with the minimum and maximum committed space relative to the budget + gc_heap* min_hp = nullptr; + gc_heap* max_hp = nullptr; + min_committed = MAX_PTR_DIFF; + max_committed = MIN_PTR_DIFF; + + + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + ptrdiff_t heap_committed = hp->free_regions[kind].get_size_committed_in_free() - heap_budget_in_region_units[i][kind]*region_size; + + dprintf (REGIONS_LOG, ("%s regions: heap %d committed %zd budget %zd difference %zd", + kind_name[kind], + i, + hp->free_regions[kind].get_size_committed_in_free(), + heap_budget_in_region_units[i][kind]*region_size, + heap_committed)); + + if (min_committed > heap_committed) + { + min_committed = heap_committed; + min_hp = hp; + } + if (max_committed < heap_committed) + { + max_committed = heap_committed; + max_hp = hp; + } + } + + if (min_hp == max_hp) + { + // in this case, all heaps are equal... + return; + } + + // the free regions should be sorted by decreasing commit, + // so swap a region with hopefully small commit from the end of the free list of the heap with small commit + // and a region with hopefully large commit from the start of the free list of the heap with large commit + assert (min_committed < max_committed); + dprintf (REGIONS_LOG, ("%s regions: heap %d has %zd committed in free, heap %d has %zd committed in free", + kind_name[kind], + min_hp->heap_number, + min_committed, + max_hp->heap_number, + max_committed)); + + heap_segment *small_region = min_hp->free_regions[kind].get_last_free_region(); + heap_segment *big_region = max_hp->free_regions[kind].get_first_free_region(); + if (small_region == nullptr || big_region == nullptr) + { + break; + } + + ptrdiff_t small_committed = get_region_committed_size (small_region); + ptrdiff_t big_committed = get_region_committed_size (big_region); + + ptrdiff_t diff_committed = big_committed - small_committed; + + // stop working if we are no longer improving balance + if ((diff_committed <= 0) || (diff_committed*2 >= max_committed - min_committed)) + { + break; + } + + dprintf (REGIONS_LOG, ("%s regions: moving %zd bytes of commit from heap %d to heap %d", + kind_name[kind], + diff_committed, + max_hp->heap_number, + min_hp->heap_number)); + + // unlink both regions from their heaps and add them to the other heap + region_free_list::unlink_region (small_region); + region_free_list::unlink_region (big_region); + + min_hp->free_regions[kind].add_region_in_descending_order (big_region); + max_hp->free_regions[kind].add_region_in_descending_order (small_region); + } + + // even out things further by moving everything over the minimum to the global free list + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + ptrdiff_t heap_committed = hp->free_regions[kind].get_size_committed_in_free() - heap_budget_in_region_units[i][kind]*region_size; + + while (heap_committed > min_committed) + { + heap_segment *small_region = hp->free_regions[kind].get_last_free_region(); + if (small_region == nullptr) + { + break; + } + + ptrdiff_t small_committed = get_region_committed_size (small_region); + + assert (small_committed <= (ptrdiff_t)hp->free_regions[kind].get_size_committed_in_free()); + if (heap_committed - small_committed < min_committed) + break; + + dprintf (REGIONS_LOG, ("%s regions: moving %zd bytes of commit from heap %d to global free region list", + kind_name[kind], + small_committed, + i)); + + region_free_list::unlink_region (small_region); + global_free_regions[kind].add_region_front (small_region); + + heap_committed -= small_committed; + } + } + // compute how much committed space we have now have in per-heap region free lists vs. the global free list + size_t all_heaps_committed = 0; + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + all_heaps_committed += hp->free_regions[kind].get_size_committed_in_free(); + } + // we desire a certain fraction of the total committed free space on the global list + // the amount we use corresponds to one heap's worth of commit + const float desired_fraction_in_global = 1.0f/n_heaps; + size_t desired_in_global = (size_t)((all_heaps_committed + global_free_regions[kind].get_size_committed_in_free())*desired_fraction_in_global); + + // remove regions from the per-heap free lists until we achieve that + while (global_free_regions[kind].get_size_committed_in_free() < desired_in_global) + { + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + heap_segment *small_region = hp->free_regions[kind].get_last_free_region(); + if (small_region == nullptr) + { + continue; + } + + ptrdiff_t small_committed = get_region_committed_size (small_region); + + assert (small_committed <= (ptrdiff_t)hp->free_regions[kind].get_size_committed_in_free()); + + dprintf (REGIONS_LOG, ("%s regions: moving %zd bytes of commit from heap %d to global free region list", + kind_name[kind], + small_committed, + i)); + + region_free_list::unlink_region (small_region); + global_free_regions[kind].add_region_front (small_region); + } + } +#ifdef TRACE_GC + size_t all_heaps_free = 0; + all_heaps_committed = 0; + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + all_heaps_free += hp->free_regions[kind].get_size_free_regions(); + all_heaps_committed += hp->free_regions[kind].get_size_committed_in_free(); + } + dprintf (REGIONS_LOG, ("dcif: %s free regions: %zd (%zd commit) heap, %zd (%zd commit) global", + kind_name[kind], + all_heaps_free, + all_heaps_committed, + global_free_regions[kind].get_size_free_regions(), + global_free_regions[kind].get_size_committed_in_free())); +#endif //TRACE_GC + + global_free_regions[kind].sort_by_committed_and_age(); +} +#endif //MULTIPLE_HEAPS + +void gc_heap::remove_old_or_small_regions (int hn, + region_free_list& from_list, +#ifdef MULTIPLE_HEAPS + region_free_list global_free_list[count_free_region_kinds], +#endif //MULTIPLE_HEAPS + BOOL last_gc_before_oom) +{ +#ifndef MULTIPLE_HEAPS + const int n_heaps = 1; +#endif //MULTIPLE_HEAPS + int age_in_free_to_decommit = min (max (AGE_IN_FREE_TO_DECOMMIT, n_heaps), MAX_AGE_IN_FREE); + heap_segment* next_region; + for (heap_segment* region = from_list.get_first_free_region(); region != nullptr; region = next_region) + { + next_region = heap_segment_next (region); + // when we are about to get OOM, we'd like to discount the free regions that just have the initial page commit as they are not useful + if ((heap_segment_age_in_free (region) >= age_in_free_to_decommit) || + ((get_region_committed_size (region) == GC_PAGE_SIZE) && last_gc_before_oom)) + { + dprintf (REGIONS_LOG, ("h%2d region %p age %2d, decommit", + hn, heap_segment_mem (region), heap_segment_age_in_free (region))); + region_free_list::unlink_region (region); + region_free_list::add_region (region, global_regions_to_decommit); + } +#ifdef MULTIPLE_HEAPS + // move regions with small committed size to the global free region list + else if ((global_free_list != nullptr) && (get_region_committed_size (region) <= get_region_size (region)/2)) + { + dprintf (REGIONS_LOG, ("move region %zx from heap %d to global free list", heap_segment_mem (region), hn)); + region_free_list::unlink_region (region); + region_free_list::add_region (region, global_free_list); + } +#endif //MULTIPLE_HEAPS + } +} #endif //USE_REGIONS void gc_heap::distribute_free_regions() @@ -12698,6 +12985,7 @@ void gc_heap::distribute_free_regions() } } #else + const int n_heaps = 1; BOOL joined_last_gc_before_oom = last_gc_before_oom; #endif //MULTIPLE_HEAPS if (settings.reason == reason_induced_aggressive) @@ -12756,17 +13044,23 @@ void gc_heap::distribute_free_regions() size_t total_num_free_regions[kind_count] = { 0, 0 }; size_t total_budget_in_region_units[kind_count] = { 0, 0 }; - size_t num_decommit_regions_by_time = 0; - size_t size_decommit_regions_by_time = 0; size_t heap_budget_in_region_units[MAX_SUPPORTED_CPUS][kind_count]; size_t min_heap_budget_in_region_units[MAX_SUPPORTED_CPUS]; size_t region_size[kind_count] = { global_region_allocator.get_region_alignment(), global_region_allocator.get_large_region_alignment() }; region_free_list surplus_regions[kind_count]; + int age_in_free_to_decommit = min (max (AGE_IN_FREE_TO_DECOMMIT, n_heaps), MAX_AGE_IN_FREE); for (int kind = basic_free_region; kind < kind_count; kind++) { // we may still have regions left on the regions_to_decommit list - // use these to fill the budget as well surplus_regions[kind].transfer_regions (&global_regions_to_decommit[kind]); + +#ifdef MULTIPLE_HEAPS + // and transfer the regions from the global free list back + surplus_regions[kind].transfer_regions (&global_free_regions[kind]); + + remove_old_or_small_regions (-1, surplus_regions[kind], global_free_regions, joined_last_gc_before_oom); +#endif //MULTIPLE_HEAPS } #ifdef MULTIPLE_HEAPS for (int i = 0; i < n_heaps; i++) @@ -12783,29 +13077,18 @@ void gc_heap::distribute_free_regions() for (int kind = basic_free_region; kind < kind_count; kind++) { // If there are regions in free that haven't been used in AGE_IN_FREE_TO_DECOMMIT GCs we always decommit them. - region_free_list& region_list = hp->free_regions[kind]; - heap_segment* next_region = nullptr; - for (heap_segment* region = region_list.get_first_free_region(); region != nullptr; region = next_region) - { - next_region = heap_segment_next (region); - int age_in_free_to_decommit = min (max (AGE_IN_FREE_TO_DECOMMIT, n_heaps), MAX_AGE_IN_FREE); - // when we are about to get OOM, we'd like to discount the free regions that just have the initial page commit as they are not useful - if ((heap_segment_age_in_free (region) >= age_in_free_to_decommit) || - ((get_region_committed_size (region) == GC_PAGE_SIZE) && joined_last_gc_before_oom)) - { - num_decommit_regions_by_time++; - size_decommit_regions_by_time += get_region_committed_size (region); - dprintf (REGIONS_LOG, ("h%2d region %p age %2d, decommit", - i, heap_segment_mem (region), heap_segment_age_in_free (region))); - region_free_list::unlink_region (region); - region_free_list::add_region (region, global_regions_to_decommit); - } - } - - total_num_free_regions[kind] += region_list.get_num_free_regions(); + remove_old_or_small_regions (i, hp->free_regions[kind], +#ifdef MULTIPLE_HEAPS + global_free_regions, +#endif //MULTIPLE_HEAPS + joined_last_gc_before_oom); + + total_num_free_regions[kind] += hp->free_regions[kind].get_num_free_regions(); } - global_free_huge_regions.transfer_regions (&hp->free_regions[huge_free_region]); +#ifdef MULTIPLE_HEAPS + global_free_regions[huge_free_region].transfer_regions (&hp->free_regions[huge_free_region]); +#endif //MULTIPLE_HEAPS heap_budget_in_region_units[i][basic_free_region] = 0; min_heap_budget_in_region_units[i] = 0; @@ -12814,15 +13097,19 @@ void gc_heap::distribute_free_regions() for (int gen = soh_gen0; gen < total_generation_count; gen++) { - if ((gen <= soh_gen2) && - total_budget_in_region_units[basic_free_region] >= (total_num_free_regions[basic_free_region] + - surplus_regions[basic_free_region].get_num_free_regions())) + size_t available_regions = (total_num_free_regions[basic_free_region] +#ifdef MULTIPLE_HEAPS + + global_free_regions[basic_free_region].get_num_free_regions() +#endif //MULTIPLE_HEAPS + + surplus_regions[basic_free_region].get_num_free_regions()); + + if ((gen <= soh_gen2) && (total_budget_in_region_units[basic_free_region] >= available_regions)) { // don't accumulate budget from higher soh generations if we cannot cover lower ones dprintf (REGIONS_LOG, ("out of free regions - skipping gen %d budget = %zd >= avail %zd", gen, total_budget_in_region_units[basic_free_region], - total_num_free_regions[basic_free_region] + surplus_regions[basic_free_region].get_num_free_regions())); + available_regions)); continue; } #ifdef MULTIPLE_HEAPS @@ -12850,11 +13137,30 @@ void gc_heap::distribute_free_regions() } } - dprintf (1, ("moved %2zd regions (%8zd) to decommit based on time", num_decommit_regions_by_time, size_decommit_regions_by_time)); +#ifdef MULTIPLE_HEAPS + global_free_regions[huge_free_region].transfer_regions (&global_regions_to_decommit[huge_free_region]); + + remove_old_or_small_regions (-1, global_free_regions[huge_free_region], nullptr, joined_last_gc_before_oom); - global_free_huge_regions.transfer_regions (&global_regions_to_decommit[huge_free_region]); + size_t free_space_in_huge_regions = global_free_regions[huge_free_region].get_size_free_regions(); +#else //MULTIPLE_HEAPS + free_regions[huge_free_region].transfer_regions (&global_regions_to_decommit[huge_free_region]); + + remove_old_or_small_regions (-1, free_regions[huge_free_region], joined_last_gc_before_oom); - size_t free_space_in_huge_regions = global_free_huge_regions.get_size_free_regions(); + size_t free_space_in_huge_regions = free_regions[huge_free_region].get_size_free_regions(); +#endif //MULTIPLE_HEAPS + +#ifdef TRACE_GC + size_t num_decommit_regions_by_time = 0; + size_t size_decommit_regions_by_time = 0; + for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) + { + num_decommit_regions_by_time += global_regions_to_decommit[kind].get_num_free_regions(); + size_decommit_regions_by_time += global_regions_to_decommit[kind].get_size_committed_in_free(); + } + dprintf (1, ("moved %2zd regions (%8zd) to decommit based on time", num_decommit_regions_by_time, size_decommit_regions_by_time)); +#endif //TRACE_GC ptrdiff_t num_regions_to_decommit[kind_count]; int region_factor[kind_count] = { 1, LARGE_REGION_FACTOR }; @@ -12862,90 +13168,38 @@ void gc_heap::distribute_free_regions() const char* kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; #endif // TRACE_GC -#ifndef MULTIPLE_HEAPS - // just to reduce the number of #ifdefs in the code below - const int n_heaps = 1; -#endif //!MULTIPLE_HEAPS - size_t num_huge_region_units_to_consider[kind_count] = { 0, free_space_in_huge_regions / region_size[large_free_region] }; for (int kind = basic_free_region; kind < kind_count; kind++) { num_regions_to_decommit[kind] = surplus_regions[kind].get_num_free_regions(); - dprintf(REGIONS_LOG, ("%zd %s free regions, %zd regions budget, %zd regions on decommit list, %zd huge regions to consider", +#ifdef MULTIPLE_HEAPS + size_t num_regions_on_global_free_list = global_free_regions[kind].get_num_free_regions(); +#else //MULTIPLE_HEAPS + size_t num_regions_on_global_free_list = 0; +#endif //MULTIPLE_HEAPS + + dprintf(REGIONS_LOG, ("%zd %s free regions, %zd regions budget, %zd regions on decommit list, %zd regions on global free list, %zd huge regions to consider", total_num_free_regions[kind], kind_name[kind], total_budget_in_region_units[kind], num_regions_to_decommit[kind], + num_regions_on_global_free_list, num_huge_region_units_to_consider[kind])); // check if the free regions exceed the budget // if so, put the highest free regions on the decommit list - total_num_free_regions[kind] += num_regions_to_decommit[kind]; + total_num_free_regions[kind] += num_regions_to_decommit[kind] + num_regions_on_global_free_list; ptrdiff_t balance = total_num_free_regions[kind] + num_huge_region_units_to_consider[kind] - total_budget_in_region_units[kind]; + // first determine how much we should decommit if ( #ifdef BACKGROUND_GC - background_running_p() || + !background_running_p() && #endif - (balance < 0)) - { - dprintf (REGIONS_LOG, ("distributing the %zd %s regions deficit", -balance, kind_name[kind])); - -#ifdef MULTIPLE_HEAPS - // we may have a deficit or - if background GC is going on - a surplus. - // adjust the budget per heap accordingly - if (balance != 0) - { - ptrdiff_t curr_balance = 0; - ptrdiff_t rem_balance = 0; - for (int i = 0; i < n_heaps; i++) - { - curr_balance += balance; - ptrdiff_t adjustment_per_heap = curr_balance / n_heaps; - curr_balance -= adjustment_per_heap * n_heaps; - ptrdiff_t new_budget = (ptrdiff_t)heap_budget_in_region_units[i][kind] + adjustment_per_heap; - ptrdiff_t min_budget = (kind == basic_free_region) ? (ptrdiff_t)min_heap_budget_in_region_units[i] : 0; - dprintf (REGIONS_LOG, ("adjusting the budget for heap %d from %zd %s regions by %zd to %zd", - i, - heap_budget_in_region_units[i][kind], - kind_name[kind], - adjustment_per_heap, - max (min_budget, new_budget))); - heap_budget_in_region_units[i][kind] = max (min_budget, new_budget); - rem_balance += new_budget - heap_budget_in_region_units[i][kind]; - } - assert (rem_balance <= 0); - dprintf (REGIONS_LOG, ("remaining balance: %zd %s regions", rem_balance, kind_name[kind])); - - // if we have a left over deficit, distribute that to the heaps that still have more than the minimum - while (rem_balance < 0) - { - for (int i = 0; i < n_heaps; i++) - { - size_t min_budget = (kind == basic_free_region) ? min_heap_budget_in_region_units[i] : 0; - if (heap_budget_in_region_units[i][kind] > min_budget) - { - dprintf (REGIONS_LOG, ("adjusting the budget for heap %d from %zd %s regions by %d to %zd", - i, - heap_budget_in_region_units[i][kind], - kind_name[kind], - -1, - heap_budget_in_region_units[i][kind] - 1)); - - heap_budget_in_region_units[i][kind] -= 1; - rem_balance += 1; - if (rem_balance == 0) - break; - } - } - } - } -#endif //MULTIPLE_HEAPS - } - else + (balance >= 0)) { num_regions_to_decommit[kind] = balance; dprintf(REGIONS_LOG, ("distributing the %zd %s regions, removing %zd regions", @@ -12980,7 +13234,79 @@ void gc_heap::distribute_free_regions() // cannot assert we moved any regions because there may be a single huge region with more than we want to decommit } } + balance = 0; } + +#ifdef MULTIPLE_HEAPS + // whatever is on the global free list is not available to distribute on the per-heap free lists + balance -= global_free_regions[kind].get_num_free_regions(); + + dprintf (REGIONS_LOG, ("distributing the %zd %s regions deficit", -balance, kind_name[kind])); + + // we may have a deficit or - if background GC is going on - a surplus. + // adjust the budget per heap accordingly + if (balance != 0) + { + ptrdiff_t curr_balance = 0; + ptrdiff_t rem_balance = 0; + for (int i = 0; i < n_heaps; i++) + { + curr_balance += balance; + ptrdiff_t adjustment_per_heap = curr_balance / n_heaps; + curr_balance -= adjustment_per_heap * n_heaps; + ptrdiff_t new_budget = (ptrdiff_t)heap_budget_in_region_units[i][kind] + adjustment_per_heap; + ptrdiff_t min_budget = (kind == basic_free_region) ? (ptrdiff_t)min_heap_budget_in_region_units[i] : 0; + dprintf (REGIONS_LOG, ("adjusting the budget for heap %d from %zd %s regions by %zd to %zd", + i, + heap_budget_in_region_units[i][kind], + kind_name[kind], + adjustment_per_heap, + max (min_budget, new_budget))); + heap_budget_in_region_units[i][kind] = max (min_budget, new_budget); + rem_balance += new_budget - heap_budget_in_region_units[i][kind]; + } + assert (rem_balance <= 0); + dprintf (REGIONS_LOG, ("remaining balance: %zd %s regions", rem_balance, kind_name[kind])); + + // if we have a left over deficit, distribute that to the heaps that still have more than the minimum + int pass = 1; + while (rem_balance < 0) + { + bool progress = false; + for (int i = 0; i < n_heaps; i++) + { + size_t min_budget = ((kind == basic_free_region) && (pass == 1)) ? min_heap_budget_in_region_units[i] : 0; + if (heap_budget_in_region_units[i][kind] > min_budget) + { + dprintf (REGIONS_LOG, ("adjusting the budget for heap %d from %zd %s regions by %zd to %zd", + i, + heap_budget_in_region_units[i][kind], + kind_name[kind], + -1, + heap_budget_in_region_units[i][kind] - 1)); + + heap_budget_in_region_units[i][kind] -= 1; + rem_balance += 1; + progress = true; + if (rem_balance == 0) + break; + } + } + if (!progress) + { + if (kind == basic_free_region && pass == 1) + { + // if we have made no progress, disregard min_heap_budget on the next pass + pass = 2; + } + else + { + break; + } + } + } + } +#endif //MULTIPLE_HEAPS } for (int kind = basic_free_region; kind < kind_count; kind++) @@ -13027,6 +13353,10 @@ void gc_heap::distribute_free_regions() hp->free_regions[kind].sort_by_committed_and_age(); } +#ifdef MULTIPLE_HEAPS + distribute_committed_in_free_regions ((free_region_kind)kind, region_size[kind], heap_budget_in_region_units); +#endif //MULTIPLE_HEAPS + if (surplus_regions[kind].get_num_free_regions() > 0) { assert (!"should have exhausted the surplus_regions"); @@ -13069,6 +13399,24 @@ void gc_heap::distribute_free_regions() } } #endif //MULTIPLE_HEAPS + + // report basic free regions as extra gen0 commit and the tails of any regions in gen0. +#ifdef MULTIPLE_HEAPS + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; +#else //MULTIPLE_HEAPS + { + gc_heap* hp = pGenGCHeap; +#endif //MULTIPLE_HEAPS + generation* gen = hp->generation_of (soh_gen0); + size_t extra_gen0_commit = hp->free_regions[basic_free_region].get_size_committed_in_free(); + for (heap_segment* region = heap_segment_rw (generation_start_segment (gen)); region != nullptr; region = heap_segment_next (region)) + { + extra_gen0_commit += heap_segment_committed (region) - heap_segment_allocated (region); + } + hp->get_gc_data_per_heap()->extra_gen0_committed = extra_gen0_commit; + } #endif //USE_REGIONS } @@ -20304,18 +20652,11 @@ bool gc_heap::try_get_new_free_region() } else { - region = allocate_new_region (__this, 0, false); + region = get_free_region (0); if (region) { - if (init_table_for_region (0, region)) - { - return_free_region (region); - dprintf (REGIONS_LOG, ("h%d got a new empty region %p", heap_number, region)); - } - else - { - region = 0; - } + return_free_region (region); + dprintf (REGIONS_LOG, ("h%d got a new empty region %p", heap_number, region)); } } @@ -21685,6 +22026,20 @@ void gc_heap::gc1() } #endif //USE_REGIONS } +#ifdef USE_REGIONS + if (settings.condemned_generation == max_generation) + { + // age and print all kinds of free regions + region_free_list::age_free_regions (global_free_regions); + region_free_list::print (global_free_regions, -1, "END"); + } + else + { + // age and print only basic free regions + global_free_regions[basic_free_region].age_free_regions(); + global_free_regions[basic_free_region].print (-1, "END"); + } +#endif //USE_REGIONS fire_pevents(); update_end_ngc_time(); @@ -21707,6 +22062,12 @@ void gc_heap::gc1() #endif //FEATURE_LOH_COMPACTION decommit_ephemeral_segment_pages(); +#ifdef USE_REGIONS + if (!(settings.concurrent)) + { + distribute_free_regions(); + } +#endif //USE_REGIONS fire_pevents(); if (!(settings.concurrent)) @@ -21714,7 +22075,6 @@ void gc_heap::gc1() rearrange_uoh_segments(); #ifdef USE_REGIONS initGCShadow(); - distribute_free_regions(); verify_region_to_generation_map (); compute_gc_and_ephemeral_range (settings.condemned_generation, true); stomp_write_barrier_ephemeral (ephemeral_low, ephemeral_high, @@ -22681,6 +23041,50 @@ void gc_heap::garbage_collect (int n) #endif //FEATURE_BASICFREEZE #ifdef MULTIPLE_HEAPS + +#ifdef TRACE_GC + size_t total_committed_in_free = 0; + for (int kind = basic_free_region; kind < count_free_region_kinds; kind++) + { + size_t all_heaps_free = 0; + size_t all_heaps_committed = 0; + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + all_heaps_free += hp->free_regions[kind].get_size_free_regions(); + all_heaps_committed += hp->free_regions[kind].get_size_committed_in_free(); + } + const char* kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; + dprintf (REGIONS_LOG, ("%s free regions: %zd (%zd commit) heap, %zd (%zd commit) global", + kind_name[kind], + all_heaps_free, + all_heaps_committed, + global_free_regions[kind].get_size_free_regions(), + global_free_regions[kind].get_size_committed_in_free())); + + total_committed_in_free += all_heaps_committed + global_free_regions[kind].get_size_committed_in_free(); + } + size_t total_committed = total_committed_in_free; + size_t total_allocated = 0; + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; + + for (int gen_idx = soh_gen0; gen_idx < total_generation_count; gen_idx++) + { + generation* gen = hp->generation_of (gen_idx); + for (heap_segment* region = heap_segment_rw (generation_start_segment (gen)); + region != nullptr; region = heap_segment_next (region)) + { + total_committed += get_region_committed_size (region); + total_allocated += heap_segment_allocated (region) - heap_segment_mem (region); + } + } + } + dprintf (REGIONS_LOG, ("total committed memory: %zd (%zd in free) total allocated: %zd", total_committed, total_committed_in_free, total_allocated)); +#endif //TRACE_GC + for (int i = 0; i < n_heaps; i++) { gc_heap* hp = g_heaps[i]; @@ -35273,6 +35677,27 @@ void gc_heap::background_mark_phase () last_mark_time = GetHighPrecisionTimeStamp(); #endif //FEATURE_EVENT_TRACE +#ifdef USE_REGIONS + // age and print all kinds of free regions + // safe to do here because the EE is suspended and we are joined +#ifdef MULTIPLE_HEAPS + for (int i = 0; i < n_heaps; i++) + { + gc_heap* hp = g_heaps[i]; +#else + { + gc_heap* hp = pGenGCHeap; + int i = 0; +#endif //MULTIPLE_HEAPS + region_free_list::age_free_regions (hp->free_regions); + region_free_list::print (hp->free_regions, i, "END"); + } +#ifdef MULTIPLE_HEAPS + region_free_list::age_free_regions (global_free_regions); + region_free_list::print (global_free_regions, -1, "END"); +#endif //MULTIPLE_HEAPS +#endif //USE_REGIONS + #ifdef MULTIPLE_HEAPS dprintf(3, ("Joining BGC threads after absorb")); bgc_t_join.restart(); @@ -41069,10 +41494,14 @@ bool gc_heap::decommit_step (uint64_t step_milliseconds) uint8_t* end = use_large_pages_p ? heap_segment_used(region) : heap_segment_committed(region); size_t size = end - page_start; bool decommit_succeeded_p = false; +#ifdef TRACE_GC + const char* kind_name[count_free_region_kinds] = { "basic", "large", "huge"}; +#endif //TRACE_GC if (!use_large_pages_p) { decommit_succeeded_p = virtual_decommit(page_start, size, recorded_committed_free_bucket); - dprintf(REGIONS_LOG, ("decommitted region %p(%p-%p) (%zu bytes) - success: %d", + dprintf(REGIONS_LOG, ("decommitted %s region %zx(%zx-%zx) (%zu bytes) - success: %d", + kind_name[kind], region, page_start, end, @@ -41082,7 +41511,8 @@ bool gc_heap::decommit_step (uint64_t step_milliseconds) if (!decommit_succeeded_p) { memclr(page_start, size); - dprintf(REGIONS_LOG, ("cleared region %p(%p-%p) (%zu bytes)", + dprintf(REGIONS_LOG, ("cleared %s region %zx(%zx-%zx) (%zu bytes)", + kind_name[kind], region, page_start, end, diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 6514729366da23..b1212728e59e7b 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -259,7 +259,7 @@ void GCLog (const char *fmt, ... ); #define dprintf(l,x) {if ((l == 1) || (l == GTC_LOG)) {GCLog x;}} #else //SIMPLE_DPRINTF #ifdef HOST_64BIT -#define dprintf(l,x) STRESS_LOG_VA(l,x); +#define dprintf(l,x) {if ((l == 1) || (l == REGIONS_LOG)) {STRESS_LOG_VA(l,x);}} #else #error Logging dprintf to stress log on 32 bits platforms is not supported. #endif @@ -1186,9 +1186,11 @@ class region_free_list heap_segment* unlink_region_front(); heap_segment* unlink_smallest_region (size_t size); size_t get_num_free_regions(); + size_t get_num_free_regions_no_verify(); size_t get_size_committed_in_free() { return size_committed_in_free_regions; } size_t get_size_free_regions() { return size_free_regions; } heap_segment* get_first_free_region() { return head_free_region; } + heap_segment* get_last_free_region() { return tail_free_region; } static void unlink_region (heap_segment* region); static void add_region (heap_segment* region, region_free_list to_free_list[count_free_region_kinds]); static void add_region_descending (heap_segment* region, region_free_list to_free_list[count_free_region_kinds]); @@ -2099,6 +2101,20 @@ class gc_heap PER_HEAP void rearrange_heap_segments(BOOL compacting); #endif //!USE_REGIONS +#if defined(MULTIPLE_HEAPS) && defined(USE_REGIONS) + PER_HEAP_ISOLATED + void distribute_committed_in_free_regions(free_region_kind kind, size_t region_size, + size_t heap_budget_in_region_units[MAX_SUPPORTED_CPUS][2]); +#endif //MULTIPLE_HEAPS && USE_REGIONS +#ifdef USE_REGIONS + PER_HEAP_ISOLATED + void remove_old_or_small_regions (int hn, + region_free_list& from_list, +#ifdef MULTIPLE_HEAPS + region_free_list global_free_list[count_free_region_kinds], +#endif //MULTIPLE_HEAPS + BOOL last_gc_before_oom); +#endif //USE_REGIONS PER_HEAP_ISOLATED void distribute_free_regions(); #ifdef BACKGROUND_GC @@ -4995,8 +5011,10 @@ class gc_heap PER_HEAP_ISOLATED region_free_list global_regions_to_decommit[count_free_region_kinds]; +#ifdef MULTIPLE_HEAPS PER_HEAP_ISOLATED - region_free_list global_free_huge_regions; + region_free_list global_free_regions[count_free_region_kinds]; +#endif #endif //USE_REGIONS PER_HEAP