[NOT READY] GC: Rearrange finalization queue to put freelist in middle

src/coreclr/gc/gc.cpp

@@ -50994,10 +50994,14 @@ bool CFinalize::Initialize()
     }
     m_EndArray = &m_Array[INITIAL_FINALIZER_ARRAY_SIZE];
 
-    for (int i =0; i < FreeList; i++)
+    for (int i = 0; i < FreeListSeg; i++)
     {
         SegQueueLimit (i) = m_Array;
     }
+    for (int i = FreeListSeg; i < MaxSeg; i++)
+    {
+        SegQueueLimit (i) = m_EndArray;
+    }
     m_PromotedCount = 0;
     lock = -1;
 #ifdef _DEBUG
@@ -51142,8 +51146,15 @@ CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
             return false;
         }
     }
+
+    // Start:   | ... | dest   |a  other  |b  gen0  |_  free   |  ...
+    // Result:  | ... | dest    o| other   a| gen0   b| free   |  ...
+    //
+    // The loop shuffles 'a' to 'b' to _, and the final store after the loop puts 'o' in place.
+
+    assert (dest < FreeListSeg);
     Object*** end_si = &SegQueueLimit (dest);
-    do
+    while (s_i > end_si)
     {
         //is the segment empty?
         if (!(*s_i == *(s_i-1)))
@@ -51155,7 +51166,7 @@ CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
         (*s_i)++;
         //go to the next segment.
         s_i--;
-    } while (s_i > end_si);
+    }
 
     // We have reached the destination segment
     // store the object
@@ -51176,14 +51187,14 @@ CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
 
     if (!IsSegEmpty(FinalizerListSeg))
     {
-        obj =  *(--SegQueueLimit (FinalizerListSeg));
+        obj =  *(SegQueue (FinalizerListSeg)++);
     }
     else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
     {
         //the FinalizerList is empty, we can adjust both
         // limit instead of moving the object to the free list
-        obj =  *(--SegQueueLimit (CriticalFinalizerListSeg));
-        --SegQueueLimit (FinalizerListSeg);
+        obj =  *(SegQueue (CriticalFinalizerListSeg)++);
+        SegQueue (FinalizerListSeg)++;
     }
     if (obj)
     {
@@ -51398,10 +51409,21 @@ CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
     unsigned int Seg = gen_segment (gen);
 
     Object** startIndex = SegQueue (Seg);
+    Object** endIndex = SegQueue (FreeListSeg);
+
+    dprintf (3, ("RelocateFinalizationData gen=%d, [%p,%p[", gen, startIndex, endIndex));
 
-    dprintf (3, ("RelocateFinalizationData gen=%d, [%p,%p[", gen, startIndex, SegQueue (FreeList)));
+    for (Object** po = startIndex; po < endIndex; po++)
+    {
+        GCHeap::Relocate (po, &sc);
+    }
+
+    startIndex = SegQueueLimit (FreeListSeg);
+    endIndex = SegQueueLimit (MaxSeg);
 
-    for (Object** po = startIndex; po < SegQueue (FreeList);po++)
+    dprintf (3, ("RelocateFinalizationData gen=%d, [%p,%p[", gen, startIndex, endIndex));
+
+    for (Object** po = startIndex; po < endIndex; po++)
     {
         GCHeap::Relocate (po, &sc);
     }
@@ -51470,15 +51492,28 @@ CFinalize::GrowArray()
     {
         return FALSE;
     }
-    memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
+
+    // Start:   | gen2 | gen1 | gen0 | free | final | crit |
+    //
+    // Result:  | gen2 | gen1 | gen0 |   free    | final | crit |
+
+    size_t prefixCount = SegQueue(FreeListSeg) - m_Array;
+    memcpy (newArray, m_Array, prefixCount * sizeof(Object*));
+    size_t postfixCount = m_EndArray - SegQueueLimit(FreeListSeg);
+    memcpy (newArray + newArraySize - postfixCount, SegQueueLimit(FreeListSeg), postfixCount * sizeof(Object*));
 
     dprintf (3, ("Grow finalizer array [%p,%p[ -> [%p,%p[", m_Array, m_EndArray, newArray, &m_Array[newArraySize]));
 
     //adjust the fill pointers
-    for (int i = 0; i < FreeList; i++)
+    for (int i = 0; i < FreeListSeg; i++)
     {
         m_FillPointers [i] += (newArray - m_Array);
     }
+    //additional space in new array is added to FreeListSeg, not the end
+    for (int i = FreeListSeg; i < MaxSeg; ++i)
+    {
+        m_FillPointers [i] += (newArray - m_Array) + (newArraySize - oldArraySize);
+    }
     delete[] m_Array;
     m_Array = newArray;
     m_EndArray = &m_Array [newArraySize];
@@ -51500,6 +51535,7 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq)
     size_t thisArraySize = (m_EndArray - m_Array);
     size_t thisNeededArraySize = UsedCount();
     size_t neededArraySize = thisNeededArraySize + otherNeededArraySize;
+    size_t growthCount = 0;
 
     Object ** newArray = m_Array;
 
@@ -51508,6 +51544,7 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq)
     {
         // if not allocate new array
         newArray = new (nothrow) Object*[neededArraySize];
+        growthCount = neededArraySize - thisArraySize;
 
         // if unsuccessful, return false without changing anything
         if (!newArray)
@@ -51519,10 +51556,11 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq)
 
     // Since the target might be the original array (with the original data),
     // the order of copying must not overwrite any data until it has been
-    // copied.
+    // copied. Both loops skip 'FreeListSeg' because there is no need to copy
+    // the unused data in the freelist.
 
-    // copy the finalization data from this and the other finalize queue
-    for (int i = FreeList - 1; i >= 0; i--)
+    // copy the generation data from this and the other finalize queue
+    for (int i = FreeListSeg - 1; i >= 0; i--)
     {
         size_t thisIndex = SegQueue (i) - m_Array;
         size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array;
@@ -51535,9 +51573,28 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq)
         memmove (&newArray[thisLimit + otherIndex], &other_fq->m_Array[otherIndex], sizeof(newArray[0])*otherSize);
     }
 
+    // copy the finalization data from this and the other finalize queue
+    //
+    // note reverse order from above to preserve the existing data
+    for (int i = FreeListSeg + 1; i <= MaxSeg; i++)
+    {
+        size_t thisIndexFromEnd = m_EndArray - SegQueue (i);
+        size_t otherIndexFromEnd = other_fq->m_EndArray - other_fq->SegQueue (i);
+        size_t thisLimitFromEnd = m_EndArray - SegQueueLimit (i);
+        size_t otherLimitFromEnd = other_fq->m_EndArray - other_fq->SegQueueLimit (i);
+        size_t thisSize = thisIndexFromEnd - thisLimitFromEnd;
+        size_t otherSize = otherIndexFromEnd - otherLimitFromEnd;
+
+        memmove (&newArray[thisArraySize + growthCount - thisIndexFromEnd - otherIndexFromEnd],           m_EndArray -  thisIndexFromEnd, sizeof(newArray[0])*thisSize );
+        memmove (&newArray[thisArraySize + growthCount - thisLimitFromEnd - otherIndexFromEnd], other_fq->m_EndArray - otherIndexFromEnd, sizeof(newArray[0])*otherSize);
+    }
+
     // adjust the m_FillPointers to reflect the sum of both queues on this queue,
     // and reflect that the other queue is now empty
-    for (int i = FreeList - 1; i >= 0; i--)
+    //
+    // unlike copying, these loops do need to set the 'FreeListSeg' boundaries,
+    // but including MaxSeg itself would set m_EndArray
+    for (int i = 0; i < FreeListSeg; ++i)
     {
         size_t thisLimit = SegQueueLimit (i) - m_Array;
         size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array;
@@ -51546,6 +51603,15 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq)
 
         other_fq->SegQueueLimit (i) = other_fq->m_Array;
     }
+    for (int i = MaxSeg; i > FreeListSeg; i--)
+    {
+        size_t thisLimit = m_EndArray - SegQueue (i);
+        size_t otherLimit = other_fq->m_EndArray - other_fq->SegQueue (i);
+
+        SegQueue (i) = &newArray[(thisArraySize + growthCount) - thisLimit - otherLimit];
+
+        other_fq->SegQueue (i) = other_fq->m_EndArray;
+    }
     if (m_Array != newArray)
     {
         delete[] m_Array;
@@ -51591,25 +51657,15 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq)
     // move half of the items in each section over to the other queue
     PTR_PTR_Object newFillPointers[MaxSeg];
     PTR_PTR_Object segQueue = m_Array;
-    for (int i = 0; i < FreeList; i++)
+    for (int i = 0; i < FreeListSeg; i++)
     {
-        size_t thisIndex = SegQueue (i) - m_Array;
-        size_t thisLimit = SegQueueLimit (i) - m_Array;
-        size_t thisSize = thisLimit - thisIndex;
-
-        // we move half to the other queue
-        size_t otherSize = thisSize / 2;
-        size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array;
-        size_t thisNewSize = thisSize - otherSize;
-
-        memmove (&other_fq->m_Array[otherIndex], &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0])*otherSize);
-        other_fq->SegQueueLimit (i) = &other_fq->m_Array[otherIndex + otherSize];
+        segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i);
+    }
 
-        // slide the unmoved half to its new position in the queue
-        // (this will delete the moved half once copies and m_FillPointers updates are completed)
-        memmove (segQueue, &m_Array[thisIndex], sizeof(m_Array[0])*thisNewSize);
-        segQueue += thisNewSize;
-        newFillPointers[i] = segQueue;
+    segQueue = m_EndArray;
+    for (int i = MaxSeg; i > FreeListSeg; i--)
+    {
+        segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i);
     }
 
     // finally update the fill pointers from the new copy we generated
@@ -51621,6 +51677,75 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq)
     return true;
 }
 
+// split finalization data for one segment
+//
+// Case: segment < FreeListSeg
+//   Start:
+//           this: | copied | ... | half1 half2 | ... |
+//                          ^
+//                       thisDest
+//
+//       other_fq: | copied | ........... |
+//
+//   Result: this: | copied | half1 | ............... |
+//                                  ^
+//                             return value
+//
+//       other_fq: | copied | half2 | ... |
+//
+// Case: segment > FreeListSeg
+//   Start:
+//           this: | ... | half1 half2 | ... | copied |
+//                                           ^
+//                                        thisDest
+//
+//       other_fq: | ........... | copied |
+//
+//   Result: this: | ............... | half1 | copied |
+//                                   ^
+//                              return value
+//
+//       other_fq: | ... | half2 | copied |
+//
+// dest might be unique or m_Array in fq1
+Object** CFinalize::SplitSegment(Object** thisDest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment)
+{
+    ASSERT(segment != FreeListSeg);
+
+    size_t thisIndex = SegQueue (segment) - m_Array;
+    size_t thisLimit = SegQueueLimit (segment) - m_Array;
+    size_t thisSize = thisLimit - thisIndex;
+
+    // we move half to the other queue
+    // and slide the other half to its new position in the queue
+    // (this will delete the moved half once copies and m_FillPointers updates are completed)
+    size_t otherSize = thisSize / 2;
+    size_t thisNewSize = thisSize - otherSize;
+
+    if (segment < FreeListSeg)
+    {
+        Object** otherDest = other_fq->SegQueue(segment);
+        memmove (otherDest, &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0]) * otherSize);
+        other_fq->SegQueueLimit (segment) = otherDest + otherSize;
+
+        memmove (thisDest, &m_Array[thisIndex], sizeof(m_Array[0]) * thisNewSize);
+        thisDest += thisNewSize;
+        newFillPointers[segment] = thisDest;
+    }
+    else
+    {
+        Object** otherDest = other_fq->SegQueueLimit(segment) - otherSize;
+        memmove (otherDest, &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0]) * otherSize);
+        other_fq->SegQueue (segment) = otherDest;
+
+        thisDest -= thisNewSize;
+        memmove (thisDest, &m_Array[thisIndex], sizeof(m_Array[0]) * thisNewSize);
+        newFillPointers[segment - 1] = thisDest;
+    }
+
+    return thisDest;
+}
+
 #ifdef VERIFY_HEAP
 void CFinalize::CheckFinalizerObjects()
 {

src/coreclr/gc/gcpriv.h

@@ -4690,19 +4690,22 @@ class CFinalize
     // Therefore, the lower bound on segment X is m_FillPointers[x-1] with a
     // special case for the first, and the upper bound on segment X is
     // m_FillPointers[x] with special cases for the last.
+    //
+    // The "generation" segments [0, FreeListSeg) grow/shrink on the 'End' side
+    // (high index), and the "finalizer" segments grow/shrink on the 'Start'
+    // side (low index).
 
     // Adjust the count and add a constant to add a segment
     static const int ExtraSegCount = 2;
+    static const int FreeListSeg = total_generation_count;
     static const int FinalizerListSeg = total_generation_count + 1;
-    static const int CriticalFinalizerListSeg = total_generation_count;
     // The end of this segment is m_EndArray, not an entry in m_FillPointers.
-    static const int FreeListSeg = total_generation_count + ExtraSegCount;
-    static const int FreeList = FreeListSeg;
+    static const int CriticalFinalizerListSeg = total_generation_count + 2;
 
-    static const int FinalizerStartSeg = CriticalFinalizerListSeg;
-    static const int FinalizerMaxSeg = FinalizerListSeg;
+    static const int FinalizerStartSeg = FinalizerListSeg;
+    static const int FinalizerMaxSeg = CriticalFinalizerListSeg;
 
-    static const int MaxSeg = FreeListSeg;
+    static const int MaxSeg = CriticalFinalizerListSeg;
 
     PTR_PTR_Object m_FillPointers[total_generation_count + ExtraSegCount];
     PTR_PTR_Object m_Array;
@@ -4739,6 +4742,8 @@ class CFinalize
         return (SegQueueLimit(i) == SegQueue (i));
     }
 
+    Object** SplitSegment (Object** dest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment);
+
 public:
     ~CFinalize();
     bool Initialize();