Merge pull request #28 from VSadov/EtwCounters

Etw counters

src/coreclr/gc/objecthandle.cpp

@@ -1629,6 +1629,9 @@ void Ref_AgeHandles(uint32_t condemned, uint32_t maxgen, uintptr_t lp1)
 
         HNDTYPE_PINNED,
         HNDTYPE_VARIABLE,
+#if FEATURE_SATORI_GC
+        HNDTYPE_DEPENDENT,
+#endif
 #if defined(FEATURE_COMINTEROP) || defined(FEATURE_COMWRAPPERS) || defined(FEATURE_OBJCMARSHAL) || defined(FEATURE_REDHAWK)
         HNDTYPE_REFCOUNTED,
 #endif // FEATURE_COMINTEROP || FEATURE_COMWRAPPERS || FEATURE_OBJCMARSHAL || FEATURE_REDHAWK
@@ -1659,6 +1662,10 @@ void Ref_RejuvenateHandles(uint32_t condemned, uint32_t maxgen, uintptr_t lp1)
 {
     WRAPPER_NO_CONTRACT;
 
+#if FEATURE_SATORI_GC
+    __UNREACHABLE();
+#endif
+
     LOG((LF_GC, LL_INFO10000, "Rejuvenating handles.\n"));
 
     // these are the handle types that need their ages updated

src/coreclr/gc/satori/SatoriAllocator.cpp

@@ -27,8 +27,9 @@
 #include "common.h"
 #include "gcenv.h"
 #include "../env/gcenv.os.h"
-#include "SatoriUtil.h"
+#include "../gceventstatus.h"
 
+#include "SatoriUtil.h"
 #include "SatoriHeap.h"
 #include "SatoriObject.h"
 #include "SatoriObject.inl"
@@ -228,6 +229,14 @@ SatoriObject* SatoriAllocator::AllocRegular(SatoriAllocationContext* context, si
                     context->alloc_ptr += size;
                     result->CleanSyncBlock();
                     region->SetIndicesForObject(result, result->Start() + size);
+
+                    FIRE_EVENT(GCAllocationTick_V4,
+                        size,
+                        /*gen_number*/ 1,
+                        /*heap_number*/ 0,
+                        (void*)result,
+                        0);
+
                     return result;
                 }
                 else
@@ -386,6 +395,13 @@ SatoriObject* SatoriAllocator::AllocLarge(SatoriAllocationContext* context, size
                     result->CleanSyncBlock();
                     context->alloc_bytes_uoh += size;
                     region->SetIndicesForObject(result, result->Start() + size);
+
+                    FIRE_EVENT(GCAllocationTick_V4,
+                        size,
+                        /*gen_number*/ 1,
+                        /*heap_number*/ 0,
+                        (void*)result,
+                        0);
                 }
                 else
                 {
@@ -480,20 +496,29 @@ SatoriObject* SatoriAllocator::AllocHuge(SatoriAllocationContext* context, size_
     // but this one is not parseable yet since the new object has no MethodTable
     // we will keep the region in gen -1 for now and make it gen1 or gen2 in PublishObject.
     hugeRegion->StopAllocating(/* allocPtr */ 0);
+
+    FIRE_EVENT(GCAllocationTick_V4,
+        size,
+        /*gen_number*/ 2,
+        /*heap_number*/ 0,
+        (void*)result,
+        0);
+
     return result;
 }
 
 SatoriWorkChunk* SatoriAllocator::TryGetWorkChunk()
 {
-    SatoriWorkChunk* chunk = m_WorkChunks->TryPop();
-
 #if _DEBUG
+    static int i = 0;
     // simulate low memory case once in a while
-    if (!chunk && GCToOSInterface::GetCurrentProcessorNumber() == 2)
+    // This is just to force more overflows. Otherwise they are very rare.
+    if (i++ % 2 == 0)
     {
         return nullptr;
     }
 #endif
+    SatoriWorkChunk* chunk = m_WorkChunks->TryPop();
 
     while (!chunk && AddMoreWorkChunks())
     {

src/coreclr/gc/satori/SatoriLock.h

@@ -71,16 +71,10 @@ class SatoriSpinLock
 
     void Enter()
     {
-        int localBackoff = m_backoff;
-        while (VolatileLoadWithoutBarrier(&m_backoff) ||
-            !CompareExchangeNf(&m_backoff, localBackoff / 4 + 1, 0))
+        if (!CompareExchangeAcq(&m_backoff, 1, 0))
         {
-            localBackoff = Backoff(localBackoff);
+            EnterSpin();
         }
-
-#if !defined(TARGET_AMD64)
-        VolatileLoadBarrier();
-#endif
     }
 
     void Leave()
@@ -90,7 +84,18 @@ class SatoriSpinLock
     }
 
 private:
+
     NOINLINE
+    void EnterSpin()
+    {
+        int localBackoff = m_backoff;
+        while (VolatileLoadWithoutBarrier(&m_backoff) ||
+            !CompareExchangeAcq(&m_backoff, localBackoff / 4 + 1, 0))
+        {
+            localBackoff = Backoff(localBackoff);
+        }
+    }
+
     int Backoff(int backoff)
     {
         // TUNING: do we care about 1-proc machines?
@@ -108,16 +113,16 @@ class SatoriSpinLock
         return (backoff * 2 + 1) & 0x3FFF;
     }
 
-    static bool CompareExchangeNf(int volatile* destination, int exchange, int comparand)
+    static bool CompareExchangeAcq(int volatile* destination, int exchange, int comparand)
     {
 #ifdef _MSC_VER
 #if defined(TARGET_AMD64)
         return _InterlockedCompareExchange((long*)destination, exchange, comparand) == comparand;
 #else
-        return _InterlockedCompareExchange_nf((long*)destination, exchange, comparand) == comparand;
+        return _InterlockedCompareExchange_acq((long*)destination, exchange, comparand) == comparand;
 #endif
 #else
-        return __atomic_compare_exchange_n(destination, &comparand, exchange, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+        return __atomic_compare_exchange_n(destination, &comparand, exchange, true, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
 #endif
     }
 };

src/coreclr/gc/satori/SatoriPage.cpp

@@ -302,9 +302,7 @@ void SatoriPage::DirtyCardsForRange(size_t start, size_t end)
         this->m_cardGroups[i * 2] = Satori::CardState::DIRTY;
     }
 
-    VolatileStoreBarrier();
-
-    this->m_cardState = Satori::CardState::DIRTY;
+    VolatileStore(&this->m_cardState, Satori::CardState::DIRTY);
 }
 
 // dirtying in nonblocking phases could be unordered since we do not clean concurrently with mutator

src/coreclr/gc/satori/SatoriRecycler.cpp

@@ -737,6 +737,16 @@ bool SatoriRecycler::IsBlockingPhase()
 //        - could collect and use past history of the program behavior
 //        - could consider user input as to favor latency or throughput
 //        - ??
+
+// we target 1/EPH_SURV_TARGET ephemeral survival rate
+#define EPH_SURV_TARGET 4
+
+// we target 1/10 of total heap to be ephemeral. If more, we promote.
+#define EPH_RATIO 10
+
+// do gen2 when total doubles
+#define GEN2_THRESHOLD 2
+
 void SatoriRecycler::MaybeTriggerGC(gc_reason reason)
 {
     int generation = 0;
@@ -746,7 +756,10 @@ void SatoriRecycler::MaybeTriggerGC(gc_reason reason)
         generation = 1;
     }
 
-    if (m_gen1AddedSinceLastCollection + m_gen2AddedSinceLastCollection > m_totalBudget)
+    size_t currentAddedEstimate = m_gen2AddedSinceLastCollection +
+        m_gen1AddedSinceLastCollection / EPH_SURV_TARGET;
+
+    if (currentAddedEstimate > m_totalBudget)
     {
         generation = 2;
     }
@@ -775,16 +788,19 @@ size_t GetAvailableMemory()
 
 void SatoriRecycler::AdjustHeuristics()
 {
+    // ocupancies as of last collection
     size_t occupancy = GetTotalOccupancy();
     size_t ephemeralOccupancy = m_occupancy[1] + m_occupancy[0];
 
     if (m_prevCondemnedGeneration == 2)
     {
-        // we do gen2 if occupancy doubles, even if not using much memory
-        m_totalLimit = occupancy * 2;
+        m_totalLimit = occupancy * GEN2_THRESHOLD;
     }
 
-    size_t currentTotalEstimate = occupancy + m_gen1AddedSinceLastCollection + m_gen2AddedSinceLastCollection;
+    size_t currentTotalEstimate = occupancy +
+        m_gen2AddedSinceLastCollection +
+        m_gen1AddedSinceLastCollection / EPH_SURV_TARGET;
+
     m_totalBudget = m_totalLimit > currentTotalEstimate ?
         max(MIN_GEN1_BUDGET, m_totalLimit - currentTotalEstimate) :
         MIN_GEN1_BUDGET;
@@ -793,18 +809,16 @@ void SatoriRecycler::AdjustHeuristics()
     size_t available = GetAvailableMemory() * 9 / 10;
     m_totalBudget = min(m_totalBudget, available);
 
-    // if prev promoted, gen1 occupancy will be low, just keep the same budget.
-    // otherwise adjust
-    if (!m_promoteAllRegions)
-    {
-        // we look for ~20% ephemeral survivorship, also
-        // at least 1/8 total budget or gen1 min.
-        size_t minGen1 = max(MIN_GEN1_BUDGET, m_totalBudget / 8);
-        size_t newGen1Budget = max(minGen1, ephemeralOccupancy * 4);
+    // we look for 1 / EPH_SURV_TARGET ephemeral survivorship, thus budget is ephemeralOccupancy * EPH_SURV_TARGET
+    // we compute that based on actual ephemeralOccupancy or (occupancy / EPH_RATIO / 2), whichever is larger
+    // and limit that to MIN_GEN1_BUDGET
+    size_t minGen1 = max(MIN_GEN1_BUDGET, occupancy * EPH_SURV_TARGET / EPH_RATIO / 2);
 
-        // TUNING: using exponential smoothing with alpha == 1/2. is it a good smooth/lag balance?
-        m_gen1Budget = (m_gen1Budget + newGen1Budget) / 2;
-    }
+    size_t newGen1Budget = max(minGen1, ephemeralOccupancy * EPH_SURV_TARGET);
+
+    // smooth the budget a bit
+    // TUNING: using exponential smoothing with alpha == 1/2. is it a good smooth/lag balance?
+    m_gen1Budget = (m_gen1Budget + newGen1Budget) / 2;
 
     if (m_condemnedGeneration == 2)
     {
@@ -832,7 +846,7 @@ void SatoriRecycler::AdjustHeuristics()
         //    m_allowPromotingRelocations = true;
         //}
 
-        if (ephemeralOccupancy * 10 > occupancy)
+        if (ephemeralOccupancy * EPH_RATIO > occupancy)
         {
             m_promoteAllRegions = true;
         }
@@ -889,6 +903,18 @@ void SatoriRecycler::BlockingCollect()
 
     _ASSERTE(m_deferredSweepRegions->IsEmpty());
 
+    FIRE_EVENT(GCHeapStats_V2,
+        m_occupancy[0], 0,
+        m_occupancy[1], 0,
+        m_occupancy[2], 0,
+        0, 0,
+        0, 0,
+        0,
+        0,
+        0,
+        0,
+        0);
+
     // now we know survivorship after the last GC
     // and we can figure what we want to do in this GC and when we will do the next one
     AdjustHeuristics();
@@ -1010,12 +1036,11 @@ void SatoriRecycler::BlockingMark()
 
 void SatoriRecycler::DrainAndCleanWorker()
 {
-    bool revisitCards;
     do
     {
         DrainMarkQueues();
-        revisitCards = CleanCards();
-    } while (!m_workList->IsEmpty() || revisitCards);
+        CleanCards();
+    } while (!m_workList->IsEmpty() || HasDirtyCards());
 }
 
 void SatoriRecycler::MarkNewReachable()
@@ -1121,16 +1146,7 @@ void SatoriRecycler::PushToMarkQueuesSlow(SatoriWorkChunk*& currentWorkChunk, Sa
         MaybeAskForHelp();
     }
 
-#ifdef _DEBUG
-    // Limit work queue in debug/chk.
-    // This is just to force more overflows. Otherwise they are very rare.
-    currentWorkChunk = nullptr;
-    if (m_workList->Count() < 10)
-#endif
-    {
-        currentWorkChunk = m_heap->Allocator()->TryGetWorkChunk();
-    }
-
+    currentWorkChunk = m_heap->Allocator()->TryGetWorkChunk();
     if (currentWorkChunk)
     {
         currentWorkChunk->Push(o);
@@ -1660,7 +1676,8 @@ void SatoriRecycler::ScheduleMarkAsChildRanges(SatoriObject* o)
             SatoriWorkChunk* chunk = m_heap->Allocator()->TryGetWorkChunk();
             if (chunk == nullptr)
             {
-                o->ContainingRegion()->ContainingPage()->DirtyCardsForRange(start, remains);
+                o->ContainingRegion()->ContainingPage()->DirtyCardsForRange(start, start + remains);
+                remains = 0;
                 break;
             }
 
@@ -2235,10 +2252,9 @@ bool SatoriRecycler::HasDirtyCards()
 }
 
 // cleaning is not concurrent, but could be parallel
-bool SatoriRecycler::CleanCards()
+void SatoriRecycler::CleanCards()
 {
     SatoriWorkChunk* dstChunk = nullptr;
-    bool revisit = false;
 
     m_heap->ForEachPage(
         [&](SatoriPage* page)
@@ -2340,11 +2356,9 @@ bool SatoriRecycler::CleanCards()
                     }
                 }
 
-                // we do not see more cleaning work so clean the page state, unless the page went dirty while we were working on it
-                // in such case record a missed clean to revisit the whole deal. 
+                // we do not see more cleaning work so clean the page state, use interlocked in case the page went dirty while we were working on it
                 int8_t origState = Interlocked::CompareExchange(&page->CardState(), Satori::CardState::REMEMBERED, Satori::CardState::PROCESSING);
                 _ASSERTE(origState != Satori::CardState::BLANK);
-                revisit |= origState == Satori::CardState::DIRTY;
             }
         }
     );
@@ -2353,8 +2367,6 @@ bool SatoriRecycler::CleanCards()
     {
         m_workList->Push(dstChunk);
     }
-
-    return revisit;
 }
 
 void SatoriRecycler::UpdatePointersThroughCards()

src/coreclr/gc/satori/SatoriRecycler.h

@@ -228,7 +228,7 @@ class SatoriRecycler
 
     bool HasDirtyCards();
     bool ScanDirtyCardsConcurrent(int64_t deadline);
-    bool CleanCards();
+    void CleanCards();
     bool MarkHandles(int64_t deadline = 0);
     void ShortWeakPtrScan();
     void ShortWeakPtrScanWorker();

src/tests/issues.targets

@@ -138,6 +138,10 @@
         <Issue>Satori GC apis, LOH expectations</Issue>
       </ExcludeList>
 
+      <ExcludeList Include="$(XunitTestBinBase)/baseservices/RuntimeConfiguration/TestConfig/*">
+        <Issue>Satori GC apis</Issue>
+      </ExcludeList>
+
       <ExcludeList Include="$(XunitTestBinBase)/profiler/unittest/getappdomainstaticaddress/*">
         <Issue>Satori profiler tracing</Issue>
       </ExcludeList>