Skip to content

Commit

Permalink
Heap accounting improvements and cleanups.
Browse files Browse the repository at this point in the history
  • Loading branch information
@christianhaeubl
christianhaeubl committed May 30, 2023
1 parent a9b50f9 commit 9d65db1
Show file tree
Hide file tree
Showing 9 changed files with 129 additions and 125 deletions.
68 changes: 32 additions & 36 deletions ...src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@

import com.oracle.svm.core.AlwaysInline;
import com.oracle.svm.core.Uninterruptible;
import com.oracle.svm.core.log.Log;

/**
* This data is only updated during a GC.
Expand All @@ -45,13 +44,14 @@ public final class GCAccounting {
private long incrementalCollectionTotalNanos = 0;
private long completeCollectionCount = 0;
private long completeCollectionTotalNanos = 0;
private UnsignedWord collectedTotalChunkBytes = WordFactory.zero();
private UnsignedWord allocatedChunkBytes = WordFactory.zero();
private UnsignedWord totalCollectedChunkBytes = WordFactory.zero();
private UnsignedWord totalAllocatedChunkBytes = WordFactory.zero();
private UnsignedWord lastIncrementalCollectionPromotedChunkBytes = WordFactory.zero();
private boolean lastIncrementalCollectionOverflowedSurvivors = false;

/* Before and after measures. */
private UnsignedWord edenChunkBytesBefore = WordFactory.zero();
private UnsignedWord edenChunkBytesAfter = WordFactory.zero();
private UnsignedWord youngChunkBytesBefore = WordFactory.zero();
private UnsignedWord youngChunkBytesAfter = WordFactory.zero();
private UnsignedWord oldChunkBytesBefore = WordFactory.zero();
Expand All @@ -61,7 +61,7 @@ public final class GCAccounting {
* Bytes allocated in Objects, as opposed to bytes of chunks. These are only maintained if
* -R:+PrintGCSummary because they are expensive.
*/
private UnsignedWord collectedTotalObjectBytes = WordFactory.zero();
private UnsignedWord totalCollectedObjectBytes = WordFactory.zero();
private UnsignedWord youngObjectBytesBefore = WordFactory.zero();
private UnsignedWord oldObjectBytesBefore = WordFactory.zero();
private UnsignedWord allocatedObjectBytes = WordFactory.zero();
Expand All @@ -78,8 +78,8 @@ public long getIncrementalCollectionTotalNanos() {
return incrementalCollectionTotalNanos;
}

UnsignedWord getAllocatedChunkBytes() {
return allocatedChunkBytes;
UnsignedWord getTotalAllocatedChunkBytes() {
return totalAllocatedChunkBytes;
}

public long getCompleteCollectionCount() {
Expand All @@ -90,12 +90,12 @@ public long getCompleteCollectionTotalNanos() {
return completeCollectionTotalNanos;
}

UnsignedWord getCollectedTotalChunkBytes() {
return collectedTotalChunkBytes;
UnsignedWord getTotalCollectedChunkBytes() {
return totalCollectedChunkBytes;
}

UnsignedWord getCollectedTotalObjectBytes() {
return collectedTotalObjectBytes;
UnsignedWord getTotalCollectedObjectBytes() {
return totalCollectedObjectBytes;
}

UnsignedWord getAllocatedObjectBytes() {
Expand All @@ -110,6 +110,10 @@ UnsignedWord getEdenChunkBytesBefore() {
return edenChunkBytesBefore;
}

UnsignedWord getEdenChunkBytesAfter() {
return edenChunkBytesAfter;
}

UnsignedWord getYoungChunkBytesBefore() {
return youngChunkBytesBefore;
}
Expand All @@ -127,30 +131,27 @@ public boolean hasLastIncrementalCollectionOverflowedSurvivors() {
}

void beforeCollection(boolean completeCollection) {
Log trace = Log.noopLog().string("[GCImpl.Accounting.beforeCollection:").newline();
/* Gather some space statistics. */
HeapImpl heap = HeapImpl.getHeapImpl();
YoungGeneration youngGen = heap.getYoungGeneration();
OldGeneration oldGen = heap.getOldGeneration();

edenChunkBytesBefore = youngGen.getEden().getChunkBytes();
youngChunkBytesBefore = youngGen.getChunkBytes();
/* This is called before the collection, so OldSpace is FromSpace. */
Space oldSpace = heap.getOldGeneration().getFromSpace();
oldChunkBytesBefore = oldSpace.getChunkBytes();
oldChunkBytesBefore = oldGen.getChunkBytes();

/* Objects are allocated in the young generation. */
allocatedChunkBytes = allocatedChunkBytes.add(youngGen.getEden().getChunkBytes());
totalAllocatedChunkBytes = totalAllocatedChunkBytes.add(youngGen.getEden().getChunkBytes());

if (SerialGCOptions.PrintGCSummary.getValue()) {
UnsignedWord edenObjectBytesBefore = youngGen.getEden().computeObjectBytes();
youngObjectBytesBefore = edenObjectBytesBefore.add(youngGen.computeSurvivorObjectBytes());
oldObjectBytesBefore = oldSpace.computeObjectBytes();
oldObjectBytesBefore = oldGen.computeObjectBytes();
allocatedObjectBytes = allocatedObjectBytes.add(edenObjectBytesBefore);
}
if (!completeCollection) {
lastIncrementalCollectionOverflowedSurvivors = false;
}
trace.string(" edenChunkBytesBefore: ").unsigned(edenChunkBytesBefore)
.string(" youngChunkBytesBefore: ").unsigned(youngChunkBytesBefore)
.string(" oldChunkBytesBefore: ").unsigned(oldChunkBytesBefore);
trace.string("]").newline();
}

/** Called after an object has been promoted from the young generation to the old generation. */
Expand All @@ -169,7 +170,6 @@ void afterCollection(boolean completeCollection, Timer collectionTimer) {
}

private void afterIncrementalCollection(Timer collectionTimer) {
Log trace = Log.noopLog().string("[GCImpl.Accounting.afterIncrementalCollection:");
/*
* Aggregating collection information is needed because any given collection policy may not
* be called for all collections, but may want to make decisions based on the aggregate
Expand All @@ -179,39 +179,35 @@ private void afterIncrementalCollection(Timer collectionTimer) {
afterCollectionCommon();
lastIncrementalCollectionPromotedChunkBytes = oldChunkBytesAfter.subtract(oldChunkBytesBefore);
incrementalCollectionTotalNanos += collectionTimer.getMeasuredNanos();
trace.string(" incrementalCollectionCount: ").signed(incrementalCollectionCount)
.string(" oldChunkBytesAfter: ").unsigned(oldChunkBytesAfter)
.string(" oldChunkBytesBefore: ").unsigned(oldChunkBytesBefore);
trace.string("]").newline();
}

private void afterCompleteCollection(Timer collectionTimer) {
Log trace = Log.noopLog().string("[GCImpl.Accounting.afterCompleteCollection:");
completeCollectionCount += 1;
afterCollectionCommon();
completeCollectionTotalNanos += collectionTimer.getMeasuredNanos();
trace.string(" completeCollectionCount: ").signed(completeCollectionCount)
.string(" oldChunkBytesAfter: ").unsigned(oldChunkBytesAfter);
trace.string("]").newline();
}

private void afterCollectionCommon() {
HeapImpl heap = HeapImpl.getHeapImpl();
// This is called after the collection, after the space flip, so OldSpace is FromSpace.
YoungGeneration youngGen = heap.getYoungGeneration();
OldGeneration oldGen = heap.getOldGeneration();

edenChunkBytesAfter = youngGen.getEden().getChunkBytes();
youngChunkBytesAfter = youngGen.getChunkBytes();
Space oldSpace = heap.getOldGeneration().getFromSpace();
oldChunkBytesAfter = oldSpace.getChunkBytes();
oldChunkBytesAfter = oldGen.getChunkBytes();

UnsignedWord beforeChunkBytes = youngChunkBytesBefore.add(oldChunkBytesBefore);
UnsignedWord afterChunkBytes = oldChunkBytesAfter.add(youngChunkBytesAfter);
UnsignedWord afterChunkBytes = youngChunkBytesAfter.add(oldChunkBytesAfter);
assert beforeChunkBytes.aboveOrEqual(afterChunkBytes);
UnsignedWord collectedChunkBytes = beforeChunkBytes.subtract(afterChunkBytes);
collectedTotalChunkBytes = collectedTotalChunkBytes.add(collectedChunkBytes);
totalCollectedChunkBytes = totalCollectedChunkBytes.add(collectedChunkBytes);

if (SerialGCOptions.PrintGCSummary.getValue()) {
UnsignedWord youngObjectBytesAfter = youngGen.computeObjectBytes();
UnsignedWord oldObjectBytesAfter = oldSpace.computeObjectBytes();
UnsignedWord oldObjectBytesAfter = oldGen.computeObjectBytes();
UnsignedWord beforeObjectBytes = youngObjectBytesBefore.add(oldObjectBytesBefore);
UnsignedWord collectedObjectBytes = beforeObjectBytes.subtract(oldObjectBytesAfter).subtract(youngObjectBytesAfter);
collectedTotalObjectBytes = collectedTotalObjectBytes.add(collectedObjectBytes);
totalCollectedObjectBytes = totalCollectedObjectBytes.add(collectedObjectBytes);
}
}
}
71 changes: 40 additions & 31 deletions ...atevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,6 @@ public final class GCImpl implements GC {
private final Timers timers = new Timers();

private final CollectionVMOperation collectOperation = new CollectionVMOperation();
private final NoAllocationVerifier noAllocationVerifier = NoAllocationVerifier.factory("GCImpl.GCImpl()", false);
private final ChunkReleaser chunkReleaser = new ChunkReleaser();

private final CollectionPolicy policy;
Expand Down Expand Up @@ -207,9 +206,11 @@ private void collectOperation(CollectionVMOperationData data) {

GCCause cause = GCCause.fromId(data.getCauseId());
printGCBefore(cause.getName());
JfrGCHeapSummaryEvent.emit(getCollectionEpoch(), JfrTicks.elapsedTicks(), getPolicy().getCurrentHeapCapacity().rawValue(), getChunkBytes().rawValue(), JfrGCWhen.BEFORE_GC);

JfrGCHeapSummaryEvent.emit(JfrGCWhen.BEFORE_GC);
boolean outOfMemory = collectImpl(cause, data.getRequestingNanoTime(), data.getForceFullGC());
JfrGCHeapSummaryEvent.emit(getCollectionEpoch(), JfrTicks.elapsedTicks(), getPolicy().getCurrentHeapCapacity().rawValue(), getChunkBytes().rawValue(), JfrGCWhen.AFTER_GC);
JfrGCHeapSummaryEvent.emit(JfrGCWhen.AFTER_GC);

printGCAfter(cause.getName());

finishCollection();
Expand All @@ -219,36 +220,34 @@ private void collectOperation(CollectionVMOperationData data) {
}

private boolean collectImpl(GCCause cause, long requestingNanoTime, boolean forceFullGC) {
boolean outOfMemory;
precondition();
HeapImpl.getHeapImpl().getAccounting().notifyBeforeCollection();

NoAllocationVerifier nav = noAllocationVerifier.open();
boolean outOfMemory;
long startTicks = JfrTicks.elapsedTicks();
try {
long startTicks = JfrTicks.elapsedTicks();
try {
outOfMemory = doCollectImpl(cause, requestingNanoTime, forceFullGC, false);
if (outOfMemory) {
// Avoid running out of memory with a full GC that reclaims softly reachable
// objects
ReferenceObjectProcessing.setSoftReferencesAreWeak(true);
try {
outOfMemory = doCollectImpl(cause, requestingNanoTime, true, true);
} finally {
ReferenceObjectProcessing.setSoftReferencesAreWeak(false);
}
outOfMemory = doCollectImpl(cause, requestingNanoTime, forceFullGC, false);
if (outOfMemory) {
// Avoid running out of memory with a full GC that reclaims softly reachable
// objects
ReferenceObjectProcessing.setSoftReferencesAreWeak(true);
try {
outOfMemory = doCollectImpl(cause, requestingNanoTime, true, true);
} finally {
ReferenceObjectProcessing.setSoftReferencesAreWeak(false);
}
} finally {
JfrGCEvents.emitGarbageCollectionEvent(getCollectionEpoch(), cause, startTicks);
}
} finally {
nav.close();
JfrGCEvents.emitGarbageCollectionEvent(getCollectionEpoch(), cause, startTicks);
}

postcondition();
HeapImpl.getHeapImpl().getAccounting().notifyAfterCollection(this.accounting);
GenScavengeMemoryPoolMXBeans.notifyAfterCollection(this.accounting);
return outOfMemory;
}

private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean forceFullGC, boolean forceNoIncremental) {
precondition();

CommittedMemoryProvider.get().beforeGarbageCollection();

boolean incremental = !forceNoIncremental && !policy.shouldCollectCompletely(false);
Expand Down Expand Up @@ -276,6 +275,8 @@ private boolean doCollectImpl(GCCause cause, long requestingNanoTime, boolean fo

HeapImpl.getChunkProvider().freeExcessAlignedChunks();
CommittedMemoryProvider.get().afterGarbageCollection();

postcondition();
return outOfMemory;
}

Expand All @@ -300,12 +301,9 @@ private boolean doCollectOnce(GCCause cause, long requestingNanoTime, boolean co
collectionTimer.close();
}

HeapImpl.getHeapImpl().getAccounting().setEdenAndYoungGenBytes(WordFactory.zero(), accounting.getYoungChunkBytesAfter());
accounting.afterCollection(completeCollection, collectionTimer);
policy.onCollectionEnd(completeCollection, cause);

GenScavengeMemoryPoolMXBeans.notifyAfterCollection(accounting);

UnsignedWord usedBytes = getChunkBytes();
UnsignedWord freeBytes = policy.getCurrentHeapCapacity().subtract(usedBytes);
ReferenceObjectProcessing.afterCollection(freeBytes);
Expand Down Expand Up @@ -1269,6 +1267,8 @@ private CollectionInProgressError() {
}

private static class CollectionVMOperation extends NativeVMOperation {
private final NoAllocationVerifier noAllocationVerifier = NoAllocationVerifier.factory("CollectionVMOperation", false);

CollectionVMOperation() {
super(VMOperationInfos.get(CollectionVMOperation.class, "Garbage collection", SystemEffect.SAFEPOINT));
}
Expand All @@ -1282,6 +1282,17 @@ public boolean isGC() {
@Override
@RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Must not allocate while collecting")
protected void operate(NativeVMOperationData data) {
NoAllocationVerifier nav = noAllocationVerifier.open();
try {
collect((CollectionVMOperationData) data);
} catch (Throwable t) {
throw VMError.shouldNotReachHere(t);
} finally {
nav.close();
}
}

private static void collect(CollectionVMOperationData data) {
/*
* Exceptions during collections are fatal. The heap is likely in an inconsistent state.
* The GC must also be allocation free, i.e., we cannot allocate exception stack traces
Expand All @@ -1292,9 +1303,7 @@ protected void operate(NativeVMOperationData data) {
*/
ImplicitExceptions.activateImplicitExceptionsAreFatal();
try {
HeapImpl.getGCImpl().collectOperation((CollectionVMOperationData) data);
} catch (Throwable t) {
throw VMError.shouldNotReachHere(t);
HeapImpl.getGCImpl().collectOperation(data);
} finally {
ImplicitExceptions.deactivateImplicitExceptionsAreFatal();
}
Expand Down Expand Up @@ -1422,11 +1431,11 @@ private void printGCSummary() {
UnsignedWord youngChunkBytes = edenSpace.getChunkBytes();
UnsignedWord youngObjectBytes = edenSpace.computeObjectBytes();

UnsignedWord allocatedChunkBytes = accounting.getAllocatedChunkBytes().add(youngChunkBytes);
UnsignedWord allocatedChunkBytes = accounting.getTotalAllocatedChunkBytes().add(youngChunkBytes);
UnsignedWord allocatedObjectBytes = accounting.getAllocatedObjectBytes().add(youngObjectBytes);

log.string(prefix).string("CollectedTotalChunkBytes: ").signed(accounting.getCollectedTotalChunkBytes()).newline();
log.string(prefix).string("CollectedTotalObjectBytes: ").signed(accounting.getCollectedTotalObjectBytes()).newline();
log.string(prefix).string("CollectedTotalChunkBytes: ").signed(accounting.getTotalCollectedChunkBytes()).newline();
log.string(prefix).string("CollectedTotalObjectBytes: ").signed(accounting.getTotalCollectedObjectBytes()).newline();
log.string(prefix).string("AllocatedNormalChunkBytes: ").signed(allocatedChunkBytes).newline();
log.string(prefix).string("AllocatedNormalObjectBytes: ").signed(allocatedObjectBytes).newline();

Expand Down
21 changes: 16 additions & 5 deletions ...c/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapAccounting.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,25 @@ public final class HeapAccounting {
private final UninterruptibleUtils.AtomicUnsigned edenUsedBytes = new UninterruptibleUtils.AtomicUnsigned();
private final UninterruptibleUtils.AtomicUnsigned youngUsedBytes = new UninterruptibleUtils.AtomicUnsigned();

/* During a GC, the values are invalid. They are updated once the GC ends. */
private boolean invalidData;

@Platforms(Platform.HOSTED_ONLY.class)
HeapAccounting() {
}

public void setEdenAndYoungGenBytes(UnsignedWord edenBytes, UnsignedWord youngBytes) {
public void notifyBeforeCollection() {
assert VMOperation.isGCInProgress();
invalidData = true;
}

public void notifyAfterCollection(GCAccounting accounting) {
assert VMOperation.isGCInProgress() : "would cause races otherwise";
youngUsedBytes.set(youngBytes);
edenUsedBytes.set(edenBytes);
assert invalidData;

youngUsedBytes.set(accounting.getYoungChunkBytesAfter());
edenUsedBytes.set(accounting.getEdenChunkBytesAfter());
invalidData = false;
}

@Uninterruptible(reason = "Must be done during TLAB registration to not race with a potential collection.", callerMustBe = true)
Expand All @@ -58,13 +69,13 @@ public void increaseEdenUsedBytes(UnsignedWord value) {

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public UnsignedWord getYoungUsedBytes() {
assert !VMOperation.isGCInProgress() : "value is incorrect during a GC";
assert !invalidData : "value is incorrect during a GC";
return youngUsedBytes.get();
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public UnsignedWord getEdenUsedBytes() {
assert !VMOperation.isGCInProgress() : "value is incorrect during a GC";
assert !invalidData : "value is incorrect during a GC";
return edenUsedBytes.get();
}

Expand Down
Loading

0 comments on commit 9d65db1

Please sign in to comment.