[GR-32940] [GR-31997] Use image heap remembered set during complete c…

…ollections.

PullRequest: graal/9507

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java

@@ -810,12 +810,12 @@ private void blackenDirtyImageHeapRoots() {
         Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open();
         try {
             ImageHeapInfo info = HeapImpl.getImageHeapInfo();
-            blackenDirtyImageHeapChunkRoots(info.getFirstAlignedImageHeapChunk(), info.getFirstUnalignedImageHeapChunk());
+            blackenDirtyImageHeapChunkRoots(info.getFirstWritableAlignedChunk(), info.getFirstWritableUnalignedChunk());
 
             if (AuxiliaryImageHeap.isPresent()) {
                 ImageHeapInfo auxInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo();
                 if (auxInfo != null) {
-                    blackenDirtyImageHeapChunkRoots(auxInfo.getFirstAlignedImageHeapChunk(), auxInfo.getFirstUnalignedImageHeapChunk());
+                    blackenDirtyImageHeapChunkRoots(auxInfo.getFirstWritableAlignedChunk(), auxInfo.getFirstWritableUnalignedChunk());
                 }
             }
         } finally {
@@ -824,20 +824,34 @@ private void blackenDirtyImageHeapRoots() {
     }
 
     private void blackenDirtyImageHeapChunkRoots(AlignedHeader firstAligned, UnalignedHeader firstUnaligned) {
+        /*
+         * We clean and remark cards of the image heap only during complete collections when we also
+         * collect the old generation and can easily remark references into it. It also only makes a
+         * difference after references to the runtime heap were nulled, which is assumed to be rare.
+         */
+        boolean clean = completeCollection;
+
         AlignedHeader aligned = firstAligned;
         while (aligned.isNonNull()) {
-            RememberedSet.get().walkDirtyObjects(aligned, greyToBlackObjectVisitor);
+            RememberedSet.get().walkDirtyObjects(aligned, greyToBlackObjectVisitor, clean);
             aligned = HeapChunk.getNext(aligned);
         }
 
         UnalignedHeader unaligned = firstUnaligned;
         while (unaligned.isNonNull()) {
-            RememberedSet.get().walkDirtyObjects(unaligned, greyToBlackObjectVisitor);
+            RememberedSet.get().walkDirtyObjects(unaligned, greyToBlackObjectVisitor, clean);
             unaligned = HeapChunk.getNext(unaligned);
         }
     }
 
     private void blackenImageHeapRoots() {
+        if (HeapImpl.usesImageHeapCardMarking()) {
+            // Avoid scanning the entire image heap even for complete collections: its remembered
+            // set contains references into both the runtime heap's old and young generations.
+            blackenDirtyImageHeapRoots();
+            return;
+        }
+
         Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open();
         try {
             HeapImpl.getHeapImpl().walkNativeImageHeapRegions(blackenImageHeapRootsVisitor);
@@ -864,7 +878,7 @@ private void blackenDirtyCardRoots() {
              * Promote any referenced young objects.
              */
             Space oldGenToSpace = HeapImpl.getHeapImpl().getOldGeneration().getToSpace();
-            RememberedSet.get().walkDirtyObjects(oldGenToSpace, greyToBlackObjectVisitor);
+            RememberedSet.get().walkDirtyObjects(oldGenToSpace, greyToBlackObjectVisitor, true);
         } finally {
             blackenDirtyCardRootsTimer.close();
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapPolicy.java

@@ -86,7 +86,7 @@ public static UnsignedWord m(long bytes) {
         return WordFactory.unsigned(bytes).multiply(1024).multiply(1024);
     }
 
-    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @Fold
     public static int getMaxSurvivorSpaces() {
         return HeapPolicyOptions.MaxSurvivorSpaces.getValue();
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapVerifier.java

@@ -77,8 +77,8 @@ private static boolean verifyImageHeapObjects() {
     private static boolean verifyChunkedImageHeap() {
         boolean success = true;
         ImageHeapInfo info = HeapImpl.getImageHeapInfo();
-        success &= verifyAlignedChunks(null, info.getFirstAlignedImageHeapChunk());
-        success &= verifyUnalignedChunks(null, info.getFirstUnalignedImageHeapChunk());
+        success &= verifyAlignedChunks(null, info.getFirstWritableAlignedChunk());
+        success &= verifyUnalignedChunks(null, info.getFirstWritableUnalignedChunk());
         return success;
     }
 
@@ -161,8 +161,8 @@ private static boolean verifyRememberedSets() {
              * GC itself may result in dirty cards.
              */
             ImageHeapInfo info = HeapImpl.getImageHeapInfo();
-            success &= rememberedSet.verify(info.getFirstAlignedImageHeapChunk());
-            success &= rememberedSet.verify(info.getFirstUnalignedImageHeapChunk());
+            success &= rememberedSet.verify(info.getFirstWritableAlignedChunk());
+            success &= rememberedSet.verify(info.getFirstWritableUnalignedChunk());
         }
 
         OldGeneration oldGeneration = HeapImpl.getHeapImpl().getOldGeneration();

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java

@@ -68,8 +68,8 @@ public final class ImageHeapInfo {
     @UnknownObjectField(types = Object.class) public Object firstObject;
     @UnknownObjectField(types = Object.class) public Object lastObject;
 
-    @UnknownPrimitiveField public long offsetOfFirstAlignedChunkWithRememberedSet;
-    @UnknownPrimitiveField public long offsetOfFirstUnalignedChunkWithRememberedSet;
+    @UnknownPrimitiveField public long offsetOfFirstWritableAlignedChunk;
+    @UnknownPrimitiveField public long offsetOfFirstWritableUnalignedChunk;
 
     @UnknownPrimitiveField public int dynamicHubCount;
 
@@ -80,10 +80,10 @@ public ImageHeapInfo() {
     public void initialize(Object firstReadOnlyPrimitiveObject, Object lastReadOnlyPrimitiveObject, Object firstReadOnlyReferenceObject, Object lastReadOnlyReferenceObject,
                     Object firstReadOnlyRelocatableObject, Object lastReadOnlyRelocatableObject, Object firstWritablePrimitiveObject, Object lastWritablePrimitiveObject,
                     Object firstWritableReferenceObject, Object lastWritableReferenceObject, Object firstWritableHugeObject, Object lastWritableHugeObject,
-                    Object firstReadOnlyHugeObject, Object lastReadOnlyHugeObject, long offsetOfFirstAlignedChunkWithRememberedSet, long offsetOfFirstUnalignedChunkWithRememberedSet,
+                    Object firstReadOnlyHugeObject, Object lastReadOnlyHugeObject, long offsetOfFirstWritableAlignedChunk, long offsetOfFirstWritableUnalignedChunk,
                     int dynamicHubCount) {
-        assert offsetOfFirstAlignedChunkWithRememberedSet == NO_CHUNK || offsetOfFirstAlignedChunkWithRememberedSet >= 0;
-        assert offsetOfFirstUnalignedChunkWithRememberedSet == NO_CHUNK || offsetOfFirstUnalignedChunkWithRememberedSet >= 0;
+        assert offsetOfFirstWritableAlignedChunk == NO_CHUNK || offsetOfFirstWritableAlignedChunk >= 0;
+        assert offsetOfFirstWritableUnalignedChunk == NO_CHUNK || offsetOfFirstWritableUnalignedChunk >= 0;
 
         this.firstReadOnlyPrimitiveObject = firstReadOnlyPrimitiveObject;
         this.lastReadOnlyPrimitiveObject = lastReadOnlyPrimitiveObject;
@@ -99,8 +99,8 @@ public void initialize(Object firstReadOnlyPrimitiveObject, Object lastReadOnlyP
         this.lastWritableHugeObject = lastWritableHugeObject;
         this.firstReadOnlyHugeObject = firstReadOnlyHugeObject;
         this.lastReadOnlyHugeObject = lastReadOnlyHugeObject;
-        this.offsetOfFirstAlignedChunkWithRememberedSet = offsetOfFirstAlignedChunkWithRememberedSet;
-        this.offsetOfFirstUnalignedChunkWithRememberedSet = offsetOfFirstUnalignedChunkWithRememberedSet;
+        this.offsetOfFirstWritableAlignedChunk = offsetOfFirstWritableAlignedChunk;
+        this.offsetOfFirstWritableUnalignedChunk = offsetOfFirstWritableUnalignedChunk;
         this.dynamicHubCount = dynamicHubCount;
 
         // Compute boundaries for checks considering partitions can be empty (first == last == null)
@@ -184,12 +184,12 @@ public boolean isInImageHeap(Pointer objectPointer) {
         return result;
     }
 
-    public AlignedHeader getFirstAlignedImageHeapChunk() {
-        return asImageHeapChunk(offsetOfFirstAlignedChunkWithRememberedSet);
+    public AlignedHeader getFirstWritableAlignedChunk() {
+        return asImageHeapChunk(offsetOfFirstWritableAlignedChunk);
     }
 
-    public UnalignedHeader getFirstUnalignedImageHeapChunk() {
-        return asImageHeapChunk(offsetOfFirstUnalignedChunkWithRememberedSet);
+    public UnalignedHeader getFirstWritableUnalignedChunk() {
+        return asImageHeapChunk(offsetOfFirstWritableUnalignedChunk);
     }
 
     @SuppressWarnings("unchecked")

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java

@@ -244,9 +244,17 @@ private UnsignedWord computeSurvivorObjectBytes() {
         return usedObjectBytes;
     }
 
+    @AlwaysInline("GC performance")
     @SuppressWarnings("static-method")
     public boolean contains(Object object) {
-        return HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
+        if (!HeapImpl.usesImageHeapCardMarking()) {
+            return HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
+        }
+        // Only objects in the young generation have no remembered set
+        UnsignedWord header = ObjectHeaderImpl.readHeaderFromObject(object);
+        boolean young = !ObjectHeaderImpl.hasRememberedSet(header);
+        assert young == HeapChunk.getSpace(HeapChunk.getEnclosingHeapChunk(object)).isYoungSpace();
+        return young;
     }
 
     @AlwaysInline("GC performance")

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java

@@ -125,7 +125,7 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) {
         }
     }
 
-    public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor) {
+    public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
         Pointer cardTableStart = getCardTableStart(chunk);
         Pointer fotStart = getFirstObjectTableStart(chunk);
         Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
@@ -135,7 +135,9 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito
 
         for (UnsignedWord index = WordFactory.zero(); index.belowThan(indexLimit); index = index.add(1)) {
             if (CardTable.isDirty(cardTableStart, index)) {
-                CardTable.setClean(cardTableStart, index);
+                if (clean) {
+                    CardTable.setClean(cardTableStart, index);
+                }
 
                 Pointer ptr = FirstObjectTable.getFirstObjectImprecise(fotStart, objectsStart, objectsLimit, index);
                 Pointer cardLimit = CardTable.indexToMemoryPointer(objectsStart, index.add(1));

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java

@@ -35,7 +35,6 @@
 import com.oracle.svm.core.UnmanagedMemoryUtil;
 import com.oracle.svm.core.genscavenge.HeapChunk;
 import com.oracle.svm.core.genscavenge.HeapImpl;
-import com.oracle.svm.core.genscavenge.Space;
 import com.oracle.svm.core.genscavenge.graal.BarrierSnippets;
 import com.oracle.svm.core.heap.ObjectReferenceVisitor;
 import com.oracle.svm.core.heap.ReferenceAccess;
@@ -167,14 +166,18 @@ private static boolean verifyReference(Object parentObject, Pointer cardTableSta
         if (referencedObject.isNonNull() && !HeapImpl.getHeapImpl().isInImageHeap(referencedObject)) {
             Object obj = referencedObject.toObject();
             HeapChunk.Header<?> objChunk = HeapChunk.getEnclosingHeapChunk(obj);
-            Space chunkSpace = HeapChunk.getSpace(objChunk);
-            // Fail if we find a reference to the young generation.
-            if (chunkSpace.isYoungSpace()) {
+            // Fail if we find a reference from the image heap to the runtime heap, or from the
+            // old generation (which is the only one with remembered sets) to the young generation.
+            boolean fromImageHeap = HeapImpl.usesImageHeapCardMarking() && HeapImpl.getHeapImpl().isInImageHeap(parentObject);
+            if (fromImageHeap || HeapChunk.getSpace(objChunk).isYoungSpace()) {
                 UnsignedWord cardTableIndex = memoryOffsetToIndex(Word.objectToUntrackedPointer(parentObject).subtract(objectsStart));
                 Pointer cardTableAddress = cardTableStart.add(indexToTableOffset(cardTableIndex));
-                Log.log().string("Object ").hex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).string(") has an object reference at ")
-                                .hex(reference).string(" that points to ").hex(referencedObject).string(" (").string(obj.getClass().getName())
-                                .string("), which is in the young generation. However, the card table at ").hex(cardTableAddress).string(" is clean.").newline();
+                Log.log().string("Object ").hex(Word.objectToUntrackedPointer(parentObject)).string(" (").string(parentObject.getClass().getName()).character(')')
+                                .string(fromImageHeap ? ", which is in the image heap, " : " ")
+                                .string("has an object reference at ")
+                                .hex(reference).string(" that points to ").hex(referencedObject).string(" (").string(obj.getClass().getName()).string("), ")
+                                .string("which is in the ").string(fromImageHeap ? "runtime heap" : "young generation").string(". ")
+                                .string("However, the card table at ").hex(cardTableAddress).string(" is clean.").newline();
                 return false;
             }
         }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java

@@ -131,8 +131,22 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) {
     @Override
     @AlwaysInline("GC performance")
     public void dirtyCardIfNecessary(Object holderObject, Object object) {
-        if (HeapPolicy.getMaxSurvivorSpaces() == 0 || holderObject == null || object == null || GCImpl.getGCImpl().isCompleteCollection() ||
-                        !HeapImpl.getHeapImpl().getYoungGeneration().contains(object)) {
+        if (holderObject == null || object == null) {
+            return;
+        }
+        // We dirty the cards of ...
+        if (HeapPolicy.getMaxSurvivorSpaces() != 0 && !GCImpl.getGCImpl().isCompleteCollection() && HeapImpl.getHeapImpl().getYoungGeneration().contains(object)) {
+            /*
+             * ...references from the old generation to the young generation, unless there cannot be
+             * any such references if we do not use survivor spaces, or if we do but are doing a
+             * complete collection: in both cases, all objects are promoted to the old generation.
+             * (We avoid an extra old generation check and might remark a few image heap cards, too)
+             */
+        } else if (HeapImpl.usesImageHeapCardMarking() && GCImpl.getGCImpl().isCompleteCollection() && HeapImpl.getHeapImpl().isInImageHeap(holderObject)) {
+            // ...references from the image heap to the runtime heap, but we clean and remark those
+            // only during complete collections.
+            assert !HeapImpl.getHeapImpl().isInImageHeap(object) : "should never be called for references to image heap objects";
+        } else {
             return;
         }
 
@@ -148,26 +162,26 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) {
     }
 
     @Override
-    public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor) {
-        AlignedChunkRememberedSet.walkDirtyObjects(chunk, visitor);
+    public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
+        AlignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean);
     }
 
     @Override
-    public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor) {
-        UnalignedChunkRememberedSet.walkDirtyObjects(chunk, visitor);
+    public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
+        UnalignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean);
     }
 
     @Override
-    public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor) {
+    public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) {
         AlignedHeader aChunk = space.getFirstAlignedHeapChunk();
         while (aChunk.isNonNull()) {
-            walkDirtyObjects(aChunk, visitor);
+            walkDirtyObjects(aChunk, visitor, clean);
             aChunk = HeapChunk.getNext(aChunk);
         }
 
         UnalignedHeader uChunk = space.getFirstUnalignedHeapChunk();
         while (uChunk.isNonNull()) {
-            walkDirtyObjects(uChunk, visitor);
+            walkDirtyObjects(uChunk, visitor, clean);
             uChunk = HeapChunk.getNext(uChunk);
         }
     }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java

@@ -132,17 +132,17 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) {
     }
 
     @Override
-    public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor) {
+    public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
         throw VMError.shouldNotReachHere();
     }
 
     @Override
-    public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor) {
+    public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) {
         throw VMError.shouldNotReachHere();
     }
 
     @Override
-    public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor) {
+    public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) {
         throw VMError.shouldNotReachHere();
     }