diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java index f0e13680c71e..6e496309559a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/graal/BarrierSnippets.java @@ -34,6 +34,7 @@ import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.debug.DebugHandlersFactory; import org.graalvm.compiler.graph.Node; +import org.graalvm.compiler.nodes.BreakpointNode; import org.graalvm.compiler.nodes.NamedLocationIdentity; import org.graalvm.compiler.nodes.extended.BranchProbabilityNode; import org.graalvm.compiler.nodes.extended.FixedValueAnchorNode; @@ -42,6 +43,7 @@ import org.graalvm.compiler.nodes.gc.WriteBarrier; import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode; import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.type.StampTool; import org.graalvm.compiler.options.Option; import org.graalvm.compiler.options.OptionValues; import org.graalvm.compiler.phases.util.Providers; @@ -64,6 +66,8 @@ import com.oracle.svm.core.util.Counter; import com.oracle.svm.core.util.CounterFeature; +import jdk.vm.ci.meta.ResolvedJavaType; + public class BarrierSnippets extends SubstrateTemplates implements Snippets { /** A LocationIdentity to distinguish card locations from other locations. */ public static final LocationIdentity CARD_REMEMBERED_SET_LOCATION = NamedLocationIdentity.mutable("CardRememberedSet"); @@ -71,6 +75,9 @@ public class BarrierSnippets extends SubstrateTemplates implements Snippets { public static class Options { @Option(help = "Instrument write barriers with counters")// public static final HostedOptionKey CountWriteBarriers = new HostedOptionKey<>(false); + + @Option(help = "Verify write barriers")// + public static final HostedOptionKey VerifyWriteBarriers = new HostedOptionKey<>(false); } @Fold @@ -90,23 +97,40 @@ public void registerLowerings(Map, NodeLoweringProvider } @Snippet - public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean verifyOnly) { + public static void postWriteBarrierSnippet(Object object, @ConstantParameter boolean alwaysAlignedChunk, @ConstantParameter boolean verifyOnly) { counters().postWriteBarrier.inc(); Object fixedObject = FixedValueAnchorNode.getObject(object); UnsignedWord objectHeader = ObjectHeaderImpl.readHeaderFromObject(fixedObject); + + if (Options.VerifyWriteBarriers.getValue() && alwaysAlignedChunk) { + /* + * To increase verification coverage, we do the verification before checking if a + * barrier is needed at all. And in addition to verifying that the object is in an + * aligned chunk, we also verify that it is not an array at all because most arrays are + * small and therefore in an aligned chunk. + */ + if (ObjectHeaderImpl.isUnalignedHeader(objectHeader) || object == null || object.getClass().isArray()) { + BreakpointNode.breakpoint(); + } + } + boolean needsBarrier = RememberedSet.get().hasRememberedSet(objectHeader); if (BranchProbabilityNode.probability(BranchProbabilityNode.FREQUENT_PROBABILITY, !needsBarrier)) { return; } - boolean aligned = ObjectHeaderImpl.isAlignedHeader(objectHeader); - if (BranchProbabilityNode.probability(BranchProbabilityNode.LIKELY_PROBABILITY, aligned)) { - counters().postWriteBarrierAligned.inc(); - RememberedSet.get().dirtyCardForAlignedObject(fixedObject, verifyOnly); - return; + + if (!alwaysAlignedChunk) { + boolean unaligned = ObjectHeaderImpl.isUnalignedHeader(objectHeader); + if (BranchProbabilityNode.probability(BranchProbabilityNode.NOT_LIKELY_PROBABILITY, unaligned)) { + counters().postWriteBarrierUnaligned.inc(); + RememberedSet.get().dirtyCardForUnalignedObject(fixedObject, verifyOnly); + return; + } } - counters().postWriteBarrierUnaligned.inc(); - RememberedSet.get().dirtyCardForUnalignedObject(fixedObject, verifyOnly); + + counters().postWriteBarrierAligned.inc(); + RememberedSet.get().dirtyCardForAlignedObject(fixedObject, verifyOnly); } private class PostWriteBarrierLowering implements NodeLoweringProvider { @@ -116,7 +140,20 @@ private class PostWriteBarrierLowering implements NodeLoweringProvider