Skip to content

Commit

Permalink
[GR-37036] [GR-35200] Virtual thread improvements.
Browse files Browse the repository at this point in the history
PullRequest: graal/11117
  • Loading branch information
peter-hofer committed Mar 20, 2022
2 parents 424847d + 06d7305 commit 8dccf24
Show file tree
Hide file tree
Showing 36 changed files with 608 additions and 238 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -909,7 +909,7 @@ private void walkStack(JavaStackWalk walk) {
if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) {
throw CodeInfoTable.reportNoReferenceMap(sp, ip, codeInfo);
}
CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, greyToBlackObjRefVisitor);
CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, greyToBlackObjRefVisitor, null);
} else {
/*
* This is a deoptimized frame. The DeoptimizedFrame object is stored in the frame,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.graalvm.compiler.core.common.NumUtil;
import org.graalvm.compiler.core.common.SuppressFBWarnings;
import org.graalvm.compiler.nodes.gc.BarrierSet;
import org.graalvm.compiler.nodes.memory.address.OffsetAddressNode;
import org.graalvm.compiler.word.Word;
import org.graalvm.nativeimage.CurrentIsolate;
import org.graalvm.nativeimage.IsolateThread;
Expand All @@ -56,6 +57,7 @@
import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader;
import com.oracle.svm.core.genscavenge.ThreadLocalAllocation.Descriptor;
import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader;
import com.oracle.svm.core.genscavenge.graal.ForcedSerialPostWriteBarrier;
import com.oracle.svm.core.genscavenge.remset.RememberedSet;
import com.oracle.svm.core.graal.snippets.SubstrateAllocationSnippets;
import com.oracle.svm.core.heap.GC;
Expand Down Expand Up @@ -684,6 +686,14 @@ public long getThreadAllocatedMemory(IsolateThread thread) {
return allocatedBytes.rawValue();
}

@Override
@Uninterruptible(reason = "Ensure that no GC can occur between modification of the object and this call.", callerMustBe = true)
public void dirtyAllReferencesOf(Object obj) {
if (obj != null) {
ForcedSerialPostWriteBarrier.force(OffsetAddressNode.address(obj, 0), false);
}
}

static Pointer getImageHeapStart() {
int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace();
if (imageHeapOffsetInAddressSpace > 0) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,8 +146,8 @@ public void lower(WriteBarrier barrier, LoweringTool tool) {
OffsetAddressNode address = (OffsetAddressNode) barrier.getAddress();

/*
* We know that instances (in contrast to arrays) are always in aligned chunks. There is
* no code anywhere that would allocate an instance into an unaligned chunk.
* We know that instances (in contrast to arrays) are always in aligned chunks, except
* for StoredContinuation objects, but these are immutable and do not need barriers.
*
* Note that arrays can be assigned to values that have the type java.lang.Object, so
* that case is excluded. Arrays can also implement some interfaces, like Serializable.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/*
* Copyright (c) 2022, 2022, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package com.oracle.svm.core.genscavenge.graal;

import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_8;
import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_4;

import org.graalvm.compiler.core.common.type.StampFactory;
import org.graalvm.compiler.graph.NodeClass;
import org.graalvm.compiler.nodeinfo.InputType;
import org.graalvm.compiler.nodeinfo.NodeInfo;
import org.graalvm.compiler.nodes.FixedWithNextNode;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.gc.BarrierSet;
import org.graalvm.compiler.nodes.gc.SerialWriteBarrier;
import org.graalvm.compiler.nodes.memory.address.AddressNode;
import org.graalvm.compiler.nodes.memory.address.AddressNode.Address;
import org.graalvm.compiler.nodes.spi.Lowerable;
import org.graalvm.compiler.nodes.spi.LoweringTool;

/**
* Post-write barrier that is injected manually and not via {@link BarrierSet}. This needs to be a
* separate node because barrier nodes may be introduced only at a certain stage of compilation.
*/
@NodeInfo(cycles = CYCLES_8, size = SIZE_4)
public final class ForcedSerialPostWriteBarrier extends FixedWithNextNode implements Lowerable {
public static final NodeClass<ForcedSerialPostWriteBarrier> TYPE = NodeClass.create(ForcedSerialPostWriteBarrier.class);

@Input(InputType.Association) AddressNode address;

private final boolean precise;

public ForcedSerialPostWriteBarrier(ValueNode address, boolean precise) {
super(TYPE, StampFactory.forVoid());
this.address = (AddressNode) address;
this.precise = precise;
}

@Override
public void lower(LoweringTool tool) {
if (graph().getGuardsStage().areFrameStatesAtDeopts()) {
SerialWriteBarrier barrier = graph().add(new SerialWriteBarrier(address, precise));
graph().replaceFixedWithFixed(this, barrier);
tool.getLowerer().lower(barrier, tool);
}
}

@NodeIntrinsic
public static native void force(Address obj, @ConstantNodeParameter boolean precise);
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,6 @@
*/
package com.oracle.svm.core.genscavenge.graal;

import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.SLOW_PATH_PROBABILITY;
import static org.graalvm.compiler.nodes.extended.BranchProbabilityNode.probability;

import java.util.Map;

import org.graalvm.compiler.api.replacements.Snippet;
Expand All @@ -43,8 +40,10 @@
import org.graalvm.compiler.replacements.SnippetTemplate;
import org.graalvm.compiler.replacements.SnippetTemplate.Arguments;
import org.graalvm.compiler.replacements.SnippetTemplate.SnippetInfo;
import org.graalvm.compiler.word.ObjectAccess;
import org.graalvm.compiler.word.Word;
import org.graalvm.nativeimage.ImageSingletons;
import org.graalvm.word.LocationIdentity;
import org.graalvm.word.UnsignedWord;

import com.oracle.svm.core.genscavenge.HeapParameters;
Expand All @@ -66,7 +65,6 @@
import com.oracle.svm.core.meta.SubstrateObjectConstant;
import com.oracle.svm.core.snippets.SnippetRuntime;
import com.oracle.svm.core.snippets.SnippetRuntime.SubstrateForeignCallDescriptor;
import com.oracle.svm.core.thread.Continuation;

final class GenScavengeAllocationSnippets extends SubstrateAllocationSnippets {
private static final SubstrateForeignCallDescriptor SLOW_NEW_INSTANCE = SnippetRuntime.findForeignCall(ThreadLocalAllocation.class, "slowPathNewInstance", true);
Expand Down Expand Up @@ -105,9 +103,6 @@ public Object formatArraySnippet(Word memory, DynamicHub hub, int length, boolea
Word objectHeader = encodeAsObjectHeader(hubNonNull, rememberedSet, unaligned);
Object obj = formatArray(objectHeader, size, length, memory, fillContents, fillStartOffset,
false, false, supportsBulkZeroing, supportsOptimizedFilling, snippetCounters);
if (probability(SLOW_PATH_PROBABILITY, Continuation.isSupported() && hub == DynamicHub.fromClass(StoredContinuation.class))) {
finishFormatStoredContinuation(obj);
}
emitMemoryBarrierIf(emitMemoryBarrier);
return obj;
}
Expand All @@ -117,19 +112,29 @@ private static Word encodeAsObjectHeader(DynamicHub hub, boolean rememberedSet,
}

@Snippet
public Object allocateStoredContinuationInstance(@Snippet.NonNullParameter DynamicHub hub, int size, @ConstantParameter AllocationProfilingData profilingData) {
int baseOffset = StoredContinuationImpl.PAYLOAD_OFFSET;
Object result = allocateArrayImpl(encodeAsTLABObjectHeader(hub), size, baseOffset, 0,
FillContent.WITH_GARBAGE_IF_ASSERTIONS_ENABLED, baseOffset, false, false, false, false, profilingData);
finishFormatStoredContinuation(result);
public Object allocateStoredContinuationInstance(@ConstantParameter DynamicHub hub, int payloadSize, @ConstantParameter DynamicHub byteArrayHub,
@ConstantParameter int byteArrayBaseOffset, @ConstantParameter AllocationProfilingData profilingData) {
/*
* We allocate a byte[] first and then convert it to a StoredContinuation. We must pass
* parameters below that match the layout of a regular byte[], or we will run into problems
* because not all parameters are passed on to the slow path.
*
* Barrier code assumes that instance objects are always in aligned chunks, but a large
* StoredContinuation can end up in an unaligned chunk. Still, no barriers are needed
* because objects are immutable once filled and are then written only by GC.
*/
int arrayLength = StoredContinuationImpl.PAYLOAD_OFFSET + payloadSize - byteArrayBaseOffset;
Object result = allocateArrayImpl(encodeAsTLABObjectHeader(byteArrayHub), arrayLength, byteArrayBaseOffset, 0, FillContent.WITH_GARBAGE_IF_ASSERTIONS_ENABLED,
afterArrayLengthOffset(), false, false, false, false, profilingData);
UnsignedWord arrayHeader = ObjectHeaderImpl.readHeaderFromObject(result);
Word header = encodeAsObjectHeader(hub, ObjectHeaderImpl.hasRememberedSet(arrayHeader), ObjectHeaderImpl.isUnalignedHeader(arrayHeader));
initializeObjectHeader(Word.objectToUntrackedPointer(result), header, false);
ObjectAccess.writeObject(result, hub.getMonitorOffset(), null, LocationIdentity.init());
StoredContinuationImpl.initializeNewlyAllocated(result, payloadSize);
emitMemoryBarrierIf(true);
return PiNode.piCastToSnippetReplaceeStamp(result);
}

private static void finishFormatStoredContinuation(Object obj) {
StoredContinuationImpl.initializeNewlyAllocated(obj);
}

@Override
public void initializeObjectHeader(Word memory, Word objectHeader, boolean isArray) {
Heap.getHeap().getObjectHeader().initializeHeaderOfNewObject(memory, objectHeader);
Expand Down Expand Up @@ -254,12 +259,17 @@ public void lower(NewStoredContinuationNode node, LoweringTool tool) {

DynamicHub hub = ((SharedType) tool.getMetaAccess().lookupJavaType(StoredContinuation.class)).getHub();
assert hub.isStoredContinuationClass();

ConstantNode hubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(hub), providers.getMetaAccess(), graph);

DynamicHub byteArrayHub = ((SharedType) tool.getMetaAccess().lookupJavaType(byte[].class)).getHub();
ConstantNode byteArrayHubConstant = ConstantNode.forConstant(SubstrateObjectConstant.forObject(byteArrayHub), providers.getMetaAccess(), graph);
int byteArrayBaseOffset = getArrayBaseOffset(byteArrayHub.getLayoutEncoding());

Arguments args = new Arguments(allocateStoredContinuationInstance, graph.getGuardsStage(), tool.getLoweringStage());
args.add("hub", hubConstant);
args.add("size", node.getSize());
args.addConst("hub", hubConstant);
args.add("payloadSize", node.getPayloadSize());
args.addConst("byteArrayHub", byteArrayHubConstant);
args.addConst("byteArrayBaseOffset", byteArrayBaseOffset);
args.addConst("profilingData", getProfilingData(node, null));

template(node, args).instantiate(providers.getMetaAccess(), node, SnippetTemplate.DEFAULT_REPLACER, args);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ public static FrameAccess singleton() {
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public abstract CodePointer readReturnAddress(Pointer sourceSp);

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public abstract void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress);

@Fold
Expand Down Expand Up @@ -79,6 +80,7 @@ public static int wordSize() {
return ConfigurationValues.getTarget().arch.getWordSize();
}

@Fold
public static int uncompressedReferenceSize() {
return wordSize();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ public CodePointer readReturnAddress(Pointer sourceSp) {
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) {
sourceSp.writeWord(-returnAddressSize(), newReturnAddress);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public CodePointer readReturnAddress(Pointer sourceSp) {
}

@Override
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public void writeReturnAddress(Pointer sourceSp, CodePointer newReturnAddress) {
sourceSp.writeWord(-returnAddressSize(), newReturnAddress);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ public static long lookupTotalFrameSize(CodeInfo info, long ip) {
return CodeInfoQueryResult.getTotalFrameSize(codeInfoQueryResult.getEncodedFrameSize());
}

@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public static NonmovableArray<Byte> getStackReferenceMapEncoding(CodeInfo info) {
return cast(info).getStackReferenceMapEncoding();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -469,7 +469,7 @@ static void verifyMethod(SharedMethod method, CompilationResult compilation, int
CodeInfoAccess.lookupCodeInfo(info, offset + compilationOffset, queryResult);

CollectingObjectReferenceVisitor visitor = new CollectingObjectReferenceVisitor();
CodeReferenceMapDecoder.walkOffsetsFromPointer(WordFactory.zero(), CodeInfoAccess.getStackReferenceMapEncoding(info), queryResult.getReferenceMapIndex(), visitor);
CodeReferenceMapDecoder.walkOffsetsFromPointer(WordFactory.zero(), CodeInfoAccess.getStackReferenceMapEncoding(info), queryResult.getReferenceMapIndex(), visitor, null);
ReferenceMapEncoder.Input expected = (ReferenceMapEncoder.Input) infopoint.debugInfo.getReferenceMap();
visitor.result.verify();
assert expected.equals(visitor.result);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo
if (referenceMapIndex == ReferenceMapIndex.NO_REFERENCE_MAP) {
throw reportNoReferenceMap(sp, ip, info);
}
return CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor);
return CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null);
}

public static RuntimeException reportNoReferenceMap(Pointer sp, CodePointer ip, CodeInfo info) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,26 @@ CodeInfo prepareCodeInfo() {
return info;
}

/**
* Use {@link CodeInfoTable#getImageCodeInfo()} and {@link CodeInfoAccess#getCodeStart} instead.
* This method is intended only for the early stages of VM initialization when
* {@link #prepareCodeInfo()} has not yet run.
*/
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public CodePointer getCodeStart() {
return codeStart;
}

/**
* Use {@link CodeInfoTable#getImageCodeInfo()} and
* {@link CodeInfoAccess#getStackReferenceMapEncoding} instead. This method is intended only for
* the early stages of VM initialization when {@link #prepareCodeInfo()} has not yet run.
*/
@Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
public NonmovableArray<Byte> getStackReferenceMapEncoding() {
return NonmovableArrays.fromImageHeap(referenceMapEncoding);
}

public boolean walkImageCode(MemoryWalker.Visitor visitor) {
return visitor.visitCode(CodeInfoTable.getImageCodeInfo(), ImageSingletons.lookup(CodeInfoMemoryWalker.class));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public static boolean walkWeakReferences(CodeInfo info, ObjectReferenceVisitor v
NonmovableArrays.walkUnmanagedObjectArray(impl.getObjectFields(), visitor, CodeInfoImpl.FIRST_WEAKLY_REFERENCED_OBJFIELD, CodeInfoImpl.WEAKLY_REFERENCED_OBJFIELD_COUNT);
if (CodeInfoAccess.isAliveState(impl.getState())) {
continueVisiting = continueVisiting && CodeReferenceMapDecoder.walkOffsetsFromPointer(impl.getCodeStart(),
impl.getCodeConstantsReferenceMapEncoding(), impl.getCodeConstantsReferenceMapIndex(), visitor);
impl.getCodeConstantsReferenceMapEncoding(), impl.getCodeConstantsReferenceMapIndex(), visitor, null);
}
continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getFrameInfoObjectConstants(), visitor);
continueVisiting = continueVisiting && NonmovableArrays.walkUnmanagedObjectArray(impl.getFrameInfoSourceClasses(), visitor);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,30 +24,32 @@
*/
package com.oracle.svm.core.graal.nodes;

import com.oracle.svm.core.heap.StoredContinuation;
import jdk.vm.ci.meta.JavaKind;
import org.graalvm.compiler.core.common.type.StampFactory;
import org.graalvm.compiler.graph.NodeClass;
import org.graalvm.compiler.nodeinfo.NodeInfo;
import org.graalvm.compiler.nodes.ValueNode;
import org.graalvm.compiler.nodes.java.AbstractNewObjectNode;
import org.graalvm.compiler.nodes.spi.Lowerable;

import com.oracle.svm.core.heap.StoredContinuation;

import jdk.vm.ci.meta.JavaKind;

@NodeInfo
public class NewStoredContinuationNode extends AbstractNewObjectNode implements Lowerable {

public static final NodeClass<NewStoredContinuationNode> TYPE = NodeClass.create(NewStoredContinuationNode.class);
@Input private ValueNode size;
@Input private ValueNode payloadSize;

public NewStoredContinuationNode(ValueNode size) {
public NewStoredContinuationNode(ValueNode payloadSize) {
super(TYPE, StampFactory.forKind(JavaKind.fromJavaClass(StoredContinuation.class)), false, null);
this.size = size;
this.payloadSize = payloadSize;
}

public ValueNode getSize() {
return size;
public ValueNode getPayloadSize() {
return payloadSize;
}

@NodeIntrinsic
public static native StoredContinuation allocate(int size);
public static native StoredContinuation allocate(int payloadSize);
}
Loading

0 comments on commit 8dccf24

Please sign in to comment.