[GR-51160] [GR-51161] [GR-51198] [GR-51204] [GR-51213] Avoid deadlock…

…s when the VM is out-of-memory and other threading-related fixes.

PullRequest: graal/16510

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java

@@ -68,15 +68,15 @@ final class HeapChunkProvider {
      * head}, but this is OK because we only need the number of chunks for policy code (to avoid
      * running down the list and counting the number of chunks).
      */
-    private final AtomicUnsigned bytesInUnusedAlignedChunks = new AtomicUnsigned();
+    private final AtomicUnsigned numUnusedAlignedChunks = new AtomicUnsigned();
 
     @Platforms(Platform.HOSTED_ONLY.class)
     HeapChunkProvider() {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public UnsignedWord getBytesInUnusedChunks() {
-        return bytesInUnusedAlignedChunks.get();
+        return numUnusedAlignedChunks.get().multiply(HeapParameters.getAlignedHeapChunkSize());
     }
 
     private static final OutOfMemoryError ALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an aligned heap chunk");
@@ -181,7 +181,7 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) {
 
         HeapChunk.setNext(chunk, unusedAlignedChunks.get());
         unusedAlignedChunks.set(chunk);
-        bytesInUnusedAlignedChunks.addAndGet(HeapParameters.getAlignedHeapChunkSize());
+        numUnusedAlignedChunks.addAndGet(WordFactory.unsigned(1));
     }
 
     /**
@@ -199,7 +199,7 @@ private AlignedHeader popUnusedAlignedChunk() {
         if (result.isNull()) {
             return WordFactory.nullPointer();
         } else {
-            bytesInUnusedAlignedChunks.subtractAndGet(HeapParameters.getAlignedHeapChunkSize());
+            numUnusedAlignedChunks.subtractAndGet(WordFactory.unsigned(1));
             return result;
         }
     }
@@ -235,7 +235,7 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) {
             released = released.add(1);
         }
         unusedAlignedChunks.set(chunk);
-        bytesInUnusedAlignedChunks.subtractAndGet(released.multiply(HeapParameters.getAlignedHeapChunkSize()));
+        numUnusedAlignedChunks.subtractAndGet(released);
     }
 
     /** Acquire an UnalignedHeapChunk from the operating system. */

substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java

@@ -108,18 +108,33 @@ protected boolean doStartThread(Thread thread, long stackSize) {
                 }
             }
 
-            ThreadStartData startData = prepareStart(thread, SizeOf.get(ThreadStartData.class));
+            /*
+             * Prevent stack overflow errors so that starting the thread and reverting back to a
+             * safe state (in case of an error) works reliably.
+             */
+            StackOverflowCheck.singleton().makeYellowZoneAvailable();
+            try {
+                return doStartThread0(thread, attributes);
+            } finally {
+                StackOverflowCheck.singleton().protectYellowZone();
+            }
+        } finally {
+            Pthread.pthread_attr_destroy(attributes);
+        }
+    }
 
+    /** Starts a thread to the point so that it is executing. */
+    private boolean doStartThread0(Thread thread, pthread_attr_t attributes) {
+        ThreadStartData startData = prepareStart(thread, SizeOf.get(ThreadStartData.class));
+        try {
             Pthread.pthread_tPointer newThread = UnsafeStackValue.get(Pthread.pthread_tPointer.class);
             if (Pthread.pthread_create(newThread, attributes, threadStartRoutine.getFunctionPointer(), startData) != 0) {
                 undoPrepareStartOnError(thread, startData);
                 return false;
             }
-
-            setPthreadIdentifier(thread, newThread.read());
             return true;
-        } finally {
-            Pthread.pthread_attr_destroy(attributes);
+        } catch (Throwable e) {
+            throw VMError.shouldNotReachHere("No exception must be thrown after creating the thread start data.", e);
         }
     }
 
@@ -145,8 +160,8 @@ static boolean hasThreadIdentifier(Thread thread) {
     protected void setNativeName(Thread thread, String name) {
         if (!hasThreadIdentifier(thread)) {
             /*
-             * The thread was not started from Java code, but started from C code and attached
-             * manually to SVM. We do not want to interfere with such threads.
+             * The thread was a. not started yet, b. not started from Java code (i.e., only attached
+             * to SVM). We do not want to interfere with such threads.
              */
             return;
         }
@@ -177,6 +192,11 @@ protected void yieldCurrent() {
         Sched.sched_yield();
     }
 
+    /**
+     * Note that this method is only executed for Java threads that are started via
+     * {@link Thread#start()}. It is not executed if an existing native thread is attached to an
+     * isolate.
+     */
     @Override
     protected void beforeThreadRun(Thread thread) {
         /* Complete the initialization of the thread, now that it is (nearly) running. */

substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsPlatformThreads.java

@@ -61,21 +61,38 @@ public final class WindowsPlatformThreads extends PlatformThreads {
     @Override
     protected boolean doStartThread(Thread thread, long stackSize) {
         int threadStackSize = NumUtil.safeToUInt(stackSize);
-
         int initFlag = 0;
         // If caller specified a stack size, don't commit it all at once.
         if (threadStackSize != 0) {
             initFlag |= Process.STACK_SIZE_PARAM_IS_A_RESERVATION();
         }
 
+        /*
+         * Prevent stack overflow errors so that starting the thread and reverting back to a safe
+         * state (in case of an error) works reliably.
+         */
+        StackOverflowCheck.singleton().makeYellowZoneAvailable();
+        try {
+            return doStartThread0(thread, threadStackSize, initFlag);
+        } finally {
+            StackOverflowCheck.singleton().protectYellowZone();
+        }
+    }
+
+    /** Starts a thread to the point so that it is executing. */
+    private boolean doStartThread0(Thread thread, int threadStackSize, int initFlag) {
         ThreadStartData startData = prepareStart(thread, SizeOf.get(ThreadStartData.class));
-        WinBase.HANDLE osThreadHandle = Process._beginthreadex(WordFactory.nullPointer(), threadStackSize, threadStartRoutine.getFunctionPointer(), startData, initFlag, WordFactory.nullPointer());
-        if (osThreadHandle.isNull()) {
-            undoPrepareStartOnError(thread, startData);
-            return false;
+        try {
+            WinBase.HANDLE osThreadHandle = Process._beginthreadex(WordFactory.nullPointer(), threadStackSize, threadStartRoutine.getFunctionPointer(), startData, initFlag, WordFactory.nullPointer());
+            if (osThreadHandle.isNull()) {
+                undoPrepareStartOnError(thread, startData);
+                return false;
+            }
+            WinBase.CloseHandle(osThreadHandle);
+            return true;
+        } catch (Throwable e) {
+            throw VMError.shouldNotReachHere("No exception must be thrown after creating the thread start data.", e);
         }
-        WinBase.CloseHandle(osThreadHandle);
-        return true;
     }
 
     @Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/JavaMainWrapper.java

@@ -72,7 +72,6 @@
 import com.oracle.svm.core.log.Log;
 import com.oracle.svm.core.thread.JavaThreads;
 import com.oracle.svm.core.thread.PlatformThreads;
-import com.oracle.svm.core.thread.ThreadListenerSupport;
 import com.oracle.svm.core.thread.VMThreads;
 import com.oracle.svm.core.thread.VMThreads.OSThreadHandle;
 import com.oracle.svm.core.util.CounterSupport;
@@ -221,8 +220,6 @@ private static int runCore0() {
                 return VMInspectionOptions.dumpImageHeap() ? 0 : 1;
             }
 
-            ThreadListenerSupport.get().beforeThreadRun();
-
             // Ensure that native code using JNI_GetCreatedJavaVMs finds this isolate.
             JNIJavaVMList.addJavaVM(JNIFunctionTables.singleton().getGlobalJavaVM());
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/function/CEntryPointActions.java

@@ -65,10 +65,11 @@ private CEntryPointActions() {
      * @param startedByIsolate Whether the current thread has been launched directly by the isolate
      *            (as opposed to being an externally started thread), which makes the isolate
      *            responsible for cleanups when the thread detaches.
-     * @param ensureJavaThread when set to true, the method ensures that the {@link Thread} object
-     *            for the newly attached thread is created. If the parameter is set to false, a
-     *            later call to one of the {@link PlatformThreads#ensureCurrentAssigned} methods
-     *            early after the prologue must be used to do the initialization manually.
+     * @param ensureJavaThread when set to true, {@link PlatformThreads#ensureCurrentAssigned()} is
+     *            called to ensure that the Java {@link Thread} is fully initialized. If the
+     *            parameter is set to false, the initialization must be done manually (early after
+     *            the prologue).
+     *
      * @return 0 on success, otherwise non-zero (see {@link CEntryPointErrors})
      */
     public static native int enterAttachThread(Isolate isolate, boolean startedByIsolate, boolean ensureJavaThread);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java

@@ -88,6 +88,7 @@
 import com.oracle.svm.core.stack.StackOverflowCheck;
 import com.oracle.svm.core.thread.PlatformThreads;
 import com.oracle.svm.core.thread.Safepoint;
+import com.oracle.svm.core.thread.ThreadListenerSupport;
 import com.oracle.svm.core.thread.VMOperationControl;
 import com.oracle.svm.core.thread.VMThreads;
 import com.oracle.svm.core.thread.VMThreads.SafepointBehavior;
@@ -155,8 +156,7 @@ public final class CEntryPointSnippets extends SubstrateTemplates implements Sni
     public static native int runtimeCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, CEntryPointCreateIsolateParameters parameters);
 
     @NodeIntrinsic(value = ForeignCallNode.class)
-    public static native int runtimeCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Isolate isolate,
-                    boolean startedByIsolate, boolean ensuringJavaThread);
+    public static native int runtimeCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Isolate isolate, boolean startedByIsolate, boolean ensuringJavaThread);
 
     @NodeIntrinsic(value = ForeignCallNode.class)
     public static native int runtimeCall(@ConstantNodeParameter ForeignCallDescriptor descriptor, Isolate isolate);
@@ -202,7 +202,6 @@ public static int createIsolateSnippet(CEntryPointCreateIsolateParameters parame
             writeCurrentVMThread(WordFactory.nullPointer());
         }
         int result = runtimeCall(CREATE_ISOLATE, parameters);
-
         if (result != CEntryPointErrors.NO_ERROR) {
             return result;
         }
@@ -269,8 +268,7 @@ private static int createIsolate0(CLongPointer parsedArgs, Isolate isolate) {
         if (error != CEntryPointErrors.NO_ERROR) {
             return error;
         }
-
-        PlatformThreads.singleton().initializeIsolate();
+        PlatformThreads.singleton().assignMainThread();
         return CEntryPointErrors.NO_ERROR;
     }
 
@@ -405,6 +403,14 @@ private static int initializeIsolateInterruptibly0(CEntryPointCreateIsolateParam
             return CEntryPointErrors.ISOLATE_INITIALIZATION_FAILED;
         }
 
+        /* The isolate is now initialized, so we can finally finish initializing the main thread. */
+        try {
+            ThreadListenerSupport.get().beforeThreadRun();
+        } catch (Throwable t) {
+            System.err.println("Uncaught exception in beforeThreadRun():");
+            t.printStackTrace();
+            return CEntryPointErrors.ISOLATE_INITIALIZATION_FAILED;
+        }
         return CEntryPointErrors.NO_ERROR;
     }
 
@@ -421,8 +427,9 @@ public static int attachThreadSnippet(Isolate isolate, boolean startedByIsolate,
         if (MultiThreaded.getValue()) {
             Safepoint.transitionNativeToJava(false);
         }
+
         if (ensureJavaThread) {
-            runtimeCallEnsureJavaThread(ENSURE_JAVA_THREAD);
+            return runtimeCallEnsureJavaThread(ENSURE_JAVA_THREAD);
         }
         return CEntryPointErrors.NO_ERROR;
     }
@@ -530,11 +537,22 @@ public static void initializeIsolateThreadForCrashHandler(Isolate isolate, Isola
     }
 
     @NodeIntrinsic(value = ForeignCallNode.class)
-    public static native void runtimeCallEnsureJavaThread(@ConstantNodeParameter ForeignCallDescriptor descriptor);
+    public static native int runtimeCallEnsureJavaThread(@ConstantNodeParameter ForeignCallDescriptor descriptor);
 
+    @Uninterruptible(reason = "Prevent stack overflow errors; thread is no longer attached in error case.", calleeMustBe = false)
     @SubstrateForeignCallTarget(stubCallingConvention = false)
-    private static void ensureJavaThread() {
-        PlatformThreads.ensureCurrentAssigned();
+    private static int ensureJavaThread() {
+        try {
+            PlatformThreads.ensureCurrentAssigned();
+            return CEntryPointErrors.NO_ERROR;
+        } catch (Throwable e) {
+            int result = CEntryPointErrors.UNCAUGHT_EXCEPTION;
+            if (e instanceof OutOfMemoryError) {
+                result = CEntryPointErrors.ALLOCATION_FAILED;
+            }
+            CEntryPointActions.leaveDetachThread();
+            return result;
+        }
     }
 
     @Snippet(allowMissingProbabilities = true)
@@ -544,12 +562,6 @@ public static int detachThreadSnippet() {
             IsolateThread thread = CurrentIsolate.getCurrentThread();
             result = runtimeCall(DETACH_THREAD_MT, thread);
         }
-
-        /*
-         * Note that we do not reset the fixed registers used for the thread and isolate to null:
-         * Since these values are not copied to different registers when they are used, we need to
-         * keep the registers intact until the last possible point where we are in Java code.
-         */
         return result;
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/sampler/JfrRecurringCallbackExecutionSampler.java

@@ -30,7 +30,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import com.oracle.svm.core.jfr.JfrExecutionSamplerSupported;
 import org.graalvm.nativeimage.CurrentIsolate;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.IsolateThread;
@@ -45,6 +44,7 @@
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
+import com.oracle.svm.core.jfr.JfrExecutionSamplerSupported;
 import com.oracle.svm.core.jfr.JfrFeature;
 import com.oracle.svm.core.jfr.SubstrateJVM;
 import com.oracle.svm.core.thread.ThreadListenerSupport;
@@ -106,6 +106,7 @@ private static void install(IsolateThread thread, RecurringCallbackTimer callbac
         }
     }
 
+    @Override
     @Uninterruptible(reason = "Prevent VM operations that modify the recurring callbacks.")
     protected void uninstall(IsolateThread thread) {
         assert thread == CurrentIsolate.getCurrentThread() || VMOperation.isInProgressAtSafepoint();

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/JavaMonitor.java

@@ -46,7 +46,7 @@
 
 /**
  * {@link JavaMonitor} is based on the code of {@link java.util.concurrent.locks.ReentrantLock} as
- * of JDK 19 (git commit hash: f640fc5a1eb876a657d0de011dcd9b9a42b88eec, JDK tag: jdk-19+30).
+ * of JDK 21+26.
  *
  * Only the relevant methods from the JDK sources have been kept. Some additional Native
  * Image-specific functionality has been added.
@@ -91,7 +91,7 @@ protected JavaMonitorConditionObject getOrCreateCondition(boolean createIfNotExi
             return existingCondition;
         }
         JavaMonitorConditionObject newCondition = new JavaMonitorConditionObject();
-        if (!U.compareAndSetObject(this, CONDITION_FIELD_OFFSET, null, newCondition)) {
+        if (!U.compareAndSetReference(this, CONDITION_FIELD_OFFSET, null, newCondition)) {
             newCondition = condition;
         }
         return newCondition;