From f05f825712ace875a5a48da84f4fc79ee681a911 Mon Sep 17 00:00:00 2001 From: Jack Lu Date: Thu, 2 Mar 2023 12:03:23 -0500 Subject: [PATCH 1/6] Remove liveVirtualThreadListMutex - Refactor jitReleaseCodeStackWalk to use GC Continuation list - Refactor vthread sync to EVMA for global vthread access - Include no spin option for acquireVThreadInspector Signed-off-by: Jack Lu --- .../classes/jdk/internal/vm/Continuation.java | 1 + runtime/compiler/control/HookedByTheJit.cpp | 92 ++++----- runtime/j9vm/javanextvmi.cpp | 186 ++++-------------- runtime/jcl/common/jclcinit.c | 10 - runtime/jcl/common/thread.cpp | 28 +-- runtime/jvmti/jvmtiHelpers.c | 34 +--- runtime/jvmti/jvmtiThread.c | 148 +++++++------- runtime/oti/VMHelpers.hpp | 1 - runtime/oti/j9nonbuilder.h | 7 +- runtime/oti/vm_api.h | 20 ++ runtime/oti/vmconstantpool.xml | 1 + runtime/vm/ContinuationHelpers.cpp | 60 ++++++ runtime/vm/intfunc.c | 2 + runtime/vm/vmthinit.c | 5 - 14 files changed, 264 insertions(+), 331 deletions(-) diff --git a/jcl/src/java.base/share/classes/jdk/internal/vm/Continuation.java b/jcl/src/java.base/share/classes/jdk/internal/vm/Continuation.java index 83618b51167..31fa1f386d6 100644 --- a/jcl/src/java.base/share/classes/jdk/internal/vm/Continuation.java +++ b/jcl/src/java.base/share/classes/jdk/internal/vm/Continuation.java @@ -34,6 +34,7 @@ */ public class Continuation { private long vmRef; /* J9VMContinuation */ + protected Thread vthread; /* Parent VirtualThread */ private final ContinuationScope scope; private final Runnable runnable; diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index 583ec41c021..fa3517a19d9 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -34,6 +34,7 @@ #include "mmhook.h" #include "mmomrhook.h" #include "vmaccess.h" +#include "HeapIteratorAPI.h" #include "codegen/CodeGenerator.hpp" #include "compile/CompilationTypes.hpp" #include "compile/Method.hpp" @@ -6459,8 +6460,46 @@ static UDATA jitReleaseCodeStackWalkFrame(J9VMThread *vmThread, J9StackWalkState return J9_STACKWALK_KEEP_ITERATING; } -static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFunctionPtr condYield = NULL) +static jvmtiIterationControl jitWalkContinuationCallBack(J9VMThread *vmThread, J9MM_IterateObjectDescriptor *object, void *userData) + { + J9InternalVMFunctions *vmFuncs = vmThread->javaVM->internalVMFunctions; + J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(vmThread, object->object); + if (NULL != continuation) + { + bool yieldHappened = false; + if ((continuation->dropFlags & 0x1) ? false : true) + { + J9StackWalkState walkState; + walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; + walkState.skipCount = 0; + walkState.frameWalkFunction = jitReleaseCodeStackWalkFrame; + vmFuncs->walkContinuationStackFrames(vmThread, continuation, &walkState); + continuation->dropFlags = 0x1; + condYieldFromGCFunctionPtr condYield = (condYieldFromGCFunctionPtr)userData; + yieldHappened = condYield(vmThread->omrVMThread, J9_GC_METRONOME_UTILIZATION_COMPONENT_JIT); + } + + if (yieldHappened) + { + /* Stop the iteration, will resume later. */ + return JVMTI_ITERATION_ABORT; + } + } + return JVMTI_ITERATION_CONTINUE; + } +static jvmtiIterationControl jitResetContinuationFlag(J9VMThread *vmThread, J9MM_IterateObjectDescriptor *object, void *userData) + { + J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(vmThread, object->object); + if (NULL != continuation) + { + continuation->dropFlags = 0; + } + + return JVMTI_ITERATION_CONTINUE; + } + +static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFunctionPtr condYield = NULL) { J9VMThread *vmThread = (J9VMThread *)omrVMThread->_language_vmthread; J9JITConfig *jitConfig = vmThread->javaVM->jitConfig; @@ -6475,6 +6514,7 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu #if JAVA_SPEC_VERSION >= 19 J9JavaVM *vm = vmThread->javaVM; J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; + PORT_ACCESS_FROM_VMC(vmThread); if (isRealTimeGC && !TR::Options::getCmdLineOptions()->getOption(TR_DisableIncrementalCCR)) { bool yieldHappened = false; @@ -6515,34 +6555,13 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu } while (yieldHappened); - if (NULL != vm->liveVirtualThreadList) + do { - do - { - yieldHappened = false; - /* Skip the root, which is a dummy virtual thread and global ref. */ - j9object_t walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, *(vm->liveVirtualThreadList), vm->virtualThreadLinkNextOffset); - while ((*(vm->liveVirtualThreadList) != walkVirtualThread) && !yieldHappened) - { - j9object_t contObject = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CONT(vmThread, walkVirtualThread); - J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(vmThread, contObject); - if ((continuation->dropFlags & 0x1) ? false : true) - { - J9StackWalkState walkState; - walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; - walkState.skipCount = 0; - walkState.frameWalkFunction = jitReleaseCodeStackWalkFrame; - vmFuncs->walkContinuationStackFrames(vmThread, continuation, &walkState); - continuation->dropFlags |= 0x1; - yieldHappened = condYield(omrVMThread, J9_GC_METRONOME_UTILIZATION_COMPONENT_JIT); - } - - if (!yieldHappened) - walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, walkVirtualThread, vm->virtualThreadLinkNextOffset); - } - } - while (yieldHappened); + jvmtiIterationControl rc = vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(vmThread, PORTLIB, 0, jitWalkContinuationCallBack, (void*)condYield); + if (JVMTI_ITERATION_ABORT == rc) + yieldHappened = true; } + while (yieldHappened); } else { J9StackWalkState walkState; walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; @@ -6665,31 +6684,14 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu do { thr->dropFlags &=0x0; -#if JAVA_SPEC_VERSION >= 19 - if (NULL != thr->currentContinuation) { - thr->currentContinuation->dropFlags &=0x0; - } -#endif /* JAVA_SPEC_VERSION >= 19 */ thr = thr->linkNext; } while (thr != vmThread); #if JAVA_SPEC_VERSION >= 19 - if (NULL != vm->liveVirtualThreadList) - { - /* Skip the root, which is a dummy virtual thread and global ref. */ - j9object_t walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, *(vm->liveVirtualThreadList), vm->virtualThreadLinkNextOffset); - while (*(vm->liveVirtualThreadList) != walkVirtualThread) - { - j9object_t contObject = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CONT(vmThread, walkVirtualThread); - J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(vmThread, contObject); - continuation->dropFlags &= 0x0; - walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, walkVirtualThread, vm->virtualThreadLinkNextOffset); - } - } + vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(vmThread, PORTLIB, 0, jitResetContinuationFlag, NULL); #endif /* JAVA_SPEC_VERSION >= 19 */ } - } static void jitHookReleaseCodeGlobalGCEnd(J9HookInterface **hook, UDATA eventNum, void *eventData, void *userData) diff --git a/runtime/j9vm/javanextvmi.cpp b/runtime/j9vm/javanextvmi.cpp index 684636ab9fd..20270e85e03 100644 --- a/runtime/j9vm/javanextvmi.cpp +++ b/runtime/j9vm/javanextvmi.cpp @@ -246,6 +246,31 @@ JVM_IsPreviewEnabled(void) return isPreviewEnabled; } +static void +enterVThreadTransitionCritical(J9VMThread *currentThread, jobject thread) +{ + J9JavaVM *vm = currentThread->javaVM; + J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; + MM_ObjectAccessBarrierAPI objectAccessBarrier = MM_ObjectAccessBarrierAPI(currentThread); + j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + + while(!objectAccessBarrier.inlineMixedObjectCompareAndSwapU64(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, 0, (U_64)-1)) { + /* Thread is being inspected or unmounted, wait. */ + vmFuncs->internalExitVMToJNI(currentThread); + VM_AtomicSupport::yieldCPU(); + /* After wait, the thread may suspend here. */ + vmFuncs->internalEnterVMFromJNI(currentThread); + threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + } +} + +static void +exitVThreadTransitionCritical(J9VMThread *currentThread, j9object_t vthread) +{ + Assert_SC_true(-1 == J9OBJECT_I64_LOAD(currentThread, vthread, currentThread->javaVM->virtualThreadInspectorCountOffset)); + J9OBJECT_I64_STORE(currentThread, vthread, currentThread->javaVM->virtualThreadInspectorCountOffset, 0); +} + JNIEXPORT void JNICALL JVM_VirtualThreadMountBegin(JNIEnv *env, jobject thread, jboolean firstMount) { @@ -256,13 +281,12 @@ JVM_VirtualThreadMountBegin(JNIEnv *env, jobject thread, jboolean firstMount) Trc_SC_VirtualThreadMountBegin_Entry(currentThread, thread, firstMount); vmFuncs->internalEnterVMFromJNI(currentThread); - f_monitorEnter(vm->liveVirtualThreadListMutex); j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread); - Assert_SC_true(IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObj)); if (TrcEnabled_Trc_SC_VirtualThread_Info) { j9object_t continuationObj = J9VMJAVALANGVIRTUALTHREAD_CONT(currentThread, threadObj); + J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, continuationObj); Trc_SC_VirtualThread_Info( currentThread, threadObj, @@ -270,30 +294,11 @@ JVM_VirtualThreadMountBegin(JNIEnv *env, jobject thread, jboolean firstMount) J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset), J9VMJAVALANGVIRTUALTHREAD_CARRIERTHREAD(currentThread, threadObj), continuationObj, - J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, continuationObj)); - } - - while (vm->inspectingLiveVirtualThreadList - || (0 != J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset)) - ) { - /* Thread is being inspected or unmounted, wait. */ - vmFuncs->internalExitVMToJNI(currentThread); - f_monitorWait(vm->liveVirtualThreadListMutex); - /* Release the monitor before suspending. */ - f_monitorExit(vm->liveVirtualThreadListMutex); - /* After wait, the thread may suspend here. */ - vmFuncs->internalEnterVMFromJNI(currentThread); - /* Acquire the monitor after resuming from suspend. */ - f_monitorEnter(vm->liveVirtualThreadListMutex); - threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + continuation); } - /* Prevent inspectors from inspecting this thread during stack swap and mount by locking from notifyJvmtiMountBegin - * to notifyJvmtiMountEnd. See getVMThread() in jvmtiHelpers.c. - */ - J9OBJECT_I64_STORE(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, -1); + enterVThreadTransitionCritical(currentThread, thread); - f_monitorExit(vm->liveVirtualThreadListMutex); vmFuncs->internalExitVMToJNI(currentThread); Trc_SC_VirtualThreadMountBegin_Exit(currentThread, thread, firstMount); @@ -328,71 +333,13 @@ JVM_VirtualThreadMountEnd(JNIEnv *env, jobject thread, jboolean firstMount) J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, continuationObj)); } - /* On some occassions, J9AllocateObject acquires exclusive VM access. It is invoked without acquiring - * liveVirtualThreadListMutex to prevent a deadlock. - */ - if (NULL == vm->liveVirtualThreadList) { - Assert_SC_true(firstMount); - J9Class *virtualThreadClass = J9OBJECT_CLAZZ(currentThread, J9_JNI_UNWRAP_REFERENCE(thread)); - J9MemoryManagerFunctions *mmFuncs = vm->memoryManagerFunctions; - - /* Allocate empty virtual thread and create a global reference to it as root for the linked list. - * This prevents the root reference from becoming stale if the GC moves the object. - */ - rootVirtualThread = mmFuncs->J9AllocateObject(currentThread, virtualThreadClass, J9_GC_ALLOCATE_OBJECT_NON_INSTRUMENTABLE); - if (NULL == rootVirtualThread) { - vmFuncs->setHeapOutOfMemoryError(currentThread); - goto release1; - } - - /* Re-fetch as the memory allocation above may have moved the object. */ - threadObj = J9_JNI_UNWRAP_REFERENCE(thread); - - J9VMJAVALANGVIRTUALTHREAD_SET_STATE(currentThread, rootVirtualThread, J9VM_VIRTUALTHREAD_ROOT_NODE_STATE); - } - - f_monitorEnter(vm->liveVirtualThreadListMutex); + /* Allow thread to be inspected again. */ + exitVThreadTransitionCritical(currentThread, threadObj); if (firstMount) { - if (NULL == vm->liveVirtualThreadList) { - Assert_SC_true(NULL != rootVirtualThread); - - jobject globalRef = vmFuncs->j9jni_createGlobalRef((JNIEnv *)currentThread, rootVirtualThread, JNI_FALSE); - if (NULL == globalRef) { - vmFuncs->setNativeOutOfMemoryError(currentThread, 0, 0); - goto release2; - } - - Trc_SC_VirtualThread_RootNodeSet(currentThread, globalRef); - vm->liveVirtualThreadList = (j9object_t *)globalRef; - /* Set linkNext/linkPrevious to itself. */ - J9OBJECT_OBJECT_STORE(currentThread, rootVirtualThread, vm->virtualThreadLinkNextOffset, rootVirtualThread); - J9OBJECT_OBJECT_STORE(currentThread, rootVirtualThread, vm->virtualThreadLinkPreviousOffset, rootVirtualThread); - } - - if (NULL != vm->liveVirtualThreadList) { - j9object_t threadPrev = J9OBJECT_OBJECT_LOAD(currentThread, threadObj, vm->virtualThreadLinkPreviousOffset); - j9object_t threadNext = J9OBJECT_OBJECT_LOAD(currentThread, threadObj, vm->virtualThreadLinkNextOffset); - - /* Non-null previous and next elements in the list suggest that the thread has - * already been added to the list. Only add to the list if the previous and - * next elements in the list are null. - */ - Assert_SC_true((NULL == threadPrev) && (NULL == threadNext)); - j9object_t root = *(vm->liveVirtualThreadList); - j9object_t rootPrev = J9OBJECT_OBJECT_LOAD(currentThread, root, vm->virtualThreadLinkPreviousOffset); - - /* Add thread to the end of the list. */ - J9OBJECT_OBJECT_STORE(currentThread, threadObj, vm->virtualThreadLinkNextOffset, root); - J9OBJECT_OBJECT_STORE(currentThread, threadObj, vm->virtualThreadLinkPreviousOffset, rootPrev); - J9OBJECT_OBJECT_STORE(currentThread, rootPrev, vm->virtualThreadLinkNextOffset, threadObj); - J9OBJECT_OBJECT_STORE(currentThread, root, vm->virtualThreadLinkPreviousOffset, threadObj); - } + TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_STARTED(vm->hookInterface, currentThread); } - - /* Allow thread to be inspected again. */ - Assert_SC_true(-1 == J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset)); - J9OBJECT_I64_STORE(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, 0); + TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_MOUNT(vm->hookInterface, currentThread); /* If isSuspendedByJVMTI is non-zero, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND is set * in currentThread->publicFlags while resetting isSuspendedByJVMTI to zero. During @@ -403,19 +350,6 @@ JVM_VirtualThreadMountEnd(JNIEnv *env, jobject thread, jboolean firstMount) vmFuncs->setHaltFlag(currentThread, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND); } - f_monitorNotifyAll(vm->liveVirtualThreadListMutex); - - /* J9Hooks can be run since no errors were encountered. */ - runJ9Hooks = TRUE; -release2: - f_monitorExit(vm->liveVirtualThreadListMutex); -release1: - if (runJ9Hooks) { - if (firstMount) { - TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_STARTED(vm->hookInterface, currentThread); - } - TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_MOUNT(vm->hookInterface, currentThread); - } vmFuncs->internalExitVMToJNI(currentThread); Trc_SC_VirtualThreadMountEnd_Exit(currentThread, thread, firstMount); @@ -431,7 +365,6 @@ JVM_VirtualThreadUnmountBegin(JNIEnv *env, jobject thread, jboolean lastUnmount) Trc_SC_VirtualThreadUnmountBegin_Entry(currentThread, thread, lastUnmount); vmFuncs->internalEnterVMFromJNI(currentThread); - f_monitorEnter(vm->liveVirtualThreadListMutex); j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread); Assert_SC_true(IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObj)); @@ -448,51 +381,18 @@ JVM_VirtualThreadUnmountBegin(JNIEnv *env, jobject thread, jboolean lastUnmount) J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, continuationObj)); } - while (vm->inspectingLiveVirtualThreadList - || (0 != J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset)) - ) { - /* Thread is being inspected, wait. */ - vmFuncs->internalExitVMToJNI(currentThread); - f_monitorWait(vm->liveVirtualThreadListMutex); - /* Release the monitor before suspending. */ - f_monitorExit(vm->liveVirtualThreadListMutex); - /* After wait, the thread may suspend here. */ - vmFuncs->internalEnterVMFromJNI(currentThread); - /* Acquire the monitor after resuming from suspend. */ - f_monitorEnter(vm->liveVirtualThreadListMutex); - threadObj = J9_JNI_UNWRAP_REFERENCE(thread); - } - - /* Prevent inspectors from inspecting this thread during stack swap and unmount by locking from notifyJvmtiUnmountBegin - * to notifyJvmtiUnmountEnd. See getVMThread() in jvmtiHelpers.c. - */ - J9OBJECT_I64_STORE(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, -1); - + TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_UNMOUNT(vm->hookInterface, currentThread); if (lastUnmount) { - if (NULL != vm->liveVirtualThreadList) { - j9object_t threadPrev = J9OBJECT_OBJECT_LOAD(currentThread, threadObj, vm->virtualThreadLinkPreviousOffset); - j9object_t threadNext = J9OBJECT_OBJECT_LOAD(currentThread, threadObj, vm->virtualThreadLinkNextOffset); - - /* Non-null previous and next elements in the list suggest that the thread has - * been added to the list. Only remove from the list if the previous and next - * elements in the list are non-null. - */ - Assert_SC_true((NULL != threadPrev) && (NULL != threadNext)); - /* Remove thread from the list. The root will never be removed. */ - J9OBJECT_OBJECT_STORE(currentThread, threadPrev, vm->virtualThreadLinkNextOffset, threadNext); - J9OBJECT_OBJECT_STORE(currentThread, threadNext, vm->virtualThreadLinkPreviousOffset, threadPrev); - - /* Reset previous and next fields in the thread object to null. */ - J9OBJECT_OBJECT_STORE(currentThread, threadObj, vm->virtualThreadLinkNextOffset, NULL); - J9OBJECT_OBJECT_STORE(currentThread, threadObj, vm->virtualThreadLinkPreviousOffset, NULL); - } + TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_END(vm->hookInterface, currentThread); } - f_monitorExit(vm->liveVirtualThreadListMutex); - - TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_UNMOUNT(vm->hookInterface, currentThread); + enterVThreadTransitionCritical(currentThread, thread); if (lastUnmount) { - TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_END(vm->hookInterface, currentThread); + /* Re-fetch reference as enterVThreadTransitionCritical may release VMAccess. */ + threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + j9object_t continuationObj = J9VMJAVALANGVIRTUALTHREAD_CONT(currentThread, threadObj); + /* Add reverse link from Continuation object to VirtualThread object, this let JVMTI code */ + J9VMJDKINTERNALVMCONTINUATION_SET_VTHREAD(currentThread, continuationObj, NULL); } vmFuncs->internalExitVMToJNI(currentThread); @@ -509,7 +409,6 @@ JVM_VirtualThreadUnmountEnd(JNIEnv *env, jobject thread, jboolean lastUnmount) Trc_SC_VirtualThreadUnmountEnd_Entry(currentThread, thread, lastUnmount); vmFuncs->internalEnterVMFromJNI(currentThread); - f_monitorEnter(vm->liveVirtualThreadListMutex); j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread); Assert_SC_true(IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObj)); @@ -534,11 +433,8 @@ JVM_VirtualThreadUnmountEnd(JNIEnv *env, jobject thread, jboolean lastUnmount) } /* Allow thread to be inspected again. */ - Assert_SC_true(-1 == J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset)); - J9OBJECT_I64_STORE(currentThread, threadObj, vm->virtualThreadInspectorCountOffset, 0); - f_monitorNotifyAll(vm->liveVirtualThreadListMutex); + exitVThreadTransitionCritical(currentThread, threadObj); - f_monitorExit(vm->liveVirtualThreadListMutex); vmFuncs->internalExitVMToJNI(currentThread); Trc_SC_VirtualThreadUnmountEnd_Exit(currentThread, thread, lastUnmount); diff --git a/runtime/jcl/common/jclcinit.c b/runtime/jcl/common/jclcinit.c index 9ac8c4cc1b9..1c81b92835f 100644 --- a/runtime/jcl/common/jclcinit.c +++ b/runtime/jcl/common/jclcinit.c @@ -650,16 +650,6 @@ initializeRequiredClasses(J9VMThread *vmThread, char* dllName) return 1; } - /* Points to the next VirtualThread in the liveVirtualThreadList. */ - if (0 != vmFuncs->addHiddenInstanceField(vm, "java/lang/VirtualThread", "linkNext", "Ljava/lang/VirtualThread;", &vm->virtualThreadLinkNextOffset)) { - return 1; - } - - /* Points to the previous VirtualThread in the liveVirtualThreadList. */ - if (0 != vmFuncs->addHiddenInstanceField(vm, "java/lang/VirtualThread", "linkPrevious", "Ljava/lang/VirtualThread;", &vm->virtualThreadLinkPreviousOffset)) { - return 1; - } - /* Counter to track if the virtual thread is being inspected by JVMTI. */ if (0 != vmFuncs->addHiddenInstanceField(vm, "java/lang/VirtualThread", "inspectorCount", "J", &vm->virtualThreadInspectorCountOffset)) { return 1; diff --git a/runtime/jcl/common/thread.cpp b/runtime/jcl/common/thread.cpp index 990401c961c..20e45f8a8f2 100644 --- a/runtime/jcl/common/thread.cpp +++ b/runtime/jcl/common/thread.cpp @@ -339,17 +339,18 @@ Java_java_lang_Thread_getStackTraceImpl(JNIEnv *env, jobject rcv) #if JAVA_SPEC_VERSION >= 19 BOOLEAN releaseInspector = FALSE; if (IS_JAVA_LANG_VIRTUALTHREAD(currentThread, receiverObject)) { - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); + /* Do not spin when acquiring access, if acquire failed, return NULL. + * The caller of getStackTraceImpl will handle if should retry or get stack using unmounted path. + */ + if (!vmFuncs->acquireVThreadInspector(currentThread, rcv, FALSE)) { + goto done; + } j9object_t carrierThread = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CARRIERTHREAD(currentThread, receiverObject); - I_64 vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, receiverObject, vm->virtualThreadInspectorCountOffset); - /* Ensure virtual thread is mounted and not during transition. */ - if ((NULL != carrierThread) && (vthreadInspectorCount >= 0)) { - J9OBJECT_I64_STORE(currentThread, receiverObject, vm->virtualThreadInspectorCountOffset, vthreadInspectorCount + 1); + if (NULL != carrierThread) { releaseInspector = TRUE; - } - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); - if (!releaseInspector) { + } else { + vmFuncs->releaseVThreadInspector(currentThread, rcv); goto done; } /* Gets targetThread from the carrierThread object. */ @@ -370,16 +371,7 @@ Java_java_lang_Thread_getStackTraceImpl(JNIEnv *env, jobject rcv) if (releaseInspector) { receiverObject = J9_JNI_UNWRAP_REFERENCE(rcv); /* Release the virtual thread (allow it to die) now that we are no longer inspecting it. */ - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - I_64 vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, receiverObject, vm->virtualThreadInspectorCountOffset); - Assert_JCL_true(vthreadInspectorCount > 0); - vthreadInspectorCount -= 1; - J9OBJECT_I64_STORE(currentThread, receiverObject, vm->virtualThreadInspectorCountOffset, vthreadInspectorCount); - - if (!vm->inspectingLiveVirtualThreadList && (0 == vthreadInspectorCount)) { - omrthread_monitor_notify_all(vm->liveVirtualThreadListMutex); - } - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); + vmFuncs->releaseVThreadInspector(currentThread, rcv); } done: #endif /* JAVA_SPEC_VERSION >= 19 */ diff --git a/runtime/jvmti/jvmtiHelpers.c b/runtime/jvmti/jvmtiHelpers.c index d0ef04050e8..09139baef90 100644 --- a/runtime/jvmti/jvmtiHelpers.c +++ b/runtime/jvmti/jvmtiHelpers.c @@ -143,15 +143,7 @@ getVMThread(J9VMThread *currentThread, jthread thread, J9VMThread **vmThreadPtr, #if JAVA_SPEC_VERSION >= 19 isVirtualThread = IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObject); if (isVirtualThread) { - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - - while (J9OBJECT_I64_LOAD(currentThread, threadObject, vm->virtualThreadInspectorCountOffset) < 0) { - /* Thread is currently in the process of mounting/unmounting, wait. */ - vm->internalVMFunctions->internalExitVMToJNI(currentThread); - omrthread_monitor_wait(vm->liveVirtualThreadListMutex); - vm->internalVMFunctions->internalEnterVMFromJNI(currentThread); - threadObject = J9_JNI_UNWRAP_REFERENCE(thread); - } + vm->internalVMFunctions->acquireVThreadInspector(currentThread, thread, TRUE); jint vthreadState = J9VMJAVALANGVIRTUALTHREAD_STATE(currentThread, threadObject); j9object_t carrierThread = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CARRIERTHREAD(currentThread, threadObject); @@ -170,7 +162,7 @@ getVMThread(J9VMThread *currentThread, jthread thread, J9VMThread **vmThreadPtr, if (OMR_ARE_ANY_BITS_SET(flags, J9JVMTI_GETVMTHREAD_ERROR_ON_DEAD_THREAD)) { #if JAVA_SPEC_VERSION >= 19 if (isVirtualThread) { - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); + vm->internalVMFunctions->releaseVThreadInspector(currentThread, thread); } #endif /* JAVA_SPEC_VERSION >= 19 */ omrthread_monitor_exit(vm->vmThreadListMutex); @@ -182,14 +174,7 @@ getVMThread(J9VMThread *currentThread, jthread thread, J9VMThread **vmThreadPtr, if (NULL != targetThread) { targetThread->inspectorCount += 1; } -#if JAVA_SPEC_VERSION >= 19 - if (isVirtualThread) { - I_64 vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, threadObject, vm->virtualThreadInspectorCountOffset) + 1; - Assert_JVMTI_true(vthreadInspectorCount > 0); - J9OBJECT_I64_STORE(currentThread, threadObject, vm->virtualThreadInspectorCountOffset, vthreadInspectorCount); - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); - } -#endif /* JAVA_SPEC_VERSION >= 19 */ + omrthread_monitor_exit(vm->vmThreadListMutex); #if JAVA_SPEC_VERSION >= 19 @@ -211,18 +196,7 @@ releaseVMThread(J9VMThread *currentThread, J9VMThread *targetThread, jthread thr if (NULL != thread) { j9object_t threadObject = J9_JNI_UNWRAP_REFERENCE(thread); if ((currentThread->threadObject != threadObject) && IS_JAVA_LANG_VIRTUALTHREAD(currentThread, threadObject)) { - J9JavaVM *vm = currentThread->javaVM; - I_64 vthreadInspectorCount = 0; - /* Release the virtual thread (allow it to die) now that we are no longer inspecting it. */ - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, threadObject, vm->virtualThreadInspectorCountOffset); - Assert_JVMTI_true(vthreadInspectorCount > 0); - vthreadInspectorCount -= 1; - J9OBJECT_I64_STORE(currentThread, threadObject, vm->virtualThreadInspectorCountOffset, vthreadInspectorCount); - if (!vm->inspectingLiveVirtualThreadList && (0 == vthreadInspectorCount)) { - omrthread_monitor_notify_all(vm->liveVirtualThreadListMutex); - } - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); + currentThread->javaVM->internalVMFunctions->releaseVThreadInspector(currentThread, thread); } } #endif /* JAVA_SPEC_VERSION >= 19 */ diff --git a/runtime/jvmti/jvmtiThread.c b/runtime/jvmti/jvmtiThread.c index b83fafb96ff..8146aa69c83 100644 --- a/runtime/jvmti/jvmtiThread.c +++ b/runtime/jvmti/jvmtiThread.c @@ -36,6 +36,19 @@ static jvmtiError resumeThread(J9VMThread *currentThread, jthread thread); static UDATA wrappedAgentThreadStart(J9PortLibrary *portLib, void *entryArg); static void ownedMonitorIterator(J9VMThread *currentThread, J9StackWalkState *walkState, j9object_t *slot, const void *stackLocation); +#if JAVA_SPEC_VERSION >= 19 +#include "HeapIteratorAPI.h" + +typedef struct jvmtiVThreadCallBackData { + const jthread *except_list; + jint except_count; + BOOLEAN is_suspend; + BOOLEAN suspend_current_thread; +} jvmtiVThreadCallBackData; + +static jvmtiIterationControl jvmtiSuspendResumeCallBack(J9VMThread *vmThread, J9MM_IterateObjectDescriptor *object, void *userData); +#endif /* JAVA_SPEC_VERSION >= 19 */ + jvmtiError JNICALL jvmtiGetThreadState(jvmtiEnv *env, jthread thread, @@ -1289,6 +1302,58 @@ jvmtiGetCurrentThread(jvmtiEnv *env, } #if JAVA_SPEC_VERSION >= 19 +static jvmtiIterationControl +jvmtiSuspendResumeCallBack(J9VMThread *vmThread, J9MM_IterateObjectDescriptor *object, void *userData) +{ + j9object_t vthread = J9VMJDKINTERNALVMCONTINUATION_VTHREAD(vmThread, object->object); + if (NULL != vthread) { + jvmtiVThreadCallBackData *data = (jvmtiVThreadCallBackData*)userData; + BOOLEAN is_excepted = FALSE; + for (jint i = 0; i < data->except_count; ++i) { + if (vthread == J9_JNI_UNWRAP_REFERENCE(data->except_list[i])) { + is_excepted = TRUE; + break; + } + } + if (!is_excepted) { + J9JavaVM *vm = vmThread->javaVM; + J9VMThread *targetThread = NULL; + j9object_t carrierThread = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CARRIERTHREAD(vmThread, vthread); + if (NULL != carrierThread) { + targetThread = J9VMJAVALANGTHREAD_THREADREF(vmThread, carrierThread); + Assert_JVMTI_notNull(targetThread); + } + if (data->is_suspend) { + if (NULL != targetThread) { + if (OMR_ARE_NO_BITS_SET(targetThread->publicFlags, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND | J9_PUBLIC_FLAGS_STOPPED)) { + setHaltFlag(targetThread, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND); + Trc_JVMTI_threadSuspended(targetThread); + } + } else { + /* NULL carrier thread imply the virtual threadis unmounted. */ + if (0 == J9OBJECT_U32_LOAD(vmThread, vthread, vm->isSuspendedByJVMTIOffset)) { + J9OBJECT_U32_STORE(vmThread, vthread, vm->isSuspendedByJVMTIOffset, 1); + } + } + } else { + /* Resume the virtual thread. */ + if (NULL != targetThread) { + if (OMR_ARE_ANY_BITS_SET(targetThread->publicFlags, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND)) { + clearHaltFlag(targetThread, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND); + Trc_JVMTI_threadResumed(targetThread); + } + } else { + /* targetThread is NULL only for virtual threads. */ + if (0 != J9OBJECT_U32_LOAD(currentThread, vthread, vm->isSuspendedByJVMTIOffset)) { + J9OBJECT_U32_STORE(currentThread, vthread, vm->isSuspendedByJVMTIOffset, 0); + } + } + } + } + } + return JVMTI_ITERATION_CONTINUE; +} + jvmtiError JNICALL jvmtiSuspendAllVirtualThreads(jvmtiEnv *env, jint except_count, @@ -1303,8 +1368,9 @@ jvmtiSuspendAllVirtualThreads(jvmtiEnv *env, rc = getCurrentVMThread(vm, ¤tThread); if (JVMTI_ERROR_NONE == rc) { jint i = 0; - BOOLEAN currentThreadEverSuspended = FALSE; J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; + PORT_ACCESS_FROM_JAVAVM(vm); + jvmtiVThreadCallBackData data = {except_list, except_count, TRUE, FALSE}; vmFuncs->internalEnterVMFromJNI(currentThread); @@ -1327,47 +1393,12 @@ jvmtiSuspendAllVirtualThreads(jvmtiEnv *env, } /* Walk all virtual threads. */ - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - while (vm->inspectingLiveVirtualThreadList) { - /* Virtual thread list is being inspected, wait. */ - vmFuncs->internalExitVMToJNI(currentThread); - omrthread_monitor_wait(vm->liveVirtualThreadListMutex); - vmFuncs->internalEnterVMFromJNI(currentThread); - } - vm->inspectingLiveVirtualThreadList = TRUE; - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); - if (NULL != vm->liveVirtualThreadList) { - j9object_t root = *(vm->liveVirtualThreadList); - /* Skip the root, which is a dummy virtual thread and global ref. */ - j9object_t walkVirtualThread = J9OBJECT_OBJECT_LOAD(currentThread, root, vm->virtualThreadLinkNextOffset); - do { - BOOLEAN suspend = TRUE; - for (i = 0; i < except_count; ++i) { - if (walkVirtualThread == J9_JNI_UNWRAP_REFERENCE(except_list[i])) { - suspend = FALSE; - break; - } - } - if (suspend) { - BOOLEAN currentThreadSuspended = FALSE; - JNIEnv *jniEnv = (JNIEnv *)currentThread; - jobject virtualThreadRef = vmFuncs->j9jni_createLocalRef(jniEnv, walkVirtualThread); - /* Ignore errors if the virtual thread is already suspended. */ - suspendThread(currentThread, (jthread)virtualThreadRef, FALSE, ¤tThreadSuspended); - walkVirtualThread = J9_JNI_UNWRAP_REFERENCE(virtualThreadRef); - vmFuncs->j9jni_deleteLocalRef(jniEnv, virtualThreadRef); - currentThreadEverSuspended |= currentThreadSuspended; - } - walkVirtualThread = J9OBJECT_OBJECT_LOAD(currentThread, walkVirtualThread, vm->virtualThreadLinkNextOffset); - } while (root != walkVirtualThread); - } - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - vm->inspectingLiveVirtualThreadList = FALSE; - omrthread_monitor_notify_all(vm->liveVirtualThreadListMutex); - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); + vmFuncs->acquireExclusiveVMAccess(currentThread); + vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(currentThread, PORTLIB, 0, jvmtiSuspendResumeCallBack, (void*)&data); + vmFuncs->releaseExclusiveVMAccess(currentThread); /* If the current thread appeared in the list (and was marked as suspended), block now until the thread is resumed. */ - if (currentThreadEverSuspended) { + if (data.suspend_current_thread) { vmFuncs->internalExitVMToJNI(currentThread); setHaltFlag(currentThread, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND); vmFuncs->internalEnterVMFromJNI(currentThread); @@ -1394,6 +1425,8 @@ jvmtiResumeAllVirtualThreads(jvmtiEnv *env, if (rc == JVMTI_ERROR_NONE) { jint i = 0; J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; + PORT_ACCESS_FROM_JAVAVM(vm); + jvmtiVThreadCallBackData data = {except_list, except_count, FALSE, FALSE}; vmFuncs->internalEnterVMFromJNI(currentThread); @@ -1416,38 +1449,9 @@ jvmtiResumeAllVirtualThreads(jvmtiEnv *env, } /* Walk all virtual threads. */ - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - while (vm->inspectingLiveVirtualThreadList) { - /* Virtual thread list is being inspected, wait. */ - vmFuncs->internalExitVMToJNI(currentThread); - omrthread_monitor_wait(vm->liveVirtualThreadListMutex); - vmFuncs->internalEnterVMFromJNI(currentThread); - } - vm->inspectingLiveVirtualThreadList = TRUE; - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); - if (NULL != vm->liveVirtualThreadList) { - j9object_t root = *(vm->liveVirtualThreadList); - /* Skip the root, which is a dummy virtual thread and global ref. */ - j9object_t walkVirtualThread = J9OBJECT_OBJECT_LOAD(currentThread, root, vm->virtualThreadLinkNextOffset); - do { - BOOLEAN resume = TRUE; - for (i = 0; i < except_count; ++i) { - if (walkVirtualThread == J9_JNI_UNWRAP_REFERENCE(except_list[i])) { - resume = FALSE; - break; - } - } - if (resume) { - /* Ignore errors if the virtual thread is already resumed. */ - resumeThread(currentThread, (jthread)&walkVirtualThread); - } - walkVirtualThread = J9OBJECT_OBJECT_LOAD(currentThread, walkVirtualThread, vm->virtualThreadLinkNextOffset); - } while (root != walkVirtualThread); - } - omrthread_monitor_enter(vm->liveVirtualThreadListMutex); - vm->inspectingLiveVirtualThreadList = FALSE; - omrthread_monitor_notify_all(vm->liveVirtualThreadListMutex); - omrthread_monitor_exit(vm->liveVirtualThreadListMutex); + vmFuncs->acquireExclusiveVMAccess(currentThread); + vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(currentThread, PORTLIB, 0, jvmtiSuspendResumeCallBack, (void*)&data); + vmFuncs->releaseExclusiveVMAccess(currentThread); done: vmFuncs->internalExitVMToJNI(currentThread); } diff --git a/runtime/oti/VMHelpers.hpp b/runtime/oti/VMHelpers.hpp index 770746175a0..cca6b70e15e 100644 --- a/runtime/oti/VMHelpers.hpp +++ b/runtime/oti/VMHelpers.hpp @@ -2251,7 +2251,6 @@ class VM_VMHelpers } } } - #endif /* JAVA_SPEC_VERSION >= 19 */ static VMINLINE void diff --git a/runtime/oti/j9nonbuilder.h b/runtime/oti/j9nonbuilder.h index 006eb68cd51..29825c91a98 100644 --- a/runtime/oti/j9nonbuilder.h +++ b/runtime/oti/j9nonbuilder.h @@ -4946,6 +4946,8 @@ typedef struct J9InternalVMFunctions { void (*freeTLS)(struct J9VMThread *currentThread, j9object_t threadObj); UDATA (*walkContinuationStackFrames)(struct J9VMThread *currentThread, struct J9VMContinuation *continuation, J9StackWalkState *walkState); UDATA (*walkAllStackFrames)(struct J9VMThread *currentThread, J9StackWalkState *walkState); + BOOLEAN (*acquireVThreadInspector)(struct J9VMThread *currentThread, jobject thread, BOOLEAN spin); + void (*releaseVThreadInspector)(struct J9VMThread *currentThread, jobject thread); #endif /* JAVA_SPEC_VERSION >= 19 */ } J9InternalVMFunctions; @@ -5818,11 +5820,6 @@ typedef struct J9JavaVM { struct J9HashTable* ensureHashedClasses; #if JAVA_SPEC_VERSION >= 19 U_64 nextTID; - j9object_t *liveVirtualThreadList; - omrthread_monitor_t liveVirtualThreadListMutex; - volatile BOOLEAN inspectingLiveVirtualThreadList; - UDATA virtualThreadLinkNextOffset; - UDATA virtualThreadLinkPreviousOffset; UDATA virtualThreadInspectorCountOffset; UDATA isSuspendedByJVMTIOffset; UDATA tlsOffset; diff --git a/runtime/oti/vm_api.h b/runtime/oti/vm_api.h index 9b738598615..de16ce4a3cd 100644 --- a/runtime/oti/vm_api.h +++ b/runtime/oti/vm_api.h @@ -4376,6 +4376,26 @@ walkContinuationStackFrames(J9VMThread *currentThread, J9VMContinuation *continu */ UDATA walkAllStackFrames(J9VMThread *currentThread, J9StackWalkState *walkState); + +/** + * @brief Acquire inspector access on VirtualThread, block until access is acquired. + * + * @param currentThread + * @param thread target VirtualThread to acquire the inspector on + * @param spin call should spin until sucessfully acquired access + * @return TRUE on success, FALSE on failure + */ +BOOLEAN +acquireVThreadInspector(J9VMThread *currentThread, jobject thread, BOOLEAN spin); + +/** + * @brief Release the inspector acquired from VirtualThread. + * + * @param currentThread + * @param thread target VirtualThread to release the inspector on + */ +void +releaseVThreadInspector(J9VMThread *currentThread, jobject thread); #endif /* JAVA_SPEC_VERSION >= 19 */ /* ---------------- hookableAsync.c ---------------- */ diff --git a/runtime/oti/vmconstantpool.xml b/runtime/oti/vmconstantpool.xml index a74ce474ea6..c78701eece7 100644 --- a/runtime/oti/vmconstantpool.xml +++ b/runtime/oti/vmconstantpool.xml @@ -416,6 +416,7 @@ SPDX-License-Identifier: EPL-2.0 OR Apache-2.0 OR GPL-2.0 WITH Classpath-excepti + diff --git a/runtime/vm/ContinuationHelpers.cpp b/runtime/vm/ContinuationHelpers.cpp index f184f7bdfee..fbd6e73bda2 100644 --- a/runtime/vm/ContinuationHelpers.cpp +++ b/runtime/vm/ContinuationHelpers.cpp @@ -380,4 +380,64 @@ walkAllStackFrames(J9VMThread *currentThread, J9StackWalkState *walkState) (void*)walkState); return rc; } + +BOOLEAN +acquireVThreadInspector(J9VMThread *currentThread, jobject thread, BOOLEAN spin) +{ + J9JavaVM *vm = currentThread->javaVM; + J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; + MM_ObjectAccessBarrierAPI objectAccessBarrier = MM_ObjectAccessBarrierAPI(currentThread); + j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + I_64 vthreadInspectorCount; +retry: + vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset); + if (vthreadInspectorCount < 0) { + /* Thread is in transition, wait. */ + vmFuncs->internalExitVMToJNI(currentThread); + VM_AtomicSupport::yieldCPU(); + /* After wait, the thread may suspend here. */ + vmFuncs->internalEnterVMFromJNI(currentThread); + threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + if (spin) { + goto retry; + } else { + return FALSE; + } + } else if (!objectAccessBarrier.inlineMixedObjectCompareAndSwapU64( + currentThread, + threadObj, + vm->virtualThreadInspectorCountOffset, + (U_64)vthreadInspectorCount, + ((U_64)(vthreadInspectorCount + 1))) + ) { + /* Field updated by another thread, try again. */ + if (spin) { + goto retry; + } else { + return FALSE; + } + } + return TRUE; +} + +void +releaseVThreadInspector(J9VMThread *currentThread, jobject thread) +{ + J9JavaVM *vm = currentThread->javaVM; + MM_ObjectAccessBarrierAPI objectAccessBarrier = MM_ObjectAccessBarrierAPI(currentThread); + j9object_t threadObj = J9_JNI_UNWRAP_REFERENCE(thread); + I_64 vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset); + /* vthreadInspectorCount must be greater than 0 before decrement. */ + Assert_VM_true(vthreadInspectorCount > 0); + while (!objectAccessBarrier.inlineMixedObjectCompareAndSwapU64( + currentThread, + threadObj, + vm->virtualThreadInspectorCountOffset, + (U_64)vthreadInspectorCount, + ((U_64)(vthreadInspectorCount - 1))) + ) { + /* Field updated by another thread, try again. */ + vthreadInspectorCount = J9OBJECT_I64_LOAD(currentThread, threadObj, vm->virtualThreadInspectorCountOffset); + } +} } /* extern "C" */ diff --git a/runtime/vm/intfunc.c b/runtime/vm/intfunc.c index c107665e516..609a2f6a845 100644 --- a/runtime/vm/intfunc.c +++ b/runtime/vm/intfunc.c @@ -433,5 +433,7 @@ J9InternalVMFunctions J9InternalFunctions = { freeTLS, walkContinuationStackFrames, walkAllStackFrames, + acquireVThreadInspector, + releaseVThreadInspector, #endif /* JAVA_SPEC_VERSION >= 19 */ }; diff --git a/runtime/vm/vmthinit.c b/runtime/vm/vmthinit.c index 6cdb3d76c8d..fa846b39a48 100644 --- a/runtime/vm/vmthinit.c +++ b/runtime/vm/vmthinit.c @@ -92,7 +92,6 @@ UDATA initializeVMThreading(J9JavaVM *vm) #if JAVA_SPEC_VERSION >= 19 /* Held when adding or removing a virtual thread from the list at virtual thread start or terminate. */ - omrthread_monitor_init_with_name(&vm->liveVirtualThreadListMutex, 0, "Live virtual thread list mutex") || omrthread_monitor_init_with_name(&vm->tlsFinalizersMutex, 0, "TLS finalizers mutex") || omrthread_monitor_init_with_name(&vm->tlsPoolMutex, 0, "TLS pool mutex") || #endif /* JAVA_SPEC_VERSION >= 19 */ @@ -197,10 +196,6 @@ void terminateVMThreading(J9JavaVM *vm) #endif /* JAVA_SPEC_VERSION >= 16 */ #if JAVA_SPEC_VERSION >= 19 - if (NULL != vm->liveVirtualThreadListMutex) { - omrthread_monitor_destroy(vm->liveVirtualThreadListMutex); - vm->liveVirtualThreadListMutex = NULL; - } if (NULL != vm->tlsFinalizersMutex) { omrthread_monitor_destroy(vm->tlsFinalizersMutex); vm->tlsFinalizersMutex = NULL; From 82b9835c281a1974f6dbd6f2e8a8f95260875c79 Mon Sep 17 00:00:00 2001 From: Jack Lu Date: Thu, 16 Mar 2023 17:16:27 -0400 Subject: [PATCH 2/6] Fix vthread suspend order to avoid dispatching mount hook for suspended thread Signed-off-by: Jack Lu --- runtime/j9vm/javanextvmi.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/runtime/j9vm/javanextvmi.cpp b/runtime/j9vm/javanextvmi.cpp index 20270e85e03..2dddeb6ff70 100644 --- a/runtime/j9vm/javanextvmi.cpp +++ b/runtime/j9vm/javanextvmi.cpp @@ -333,23 +333,23 @@ JVM_VirtualThreadMountEnd(JNIEnv *env, jobject thread, jboolean firstMount) J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, continuationObj)); } - /* Allow thread to be inspected again. */ - exitVThreadTransitionCritical(currentThread, threadObj); - - if (firstMount) { - TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_STARTED(vm->hookInterface, currentThread); - } - TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_MOUNT(vm->hookInterface, currentThread); - /* If isSuspendedByJVMTI is non-zero, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND is set * in currentThread->publicFlags while resetting isSuspendedByJVMTI to zero. During * mount, this suspends the thread if the thread was unmounted when JVMTI suspended it. */ if (0 != J9OBJECT_U32_LOAD(currentThread, threadObj, vm->isSuspendedByJVMTIOffset)) { - J9OBJECT_U32_STORE(currentThread, threadObj, vm->isSuspendedByJVMTIOffset, 0); vmFuncs->setHaltFlag(currentThread, J9_PUBLIC_FLAGS_HALT_THREAD_JAVA_SUSPEND); + J9OBJECT_U32_STORE(currentThread, threadObj, vm->isSuspendedByJVMTIOffset, 0); } + /* Allow thread to be inspected again. */ + exitVThreadTransitionCritical(currentThread, threadObj); + + if (firstMount) { + TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_STARTED(vm->hookInterface, currentThread); + } + TRIGGER_J9HOOK_VM_VIRTUAL_THREAD_MOUNT(vm->hookInterface, currentThread); + vmFuncs->internalExitVMToJNI(currentThread); Trc_SC_VirtualThreadMountEnd_Exit(currentThread, thread, firstMount); From 69b01b519eef42d31b46d4281c4040e56573f893 Mon Sep 17 00:00:00 2001 From: Jack Lu Date: Wed, 8 Mar 2023 14:46:53 -0500 Subject: [PATCH 3/6] Change GC continuation list option default to enable Signed-off-by: Jack Lu --- runtime/gc_base/GCExtensions.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/gc_base/GCExtensions.hpp b/runtime/gc_base/GCExtensions.hpp index 7a5dce2d90a..6556f041733 100644 --- a/runtime/gc_base/GCExtensions.hpp +++ b/runtime/gc_base/GCExtensions.hpp @@ -356,7 +356,7 @@ class MM_GCExtensions : public MM_GCExtensionsBase { , minimumFreeSizeForSurvivor(DEFAULT_SURVIVOR_MINIMUM_FREESIZE) , freeSizeThresholdForSurvivor(DEFAULT_SURVIVOR_THRESHOLD) , recycleRemainders(true) - , continuationListOption(verify_continuation_list) + , continuationListOption(enable_continuation_list) { _typeId = __FUNCTION__; } From ee5c02b396e0d10ffaa2bbd2b9f7e640a06b6e87 Mon Sep 17 00:00:00 2001 From: Jack Lu Date: Mon, 20 Mar 2023 21:25:47 -0400 Subject: [PATCH 4/6] Forces cache flush before walking Continuation list Signed-off-by: Jack Lu --- runtime/compiler/control/HookedByTheJit.cpp | 1 + runtime/jvmti/jvmtiThread.c | 2 ++ runtime/vm/ContinuationHelpers.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index fa3517a19d9..369ceea6895 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -6557,6 +6557,7 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu do { + vm->memoryManagerFunctions->j9gc_flush_nonAllocationCaches_for_walk(vm); jvmtiIterationControl rc = vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(vmThread, PORTLIB, 0, jitWalkContinuationCallBack, (void*)condYield); if (JVMTI_ITERATION_ABORT == rc) yieldHappened = true; diff --git a/runtime/jvmti/jvmtiThread.c b/runtime/jvmti/jvmtiThread.c index 8146aa69c83..e4f5cbb91f2 100644 --- a/runtime/jvmti/jvmtiThread.c +++ b/runtime/jvmti/jvmtiThread.c @@ -1394,6 +1394,7 @@ jvmtiSuspendAllVirtualThreads(jvmtiEnv *env, /* Walk all virtual threads. */ vmFuncs->acquireExclusiveVMAccess(currentThread); + vm->memoryManagerFunctions->j9gc_flush_nonAllocationCaches_for_walk(vm); vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(currentThread, PORTLIB, 0, jvmtiSuspendResumeCallBack, (void*)&data); vmFuncs->releaseExclusiveVMAccess(currentThread); @@ -1450,6 +1451,7 @@ jvmtiResumeAllVirtualThreads(jvmtiEnv *env, /* Walk all virtual threads. */ vmFuncs->acquireExclusiveVMAccess(currentThread); + vm->memoryManagerFunctions->j9gc_flush_nonAllocationCaches_for_walk(vm); vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects(currentThread, PORTLIB, 0, jvmtiSuspendResumeCallBack, (void*)&data); vmFuncs->releaseExclusiveVMAccess(currentThread); done: diff --git a/runtime/vm/ContinuationHelpers.cpp b/runtime/vm/ContinuationHelpers.cpp index fbd6e73bda2..7ec469c6789 100644 --- a/runtime/vm/ContinuationHelpers.cpp +++ b/runtime/vm/ContinuationHelpers.cpp @@ -372,6 +372,7 @@ walkAllStackFrames(J9VMThread *currentThread, J9StackWalkState *walkState) /* Walk all live continuation stacks using the GC Continuation object iterator */ PORT_ACCESS_FROM_VMC(currentThread); + vm->memoryManagerFunctions->j9gc_flush_nonAllocationCaches_for_walk(vm); vm->memoryManagerFunctions->j9mm_iterate_all_continuation_objects( currentThread, PORTLIB, From b5f3a0ca09f77edbddf5ffaf7cb1a5b0fefd8bf1 Mon Sep 17 00:00:00 2001 From: Jack Lu Date: Thu, 30 Mar 2023 13:56:51 -0400 Subject: [PATCH 5/6] Refactor freeContinuation assert Continuation must be unmounted/not being scanned when free. Signed-off-by: Jack Lu --- runtime/vm/ContinuationHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/vm/ContinuationHelpers.cpp b/runtime/vm/ContinuationHelpers.cpp index 7ec469c6789..760cada2bf1 100644 --- a/runtime/vm/ContinuationHelpers.cpp +++ b/runtime/vm/ContinuationHelpers.cpp @@ -261,7 +261,7 @@ freeContinuation(J9VMThread *currentThread, j9object_t continuationObject) currentStack = previous; } while (NULL != currentStack); - Assert_VM_true(VM_VMHelpers::isFinished(continuation->state)); + Assert_VM_true(!VM_VMHelpers::isConcurrentlyScanned(continuation->state) && (NULL == VM_VMHelpers::getCarrierThread(continuation->state))); /* Free the J9VMContinuation struct */ j9mem_free_memory(continuation); From 3140c22eb1d9a702e923534352735a771235ae24 Mon Sep 17 00:00:00 2001 From: Jack Lu Date: Fri, 31 Mar 2023 16:07:11 -0400 Subject: [PATCH 6/6] Correctly ifdef Java19 code Signed-off-by: Jack Lu --- runtime/compiler/control/HookedByTheJit.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index 369ceea6895..8705202f964 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -34,7 +34,9 @@ #include "mmhook.h" #include "mmomrhook.h" #include "vmaccess.h" +#if JAVA_SPEC_VERSION >= 19 #include "HeapIteratorAPI.h" +#endif /* JAVA_SPEC_VERSION >= 19 */ #include "codegen/CodeGenerator.hpp" #include "compile/CompilationTypes.hpp" #include "compile/Method.hpp" @@ -6460,6 +6462,7 @@ static UDATA jitReleaseCodeStackWalkFrame(J9VMThread *vmThread, J9StackWalkState return J9_STACKWALK_KEEP_ITERATING; } +#if JAVA_SPEC_VERSION >= 19 static jvmtiIterationControl jitWalkContinuationCallBack(J9VMThread *vmThread, J9MM_IterateObjectDescriptor *object, void *userData) { J9InternalVMFunctions *vmFuncs = vmThread->javaVM->internalVMFunctions; @@ -6498,6 +6501,7 @@ static jvmtiIterationControl jitResetContinuationFlag(J9VMThread *vmThread, J9MM return JVMTI_ITERATION_CONTINUE; } +#endif /* JAVA_SPEC_VERSION >= 19 */ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFunctionPtr condYield = NULL) {