diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index 36119a8111d..ea8c42537dd 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -6469,10 +6469,89 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu return; // nothing to do + bool isRealTimeGC = TR::Options::getCmdLineOptions()->realTimeGC(); +#if JAVA_SPEC_VERSION >= 19 + J9JavaVM *vm = vmThread->javaVM; + J9InternalVMFunctions *vmFuncs = vm->internalVMFunctions; + if (isRealTimeGC && !TR::Options::getCmdLineOptions()->getOption(TR_DisableIncrementalCCR)) + { + bool yieldHappened = false; + do + { + J9VMThread *thread = vmThread; + yieldHappened = false; + do + { + if ((thread->dropFlags & 0x1) ? false : true) + { + J9StackWalkState walkState; + walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; + walkState.skipCount = 0; + walkState.frameWalkFunction = jitReleaseCodeStackWalkFrame; + walkState.walkThread = thread; + vm->walkStackFrames(vmThread, &walkState); + thread->dropFlags |= 0x1; + } + + if ((NULL!= thread->currentContinuation) && ((thread->currentContinuation->dropFlags & 0x1) ? false : true)) + /* If a continuation is mounted, always walk the continuation as that represent the CarrierThread */ + { + J9StackWalkState walkState; + walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; + walkState.skipCount = 0; + walkState.frameWalkFunction = jitReleaseCodeStackWalkFrame; + vmFuncs->walkContinuationStackFrames(vmThread, thread->currentContinuation, &walkState); + thread->currentContinuation->dropFlags |= 0x1; + } + + yieldHappened = condYield(omrVMThread, J9_GC_METRONOME_UTILIZATION_COMPONENT_JIT); + + if (!yieldHappened) + thread = thread->linkNext; + } + while ((thread != vmThread) && !yieldHappened); + } + while (yieldHappened); + + if (NULL != vm->liveVirtualThreadList) + { + do + { + yieldHappened = false; + /* Skip the root, which is a dummy virtual thread and global ref. */ + j9object_t walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, *(vm->liveVirtualThreadList), vm->virtualThreadLinkNextOffset); + while ((*(vm->liveVirtualThreadList) != walkVirtualThread) && !yieldHappened) + { + j9object_t contObject = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CONT(vmThread, walkVirtualThread); + J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(vmThread, contObject); + if ((continuation->dropFlags & 0x1) ? false : true) + { + J9StackWalkState walkState; + walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; + walkState.skipCount = 0; + walkState.frameWalkFunction = jitReleaseCodeStackWalkFrame; + vmFuncs->walkContinuationStackFrames(vmThread, continuation, &walkState); + continuation->dropFlags |= 0x1; + yieldHappened = condYield(omrVMThread, J9_GC_METRONOME_UTILIZATION_COMPONENT_JIT); + } + + if (!yieldHappened) + walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, walkVirtualThread, vm->virtualThreadLinkNextOffset); + } + } + while (yieldHappened); + } + } else { + J9StackWalkState walkState; + walkState.flags = J9_STACKWALK_ITERATE_HIDDEN_JIT_FRAMES | J9_STACKWALK_ITERATE_FRAMES | J9_STACKWALK_SKIP_INLINES; + walkState.skipCount = 0; + walkState.frameWalkFunction = jitReleaseCodeStackWalkFrame; + + vmFuncs->walkAllStackFrames(vmThread, &walkState); + } +#else /* JAVA_SPEC_VERSION >= 19 */ bool yieldHappened = false; bool doStackWalkForThread = true; - - bool isRealTimeGC = TR::Options::getCmdLineOptions()->realTimeGC(); do { J9VMThread *thread = vmThread; @@ -6504,6 +6583,7 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu while ((thread != vmThread) && !yieldHappened); } while (yieldHappened); +#endif /* JAVA_SPEC_VERSION >= 19 */ TR::CompilationInfo * compInfo = TR::CompilationInfo::get(jitConfig); @@ -6583,9 +6663,28 @@ static void jitReleaseCodeStackWalk(OMR_VMThread *omrVMThread, condYieldFromGCFu do { thr->dropFlags &=0x0; +#if JAVA_SPEC_VERSION >= 19 + if (NULL != thr->currentContinuation) { + thr->currentContinuation->dropFlags &=0x0; + } +#endif /* JAVA_SPEC_VERSION >= 19 */ thr = thr->linkNext; } while (thr != vmThread); +#if JAVA_SPEC_VERSION >= 19 + if (NULL != vm->liveVirtualThreadList) + { + /* Skip the root, which is a dummy virtual thread and global ref. */ + j9object_t walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, *(vm->liveVirtualThreadList), vm->virtualThreadLinkNextOffset); + while (*(vm->liveVirtualThreadList) != walkVirtualThread) + { + j9object_t contObject = (j9object_t)J9VMJAVALANGVIRTUALTHREAD_CONT(vmThread, walkVirtualThread); + J9VMContinuation *continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(vmThread, contObject); + continuation->dropFlags &= 0x0; + walkVirtualThread = J9OBJECT_OBJECT_LOAD(vmThread, walkVirtualThread, vm->virtualThreadLinkNextOffset); + } + } +#endif /* JAVA_SPEC_VERSION >= 19 */ } diff --git a/runtime/oti/j9nonbuilder.h b/runtime/oti/j9nonbuilder.h index 03228740411..1979b69c565 100644 --- a/runtime/oti/j9nonbuilder.h +++ b/runtime/oti/j9nonbuilder.h @@ -4942,6 +4942,7 @@ typedef struct J9InternalVMFunctions { void (*freeContinuation)(struct J9VMThread *currentThread, j9object_t continuationObject); void (*freeTLS)(struct J9VMThread *currentThread, j9object_t threadObj); UDATA (*walkContinuationStackFrames)(struct J9VMThread *currentThread, struct J9VMContinuation *continuation, J9StackWalkState *walkState); + UDATA (*walkAllStackFrames)(struct J9VMThread *currentThread, J9StackWalkState *walkState); #endif /* JAVA_SPEC_VERSION >= 19 */ } J9InternalVMFunctions; @@ -5031,6 +5032,7 @@ typedef struct J9VMContinuation { struct J9I2JState i2jState; struct J9VMEntryLocalStorage* oldEntryLocalStorage; volatile UDATA state; /* it's a bit-wise struct of CarrierThread ID and ConcurrentlyScanned flag */ + UDATA dropFlags; } J9VMContinuation; #endif /* JAVA_SPEC_VERSION >= 19 */ diff --git a/runtime/oti/vm_api.h b/runtime/oti/vm_api.h index 3ee192d399a..19f6dde8a93 100644 --- a/runtime/oti/vm_api.h +++ b/runtime/oti/vm_api.h @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 1991, 2022 IBM Corp. and others + * Copyright (c) 1991, 2023 IBM Corp. and others * * This program and the accompanying materials are made available under * the terms of the Eclipse Public License 2.0 which accompanies this @@ -4367,6 +4367,16 @@ copyFieldsFromContinuation(J9VMThread *currentThread, J9VMThread *vmThread, J9VM */ UDATA walkContinuationStackFrames(J9VMThread *currentThread, J9VMContinuation *continuation, J9StackWalkState *walkState); + +/** + * @brief Walk all stackframes in the VM. + * + * @param currentThread + * @param walkState walkstate holding initial walk parameters to be used in each stackwalk + * @return 0 on success and non-zero on failure + */ +UDATA +walkAllStackFrames(J9VMThread *currentThread, J9StackWalkState *walkState); #endif /* JAVA_SPEC_VERSION >= 19 */ /* ---------------- hookableAsync.c ---------------- */ diff --git a/runtime/vm/ContinuationHelpers.cpp b/runtime/vm/ContinuationHelpers.cpp index 4127f225734..a0bb1bf0a87 100644 --- a/runtime/vm/ContinuationHelpers.cpp +++ b/runtime/vm/ContinuationHelpers.cpp @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2022, 2022 IBM Corp. and others + * Copyright (c) 2022, 2023 IBM Corp. and others * * This program and the accompanying materials are made available under * the terms of the Eclipse Public License 2.0 which accompanies this @@ -286,4 +286,65 @@ walkContinuationStackFrames(J9VMThread *currentThread, J9VMContinuation *continu return rc; } + +UDATA +walkAllStackFrames(J9VMThread *currentThread, J9StackWalkState *walkState) +{ + J9JavaVM *vm = currentThread->javaVM; + J9StackWalkState localWalkState = {0}; + UDATA rc = J9_STACKWALK_RC_NONE; + + Assert_VM_true((J9_XACCESS_EXCLUSIVE == vm->exclusiveAccessState) || (J9_XACCESS_EXCLUSIVE == vm->safePointState)); + + /* Walk all vmThread stacks */ + J9VMThread *targetThread = vm->mainThread; + do { + /* Reset localWalkState */ + localWalkState = *walkState; + localWalkState.walkThread = targetThread; + rc = vm->walkStackFrames(currentThread, &localWalkState); + + if (NULL != targetThread->currentContinuation) { + localWalkState = *walkState; + walkContinuationStackFrames(currentThread, targetThread->currentContinuation, &localWalkState); + } + targetThread = targetThread->linkNext; + } while (targetThread != vm->mainThread); + + /* Walk all contination stacks */ + if (NULL != vm->liveVirtualThreadList) { + j9object_t rootNode = J9_JNI_UNWRAP_REFERENCE(vm->liveVirtualThreadList); + j9object_t nextVThread = J9OBJECT_OBJECT_LOAD(currentThread, rootNode, vm->virtualThreadLinkNextOffset); + J9VMContinuation *continuation = NULL; + while (nextVThread != rootNode) { + j9object_t cont = J9VMJAVALANGVIRTUALTHREAD_CONT(currentThread, nextVThread); + continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, cont); + if (NULL != continuation) { + /* Reset localWalkState */ + localWalkState = *walkState; + /* walk live continuation's stack */ + rc = walkContinuationStackFrames(currentThread, continuation, &localWalkState); + } + nextVThread = J9OBJECT_OBJECT_LOAD(currentThread, nextVThread, vm->virtualThreadLinkNextOffset); + } + } + /* Alternate approach to rely on GC Continuation object iterator + GC_ContinuationIterator iter = getContinuationIterator(); + J9VMContinuation *continuation = NULL; + while(NULL != iter) { + continuation = J9VMJDKINTERNALVMCONTINUATION_VMREF(currentThread, nextCont); + if (NULL != continuation) { + // Reset localWalkState + localWalkState = *walkState; + // walk live continuation's stack + rc = walkContinuationStackFrames(currentThread, continuation, &localWalkState); + if (J9_STACKWALK_RC_NONE != rc) { + goto exit; + } + } + iter = iter.next(); + } + */ + return rc; +} } /* extern "C" */ diff --git a/runtime/vm/ContinuationHelpers.hpp b/runtime/vm/ContinuationHelpers.hpp index d9092a6b321..40d86e6812b 100644 --- a/runtime/vm/ContinuationHelpers.hpp +++ b/runtime/vm/ContinuationHelpers.hpp @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2022, 2022 IBM Corp. and others + * Copyright (c) 2022, 2023 IBM Corp. and others * * This program and the accompanying materials are made available under * the terms of the Eclipse Public License 2.0 which accompanies this @@ -69,6 +69,7 @@ class VM_ContinuationHelpers { SWAP_MEMBER(stackObject, J9JavaStack*, vmThread, continuation); SWAP_MEMBER(decompilationStack, J9JITDecompilationInfo*, vmThread, continuation); SWAP_MEMBER(j2iFrame, UDATA*, vmThread, continuation); + SWAP_MEMBER(dropFlags, UDATA, vmThread, continuation); J9VMEntryLocalStorage *threadELS = vmThread->entryLocalStorage; /* Swap the JIT GPR registers data referenced by ELS */ diff --git a/runtime/vm/intfunc.c b/runtime/vm/intfunc.c index 11f80cd119f..c107665e516 100644 --- a/runtime/vm/intfunc.c +++ b/runtime/vm/intfunc.c @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 1991, 2022 IBM Corp. and others + * Copyright (c) 1991, 2023 IBM Corp. and others * * This program and the accompanying materials are made available under * the terms of the Eclipse Public License 2.0 which accompanies this @@ -432,5 +432,6 @@ J9InternalVMFunctions J9InternalFunctions = { freeContinuation, freeTLS, walkContinuationStackFrames, + walkAllStackFrames, #endif /* JAVA_SPEC_VERSION >= 19 */ };