[wasm-mt] Support async JS interop on threadpool threads

src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems

@@ -2529,9 +2529,9 @@
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.ThreadCounts.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WaitThread.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerThread.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerThread.NonBrowser.cs" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerThread.NonBrowser.cs" Condition="'$(TargetsBrowser)' != 'true'"/>
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.WorkerTracking.cs" />
-    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.Unix.cs" Condition="'$(TargetsUnix)' == 'true' or '$(TargetsBrowser)' == 'true' or '$(TargetsWasi)' == 'true'" />
+    <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.Unix.cs" Condition="'$(TargetsUnix)' == 'true' or ('$(TargetsBrowser)' == 'true' and '$(FeatureWasmThreads)' != 'true') or '$(TargetsWasi)' == 'true'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\PortableThreadPool.Windows.cs" Condition="'$(TargetsWindows)' == 'true'" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\LowLevelLifoSemaphore.cs" />
     <Compile Include="$(MSBuildThisFileDirectory)System\Threading\LowLevelLifoSemaphore.Windows.cs" Condition="'$(TargetsWindows)' == 'true'" />

src/libraries/System.Private.CoreLib/src/System/Threading/ThreadPool.Portable.cs

@@ -19,7 +19,9 @@ public static partial class ThreadPool
     {
         // Indicates whether the thread pool should yield the thread from the dispatch loop to the runtime periodically so that
         // the runtime may use the thread for processing other work
+#if !(TARGET_BROWSER && FEATURE_WASM_THREADS)
         internal static bool YieldFromDispatchLoop => false;
+#endif
 
 #if NATIVEAOT
         private const bool IsWorkerTrackingEnabledInConfig = false;

src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Interop/LegacyExports.cs

@@ -32,18 +32,15 @@ internal static void PreventTrimming()
 
         public static void GetCSOwnedObjectByJSHandleRef(nint jsHandle, int shouldAddInflight, out JSObject? result)
         {
-            lock (JSHostImplementation.s_csOwnedObjects)
+            if (JSHostImplementation.ThreadCsOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference))
             {
-                if (JSHostImplementation.s_csOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference))
+                reference.TryGetTarget(out JSObject? jsObject);
+                if (shouldAddInflight != 0)
                 {
-                    reference.TryGetTarget(out JSObject? jsObject);
-                    if (shouldAddInflight != 0)
-                    {
-                        jsObject?.AddInFlight();
-                    }
-                    result = jsObject;
-                    return;
+                    jsObject?.AddInFlight();
                 }
+                result = jsObject;
+                return;
             }
             result = null;
         }
@@ -77,14 +74,12 @@ public static void CreateCSOwnedProxyRef(nint jsHandle, LegacyHostImplementation
 
             JSObject? res = null;
 
-            lock (JSHostImplementation.s_csOwnedObjects)
+            if (!JSHostImplementation.ThreadCsOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
+                !reference.TryGetTarget(out res) ||
+                res.IsDisposed)
             {
-                if (!JSHostImplementation.s_csOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
-                    !reference.TryGetTarget(out res) ||
-                    res.IsDisposed)
-                {
 #pragma warning disable CS0612 // Type or member is obsolete
-                    res = mappedType switch
+                res = mappedType switch
                     {
                         LegacyHostImplementation.MappedType.JSObject => new JSObject(jsHandle),
                         LegacyHostImplementation.MappedType.Array => new Array(jsHandle),
@@ -95,8 +90,7 @@ public static void CreateCSOwnedProxyRef(nint jsHandle, LegacyHostImplementation
                         _ => throw new ArgumentOutOfRangeException(nameof(mappedType))
                     };
 #pragma warning restore CS0612 // Type or member is obsolete
-                    JSHostImplementation.s_csOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
-                }
+                JSHostImplementation.ThreadCsOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
             }
             if (shouldAddInflight != 0)
             {

src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.cs

@@ -15,7 +15,20 @@ internal static partial class JSHostImplementation
         private const string TaskGetResultName = "get_Result";
         private static MethodInfo? s_taskGetResultMethodInfo;
         // we use this to maintain identity of JSHandle for a JSObject proxy
-        public static readonly Dictionary<int, WeakReference<JSObject>> s_csOwnedObjects = new Dictionary<int, WeakReference<JSObject>>();
+#if FEATURE_WASM_THREADS
+        [ThreadStatic]
+#endif
+        private static Dictionary<int, WeakReference<JSObject>>? s_csOwnedObjects;
+
+        public static Dictionary<int, WeakReference<JSObject>> ThreadCsOwnedObjects
+        {
+            get
+            {
+                s_csOwnedObjects ??= new ();
+                return s_csOwnedObjects;
+            }
+        }
+
         // we use this to maintain identity of GCHandle for a managed object
         public static Dictionary<object, IntPtr> s_gcHandleFromJSOwnedObject = new Dictionary<object, IntPtr>(ReferenceEqualityComparer.Instance);
 
@@ -24,10 +37,7 @@ public static void ReleaseCSOwnedObject(nint jsHandle)
         {
             if (jsHandle != IntPtr.Zero)
             {
-                lock (s_csOwnedObjects)
-                {
-                    s_csOwnedObjects.Remove((int)jsHandle);
-                }
+                ThreadCsOwnedObjects.Remove((int)jsHandle);
                 Interop.Runtime.ReleaseCSOwnedObject(jsHandle);
             }
         }
@@ -175,17 +185,14 @@ public static unsafe void FreeMethodSignatureBuffer(JSFunctionBinding signature)
 
         public static JSObject CreateCSOwnedProxy(nint jsHandle)
         {
-            JSObject? res = null;
+            JSObject? res;
 
-            lock (s_csOwnedObjects)
+            if (!ThreadCsOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
+                !reference.TryGetTarget(out res) ||
+                res.IsDisposed)
             {
-                if (!s_csOwnedObjects.TryGetValue((int)jsHandle, out WeakReference<JSObject>? reference) ||
-                    !reference.TryGetTarget(out res) ||
-                    res.IsDisposed)
-                {
-                    res = new JSObject(jsHandle);
-                    s_csOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
-                }
+                res = new JSObject(jsHandle);
+                ThreadCsOwnedObjects[(int)jsHandle] = new WeakReference<JSObject>(res, trackResurrection: true);
             }
             return res;
         }

src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Legacy/LegacyHostImplementation.cs

@@ -21,10 +21,7 @@ public static void ReleaseInFlight(object obj)
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static void RegisterCSOwnedObject(JSObject proxy)
         {
-            lock (JSHostImplementation.s_csOwnedObjects)
-            {
-                JSHostImplementation.s_csOwnedObjects[(int)proxy.JSHandle] = new WeakReference<JSObject>(proxy, trackResurrection: true);
-            }
+            JSHostImplementation.ThreadCsOwnedObjects[(int)proxy.JSHandle] = new WeakReference<JSObject>(proxy, trackResurrection: true);
         }
 
         public static MarshalType GetMarshalTypeFromType(Type type)

src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj

@@ -281,6 +281,9 @@
   <ItemGroup Condition="('$(TargetsBrowser)' == 'true'  or '$(TargetsWasi)' == 'true') and '$(FeatureWasmThreads)' == 'true'">
       <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPoolBoundHandle.Browser.Threads.Mono.cs" />
       <Compile Include="$(BclSourcesRoot)\System\Threading\LowLevelLifoAsyncWaitSemaphore.Browser.Threads.Mono.cs" />
+      <Compile Include="$(BclSourcesRoot)\System\Threading\PortableThreadPool.Browser.Threads.Mono.cs" />
+      <Compile Include="$(BclSourcesRoot)\System\Threading\PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs" />
+      <Compile Include="$(BclSourcesRoot)\System\Threading\ThreadPool.Browser.Threads.Mono.cs" />
       <Compile Include="$(BclSourcesRoot)\System\Threading\WebWorkerEventLoop.Browser.Threads.Mono.cs" />
   </ItemGroup>
   <ItemGroup Condition="('$(TargetsBrowser)' == 'true' or '$(TargetsWasi)' == 'true') and '$(FeatureWasmThreads)' != 'true'">

src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.Browser.Threads.Mono.cs

@@ -0,0 +1,19 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Threading;
+
+internal sealed partial class PortableThreadPool
+{
+    private static partial class WorkerThread
+    {
+        private static bool IsIOPending => WebWorkerEventLoop.HasJavaScriptInteropDependents;
+    }
+
+    private struct CpuUtilizationReader
+    {
+#pragma warning disable CA1822
+        public double CurrentUtilization => 0.0; // FIXME: can we do better
+#pragma warning restore CA1822
+    }
+}

src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs

@@ -0,0 +1,122 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.Tracing;
+using System.Runtime.CompilerServices;
+
+namespace System.Threading
+{
+    internal sealed partial class PortableThreadPool
+    {
+        /// <summary>
+        /// The worker thread infastructure for the CLR thread pool.
+        /// </summary>
+        private static partial class WorkerThread
+        {
+            /// <summary>
+            /// Semaphore for controlling how many threads are currently working.
+            /// </summary>
+            private static readonly LowLevelLifoAsyncWaitSemaphore s_semaphore =
+                new LowLevelLifoAsyncWaitSemaphore(
+                    0,
+                    MaxPossibleThreadCount,
+                    AppContextConfigHelper.GetInt32Config(
+                        "System.Threading.ThreadPool.UnfairSemaphoreSpinLimit",
+                        SemaphoreSpinCountDefault,
+                        false),
+                    onWait: () =>
+                    {
+                        if (NativeRuntimeEventSource.Log.IsEnabled())
+                        {
+                            NativeRuntimeEventSource.Log.ThreadPoolWorkerThreadWait(
+                                (uint)ThreadPoolInstance._separated.counts.VolatileRead().NumExistingThreads);
+                        }
+                    });
+
+            private static readonly ThreadStart s_workerThreadStart = WorkerThreadStart;
+
+            private sealed record SemaphoreWaitState(PortableThreadPool ThreadPoolInstance, LowLevelLock ThreadAdjustmentLock, WebWorkerEventLoop.KeepaliveToken KeepaliveToken)
+            {
+                public bool SpinWait = true;
+
+                public void ResetIteration() {
+                    SpinWait = true;
+                }
+            }
+
+            private static void WorkerThreadStart()
+            {
+                Thread.CurrentThread.SetThreadPoolWorkerThreadName();
+
+                PortableThreadPool threadPoolInstance = ThreadPoolInstance;
+
+                if (NativeRuntimeEventSource.Log.IsEnabled())
+                {
+                    NativeRuntimeEventSource.Log.ThreadPoolWorkerThreadStart(
+                        (uint)threadPoolInstance._separated.counts.VolatileRead().NumExistingThreads);
+                }
+
+                LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock;
+                var keepaliveToken = WebWorkerEventLoop.KeepalivePush();
+                SemaphoreWaitState state = new(threadPoolInstance, threadAdjustmentLock, keepaliveToken) { SpinWait = true };
+                // set up the callbacks for semaphore waits, tell
+                // emscripten to keep the thread alive, and return to
+                // the JS event loop.
+                WaitForWorkLoop(s_semaphore, state);
+                // return from thread start with keepalive - the thread will stay alive in the JS event loop
+            }
+
+            private static readonly Action<LowLevelLifoAsyncWaitSemaphore, object?> s_WorkLoopSemaphoreSuccess = new(WorkLoopSemaphoreSuccess);
+            private static readonly Action<LowLevelLifoAsyncWaitSemaphore, object?> s_WorkLoopSemaphoreTimedOut = new(WorkLoopSemaphoreTimedOut);
+
+            private static void WaitForWorkLoop(LowLevelLifoAsyncWaitSemaphore semaphore, SemaphoreWaitState state)
+            {
+                semaphore.PrepareAsyncWait(ThreadPoolThreadTimeoutMs, s_WorkLoopSemaphoreSuccess, s_WorkLoopSemaphoreTimedOut, state);
+                // thread should still be kept alive
+                Debug.Assert(state.KeepaliveToken.Valid);
+            }
+
+            private static void WorkLoopSemaphoreSuccess(LowLevelLifoAsyncWaitSemaphore semaphore, object? stateObject)
+            {
+                SemaphoreWaitState state = (SemaphoreWaitState)stateObject!;
+                WorkerDoWork(state.ThreadPoolInstance, ref state.SpinWait);
+                // Go around the loop one more time, keeping existing mutated state
+                WaitForWorkLoop(semaphore, state);
+            }
+
+            private static void WorkLoopSemaphoreTimedOut(LowLevelLifoAsyncWaitSemaphore semaphore, object? stateObject)
+            {
+                SemaphoreWaitState state = (SemaphoreWaitState)stateObject!;
+                if (ShouldExitWorker(state.ThreadPoolInstance, state.ThreadAdjustmentLock)) {
+                    // we're done, kill the thread.
+
+                    // we're wrapped in an emscripten eventloop handler which will consult the
+                    // keepalive count, destroy the thread and run the TLS dtor which will
+                    // unregister the thread from Mono
+                    state.KeepaliveToken.Pop();
+                    return;
+                } else {
+                    // more work showed up while we were shutting down, go around one more time
+                    state.ResetIteration();
+                    WaitForWorkLoop(semaphore, state);
+                }
+            }
+
+            private static void CreateWorkerThread()
+            {
+                // Thread pool threads must start in the default execution context without transferring the context, so
+                // using captureContext: false.
+                Thread workerThread = new Thread(s_workerThreadStart);
+                workerThread.IsThreadPoolThread = true;
+                workerThread.IsBackground = true;
+                // thread name will be set in thread proc
+
+                // This thread will return to the JS event loop - tell the runtime not to cleanup
+                // after the start function returns, if the Emscripten keepalive is non-zero.
+                WebWorkerEventLoop.StartExitable(workerThread, captureContext: false);
+            }
+        }
+    }
+}

src/mono/System.Private.CoreLib/src/System/Threading/Thread.Mono.cs

@@ -37,6 +37,7 @@ public partial class Thread
         private int interruption_requested;
         private IntPtr longlived;
         internal bool threadpool_thread;
+        internal bool external_eventloop; // browser-wasm: thread will return to the JS eventloop
         /* These are used from managed code */
         internal byte apartment_state;
         internal int managed_id;
@@ -352,5 +353,17 @@ private static void SpinWait_nop()
         private static extern void SetPriority(Thread thread, int priority);
 
         internal int GetSmallId() => small_id;
+
+        internal bool HasExternalEventLoop
+        {
+            get
+            {
+                return external_eventloop;
+            }
+            set
+            {
+                external_eventloop = value;
+            }
+        }
     }
 }

src/mono/System.Private.CoreLib/src/System/Threading/ThreadPool.Browser.Threads.Mono.cs

@@ -0,0 +1,13 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Threading
+{
+    public static partial class ThreadPool
+    {
+        // Indicates that the threadpool should yield the thread from the dispatch loop to the
+        // runtime periodically.  We use this to return back to the JS event loop so that the JS
+        // event queue can be drained
+        internal static bool YieldFromDispatchLoop => true;
+    }
+}

src/mono/System.Private.CoreLib/src/System/Threading/WebWorkerEventLoop.Browser.Threads.Mono.cs

@@ -67,10 +67,10 @@ internal static void StartExitable(Thread thread, bool captureContext)
         // not needed by PortableThreadPool.WorkerThread
         if (captureContext)
             throw new InvalidOperationException();
-        // hack: threadpool threads are exitable, and nothing else is.
-        // see create_thread() in mono/metadata/threads.c
+        // for now, threadpool threads are exitable, and nothing else is.
         if (!thread.IsThreadPoolThread)
             throw new InvalidOperationException();
+        thread.HasExternalEventLoop = true;
         thread.UnsafeStart();
     }
 

src/mono/mono/metadata/object-internals.h

@@ -616,6 +616,7 @@ struct _MonoInternalThread {
 	 * longer */
 	MonoLongLivedThreadData *longlived;
 	MonoBoolean threadpool_thread;
+	MonoBoolean external_eventloop;
 	guint8	apartment_state;
 	gint32 managed_id;
 	guint32 small_id;

src/mono/mono/metadata/threads-types.h

@@ -78,6 +78,10 @@ typedef enum {
 	MONO_THREAD_CREATE_FLAGS_DEBUGGER		= 0x02,
 	MONO_THREAD_CREATE_FLAGS_FORCE_CREATE	= 0x04,
 	MONO_THREAD_CREATE_FLAGS_SMALL_STACK	= 0x08,
+	// "external eventloop" means the thread main function can return without killing the thread
+	// and the thread will continue to be attached to the runtime and may invoke embedding APIs
+	// and managed calls.  There is usually some platform-specific way to shut down the thread.
+	MONO_THREAD_CREATE_FLAGS_EXTERNAL_EVENTLOOP = 0x10,
 } MonoThreadCreateFlags;
 
 MONO_COMPONENT_API MonoInternalThread*