[wasm] Misc Debugger improvements (including tests) (#65752)

* [wasm][debugger] Fail test if an assertion is detected

* [wasm][debugger] Make DevToolsProxy's `pending_ops` thread-safe

Currently, `pending_ops` can get written by different threads at the
same time, and also read in parallel. This causes tests to randomly fail
with:

```
DevToolsProxy::Run: Exception System.ArgumentException: The tasks argument included a null value. (Parameter 'tasks')
    at System.Threading.Tasks.Task.WhenAny(Task[] tasks)
    at Microsoft.WebAssembly.Diagnostics.DevToolsProxy.Run(Uri browserUri, WebSocket ideSocket) in /_/src/mono/wasm/debugger/BrowserDebugProxy/DevToolsProxy.cs:line 269
```

Instead, we use `Channel<T>` to add the ops, and then read those in
the main loop, and add to the *local* `pending_ops` list.

* [wasm] Install chrome for debugger tests

- controlled by `$(InstallChromeForDebuggerTests)` which defaults to
`true` for non-CI docker containers
- Useful for using the correct chrome version when testing locally, or
on codespaces

- Also, add support for detecting, and defaulting to such an
installation

* [wasm][debugger] Disable tests failing with runtime assertions

`DebuggerTests.EvaluateOnCallFrameTests.EvaluateSimpleMethodCallsError`: #65744
`DebuggerTests.ArrayTests.InvalidArrayId`: #65742

* [wasm][debugger] Fix NRE with `"null"` condition for a breakpoint

`ConditionalBreakpoint` test fails.
The test with condition=`"null"` failed with a NRE, instead of correctly
handling it as a condition that returns null.

```
Unable evaluate conditional breakpoint: System.Exception: Internal Error: Unable to run (null ),
    error: Object reference not set to an instance of an object..
    ---> System.NullReferenceException: Object reference not set to an instance of an object.
            at Microsoft.WebAssembly.Diagnostics.EvaluateExpression.CompileAndRunTheExpression(String expression, MemberReferenceResolver resolver, CancellationToken token) in /workspaces/runtime/src/mono/wasm/debugger/BrowserDebugProxy/EvaluateExpression.cs:line 399
            --- End of inner exception stack trace ---
            at Microsoft.WebAssembly.Diagnostics.EvaluateExpression.CompileAndRunTheExpression(String expression, MemberReferenceResolver resolver, CancellationToken token) in /workspaces/runtime/src/mono/wasm/debugger/BrowserDebugProxy/EvaluateExpression.cs:line 407
            at Microsoft.WebAssembly.Diagnostics.MonoProxy.EvaluateCondition(SessionId sessionId, ExecutionContext context, Frame mono_frame, Breakpoint bp, CancellationToken token) in /workspaces/runtime/src/mono/wasm/debugger/BrowserDebugProxy/MonoProxy.cs:line 801 condition:null
```

* [wasm] Improve default message for ReturnAsErrorException,

.. since we seem to be not catching them as intended in many places.

* [wasm] Better log, and surface errors in evaluation conditional

.. breakpoints.
Catch the proper exception `ReturnAsErrorException`, so we can get the
actual error message. And log that as an error for the user too.

* [wasm][debugger] Allow setting proxy port for BrowserDebugHost

.. with `--proxy-port=<port>`.

Fixes #65209 .

* [wasm][debugger] Handle errors in getting value for locals

Essentially, catch, and skip.

* message cleanup

* remove trailing space

src/libraries/sendtohelix-wasm.targets

@@ -38,8 +38,8 @@
     <HelixPreCommand Include="export XHARNESS_DISABLE_COLORED_OUTPUT=true" />
     <HelixPreCommand Include="export XHARNESS_LOG_WITH_TIMESTAMPS=true" />
 
-    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="export PATH=$HELIX_CORRELATION_PAYLOAD/chromedriver_linux64:$PATH" />
-    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="export PATH=$HELIX_CORRELATION_PAYLOAD/chrome-linux:$PATH" />
+    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="export PATH=$HELIX_CORRELATION_PAYLOAD/$(ChromeDriverDirName):$PATH" />
+    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="export PATH=$HELIX_CORRELATION_PAYLOAD/$(ChromiumDirName):$PATH" />
 
     <!-- Fix symbolic links that are broken already on build machine and also in the correlation payload -->
     <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="export HELIX_NODEJS_VERSION=%24(ls $HELIX_CORRELATION_PAYLOAD/build/emsdk/node)" />
@@ -60,8 +60,8 @@
     <HelixPreCommand Include="set XHARNESS_DISABLE_COLORED_OUTPUT=true" />
     <HelixPreCommand Include="set XHARNESS_LOG_WITH_TIMESTAMPS=true" />
 
-    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\chromedriver_win32%3B%PATH%" />
-    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\chrome-win%3B%PATH%" />
+    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\$(ChromeDriverDirName)%3B%PATH%" />
+    <HelixPreCommand Condition="'$(NeedsToRunOnBrowser)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\$(ChromiumDirName)%3B%PATH%" />
 
     <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="for /f %%i in ('dir %HELIX_CORRELATION_PAYLOAD%\build\emsdk\node /b') do set HELIX_NODEJS_VERSION=%%i" />
     <HelixPreCommand Condition="'$(NeedsEMSDKNode)' == 'true'" Include="set PATH=%HELIX_CORRELATION_PAYLOAD%\build\emsdk\node\%HELIX_NODEJS_VERSION%\bin%3B%PATH%" />
@@ -96,6 +96,8 @@
     </When>
   </Choose>
 
+  <Import Project="$(RepoRoot)src\mono\wasm\BrowsersForTesting.props" />
+
   <Target Name="PrepareForBuildHelixWorkItems_Wasm">
     <PropertyGroup>
       <WasmBuildTargetsDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono', 'wasm', 'build'))</WasmBuildTargetsDir>
@@ -106,26 +108,6 @@
       <WorkItemPrefix Condition="'$(WorkItemPrefix)' == '' and '$(Scenario)' != ''">$(Scenario)-</WorkItemPrefix>
     </PropertyGroup>
 
-    <!-- Version number to revision number mapping from http://omahaproxy.appspot.com/ and find the closest one in
-         https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
-         and https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Win_x64/
-
-         Eg. latest stable version is 96.0.4664.45 with revision 929512.
-         but the closest one available in the snapshosts is 929513.
-         Please make sure to check both platforms as sometime
-         the same snapshot might not be available in one of them.
-    -->
-    <PropertyGroup Condition="'$(BrowserHost)' != 'windows'">
-      <ChromiumRevision>929513</ChromiumRevision>
-      <ChromiumUrl>https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/$(ChromiumRevision)/chrome-linux.zip</ChromiumUrl>
-      <ChromeDriverUrl>https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/$(ChromiumRevision)/chromedriver_linux64.zip</ChromeDriverUrl>
-    </PropertyGroup>
-    <PropertyGroup Condition="'$(BrowserHost)' == 'windows'">
-      <ChromiumRevision>929513</ChromiumRevision>
-      <ChromiumUrl>https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/$(ChromiumRevision)/chrome-win.zip</ChromiumUrl>
-      <ChromeDriverUrl>https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/$(ChromiumRevision)/chromedriver_win32.zip</ChromeDriverUrl>
-    </PropertyGroup>
-
     <ItemGroup Condition="'$(NeedsToRunOnBrowser)' == 'true'">
       <HelixCorrelationPayload Include="chromium" Uri="$(ChromiumUrl)" />
       <HelixCorrelationPayload Include="chromedriver" Uri="$(ChromeDriverUrl)" />

src/mono/wasm/BrowsersForTesting.props

@@ -0,0 +1,28 @@
+<Project>
+  <!-- Version number to revision number mapping from http://omahaproxy.appspot.com/ and find the closest one in
+    https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
+    and https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Win_x64/
+
+    Eg. latest stable version is 96.0.4664.45 with revision 929512.
+    but the closest one available in the snapshosts is 929513.
+    Please make sure to check both platforms as sometime
+    the same snapshot might not be available in one of them.
+  -->
+  <PropertyGroup Condition="'$(BrowserHost)' != 'windows'">
+    <ChromiumRevision>929513</ChromiumRevision>
+    <ChromiumUrl>https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/$(ChromiumRevision)/chrome-linux.zip</ChromiumUrl>
+    <ChromeDriverUrl>https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/$(ChromiumRevision)/chromedriver_linux64.zip</ChromeDriverUrl>
+    <ChromiumDirName>chrome-linux</ChromiumDirName>
+    <ChromeDriverDirName>chromedriver_linux64</ChromeDriverDirName>
+    <ChromiumBinaryName>chrome</ChromiumBinaryName>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(BrowserHost)' == 'windows'">
+    <ChromiumRevision>929513</ChromiumRevision>
+    <ChromiumUrl>https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/$(ChromiumRevision)/chrome-win.zip</ChromiumUrl>
+    <ChromeDriverUrl>https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/$(ChromiumRevision)/chromedriver_win32.zip</ChromeDriverUrl>
+    <ChromiumDirName>chrome-win</ChromiumDirName>
+    <ChromeDriverDirName>chromedriver_win32</ChromeDriverDirName>
+    <ChromiumBinaryName>chrome.exe</ChromiumBinaryName>
+  </PropertyGroup>
+</Project>

src/mono/wasm/README.md

@@ -143,6 +143,8 @@ To run a test with `FooBar` in the name:
 
 Additional arguments for `dotnet test` can be passed via `MSBUILD_ARGS` or `TEST_ARGS`. For example `MSBUILD_ARGS="/p:WasmDebugLevel=5"`. Though only one of `TEST_ARGS`, or `TEST_FILTER` can be used at a time.
 
+- Chrome can be installed for testing by setting `InstallChromeForDebuggerTests=true` when building the tests.
+
 ## Run samples
 
 The samples in `src/mono/sample/wasm` can be build and run like this:

src/mono/wasm/debugger/BrowserDebugProxy/DevToolsProxy.cs

@@ -8,6 +8,7 @@
 using System.Net.WebSockets;
 using System.Text;
 using System.Threading;
+using System.Threading.Channels;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
@@ -24,7 +25,8 @@ internal class DevToolsProxy
         private ClientWebSocket browser;
         private WebSocket ide;
         private int next_cmd_id;
-        private List<Task> pending_ops = new List<Task>();
+        private readonly ChannelWriter<Task> _channelWriter;
+        private readonly ChannelReader<Task> _channelReader;
         private List<DevToolsQueue> queues = new List<DevToolsQueue>();
 
         protected readonly ILogger logger;
@@ -33,6 +35,10 @@ public DevToolsProxy(ILoggerFactory loggerFactory, string loggerId)
         {
             string loggerSuffix = string.IsNullOrEmpty(loggerId) ? string.Empty : $"-{loggerId}";
             logger = loggerFactory.CreateLogger($"{nameof(DevToolsProxy)}{loggerSuffix}");
+
+            var channel = Channel.CreateUnbounded<Task>(new UnboundedChannelOptions { SingleReader = true });
+            _channelWriter = channel.Writer;
+            _channelReader = channel.Reader;
         }
 
         protected virtual Task<bool> AcceptEvent(SessionId sessionId, string method, JObject args, CancellationToken token)
@@ -94,7 +100,7 @@ private DevToolsQueue GetQueueForTask(Task task)
             return queues.FirstOrDefault(q => q.CurrentSend == task);
         }
 
-        private void Send(WebSocket to, JObject o, CancellationToken token)
+        private async Task Send(WebSocket to, JObject o, CancellationToken token)
         {
             string sender = browser == to ? "Send-browser" : "Send-ide";
 
@@ -106,7 +112,7 @@ private void Send(WebSocket to, JObject o, CancellationToken token)
 
             Task task = queue.Send(bytes, token);
             if (task != null)
-                pending_ops.Add(task);
+                await _channelWriter.WriteAsync(task, token);
         }
 
         private async Task OnEvent(SessionId sessionId, string method, JObject args, CancellationToken token)
@@ -116,7 +122,7 @@ private async Task OnEvent(SessionId sessionId, string method, JObject args, Can
                 if (!await AcceptEvent(sessionId, method, args, token))
                 {
                     //logger.LogDebug ("proxy browser: {0}::{1}",method, args);
-                    SendEventInternal(sessionId, method, args, token);
+                    await SendEventInternal(sessionId, method, args, token);
                 }
             }
             catch (Exception e)
@@ -132,7 +138,7 @@ private async Task OnCommand(MessageId id, string method, JObject args, Cancella
                 if (!await AcceptCommand(id, method, args, token))
                 {
                     Result res = await SendCommandInternal(id, method, args, token);
-                    SendResponseInternal(id, res, token);
+                    await SendResponseInternal(id, res, token);
                 }
             }
             catch (Exception e)
@@ -153,31 +159,38 @@ private void OnResponse(MessageId id, Result result)
             logger.LogError("Cannot respond to command: {id} with result: {result} - command is not pending", id, result);
         }
 
-        private void ProcessBrowserMessage(string msg, CancellationToken token)
+        private Task ProcessBrowserMessage(string msg, CancellationToken token)
         {
             var res = JObject.Parse(msg);
 
             //if (method != "Debugger.scriptParsed" && method != "Runtime.consoleAPICalled")
             Log("protocol", $"browser: {msg}");
 
             if (res["id"] == null)
-                pending_ops.Add(OnEvent(res.ToObject<SessionId>(), res["method"].Value<string>(), res["params"] as JObject, token));
+            {
+                return OnEvent(res.ToObject<SessionId>(), res["method"].Value<string>(), res["params"] as JObject, token);
+            }
             else
+            {
                 OnResponse(res.ToObject<MessageId>(), Result.FromJson(res));
+                return null;
+            }
         }
 
-        private void ProcessIdeMessage(string msg, CancellationToken token)
+        private Task ProcessIdeMessage(string msg, CancellationToken token)
         {
             Log("protocol", $"ide: {msg}");
             if (!string.IsNullOrEmpty(msg))
             {
                 var res = JObject.Parse(msg);
                 var id = res.ToObject<MessageId>();
-                pending_ops.Add(OnCommand(
+                return OnCommand(
                     id,
                     res["method"].Value<string>(),
-                    res["params"] as JObject, token));
+                    res["params"] as JObject, token);
             }
+
+            return null;
         }
 
         internal async Task<Result> SendCommand(SessionId id, string method, JObject args, CancellationToken token)
@@ -186,7 +199,7 @@ internal async Task<Result> SendCommand(SessionId id, string method, JObject arg
             return await SendCommandInternal(id, method, args, token);
         }
 
-        private Task<Result> SendCommandInternal(SessionId sessionId, string method, JObject args, CancellationToken token)
+        private async Task<Result> SendCommandInternal(SessionId sessionId, string method, JObject args, CancellationToken token)
         {
             int id = Interlocked.Increment(ref next_cmd_id);
 
@@ -204,17 +217,17 @@ private Task<Result> SendCommandInternal(SessionId sessionId, string method, JOb
             //Log ("verbose", $"add cmd id {sessionId}-{id}");
             pending_cmds[msgId] = tcs;
 
-            Send(this.browser, o, token);
-            return tcs.Task;
+            await Send(browser, o, token);
+            return await tcs.Task;
         }
 
-        public void SendEvent(SessionId sessionId, string method, JObject args, CancellationToken token)
+        public Task SendEvent(SessionId sessionId, string method, JObject args, CancellationToken token)
         {
             //Log ("verbose", $"sending event {method}: {args}");
-            SendEventInternal(sessionId, method, args, token);
+            return SendEventInternal(sessionId, method, args, token);
         }
 
-        private void SendEventInternal(SessionId sessionId, string method, JObject args, CancellationToken token)
+        private Task SendEventInternal(SessionId sessionId, string method, JObject args, CancellationToken token)
         {
             var o = JObject.FromObject(new
             {
@@ -224,21 +237,21 @@ private void SendEventInternal(SessionId sessionId, string method, JObject args,
             if (sessionId.sessionId != null)
                 o["sessionId"] = sessionId.sessionId;
 
-            Send(this.ide, o, token);
+            return Send(ide, o, token);
         }
 
         internal void SendResponse(MessageId id, Result result, CancellationToken token)
         {
             SendResponseInternal(id, result, token);
         }
 
-        private void SendResponseInternal(MessageId id, Result result, CancellationToken token)
+        private Task SendResponseInternal(MessageId id, Result result, CancellationToken token)
         {
             JObject o = result.ToJObject(id);
             if (!result.IsOk)
                 logger.LogError($"sending error response for id: {id} -> {result}");
 
-            Send(this.ide, o, token);
+            return Send(this.ide, o, token);
         }
 
         // , HttpContext context)
@@ -258,10 +271,14 @@ public async Task Run(Uri browserUri, WebSocket ideSocket)
                     Log("verbose", $"DevToolsProxy: Client connected on {browserUri}");
                     var x = new CancellationTokenSource();
 
+                    List<Task> pending_ops = new();
+
                     pending_ops.Add(ReadOne(browser, x.Token));
                     pending_ops.Add(ReadOne(ide, x.Token));
                     pending_ops.Add(side_exception.Task);
                     pending_ops.Add(client_initiated_close.Task);
+                    Task<bool> readerTask = _channelReader.WaitToReadAsync(x.Token).AsTask();
+                    pending_ops.Add(readerTask);
 
                     try
                     {
@@ -278,14 +295,26 @@ public async Task Run(Uri browserUri, WebSocket ideSocket)
                                 break;
                             }
 
+                            if (readerTask.IsCompleted)
+                            {
+                                while (_channelReader.TryRead(out Task newTask))
+                                {
+                                    pending_ops.Add(newTask);
+                                }
+
+                                pending_ops[4] = _channelReader.WaitToReadAsync(x.Token).AsTask();
+                            }
+
                             //logger.LogTrace ("pump {0} {1}", task, pending_ops.IndexOf (task));
                             if (completedTask == pending_ops[0])
                             {
                                 string msg = ((Task<string>)completedTask).Result;
                                 if (msg != null)
                                 {
                                     pending_ops[0] = ReadOne(browser, x.Token); //queue next read
-                                    ProcessBrowserMessage(msg, x.Token);
+                                    Task newTask = ProcessBrowserMessage(msg, x.Token);
+                                    if (newTask != null)
+                                        pending_ops.Add(newTask);
                                 }
                             }
                             else if (completedTask == pending_ops[1])
@@ -294,7 +323,9 @@ public async Task Run(Uri browserUri, WebSocket ideSocket)
                                 if (msg != null)
                                 {
                                     pending_ops[1] = ReadOne(ide, x.Token); //queue next read
-                                    ProcessIdeMessage(msg, x.Token);
+                                    Task newTask = ProcessIdeMessage(msg, x.Token);
+                                    if (newTask != null)
+                                        pending_ops.Add(newTask);
                                 }
                             }
                             else if (completedTask == pending_ops[2])
@@ -314,10 +345,13 @@ public async Task Run(Uri browserUri, WebSocket ideSocket)
                                 }
                             }
                         }
+
+                        _channelWriter.Complete();
                     }
                     catch (Exception e)
                     {
                         Log("error", $"DevToolsProxy::Run: Exception {e}");
+                        _channelWriter.Complete(e);
                         //throw;
                     }
                     finally

src/mono/wasm/debugger/BrowserDebugProxy/EvaluateExpression.cs

@@ -396,7 +396,7 @@ internal static async Task<JObject> CompileAndRunTheExpression(string expression
                     string.Join("\n", findVarNMethodCall.variableDefinitions) + "\nreturn " + syntaxTree.ToString());
 
                 var state = await newScript.RunAsync(cancellationToken: token);
-                return JObject.FromObject(ConvertCSharpToJSType(state.ReturnValue, state.ReturnValue.GetType()));
+                return JObject.FromObject(ConvertCSharpToJSType(state.ReturnValue, state.ReturnValue?.GetType()));
             }
             catch (CompilationErrorException cee)
             {
@@ -419,7 +419,7 @@ internal static async Task<JObject> CompileAndRunTheExpression(string expression
         private static object ConvertCSharpToJSType(object v, Type type)
         {
             if (v == null)
-                return new { type = "object", subtype = "null", className = type.ToString(), description = type.ToString() };
+                return new { type = "object", subtype = "null", className = type?.ToString(), description = type?.ToString() };
             if (v is string s)
                 return new { type = "string", value = s, description = s };
             if (NumericTypes.Contains(v.GetType()))
@@ -443,10 +443,11 @@ public Result Error
             }
             set { }
         }
-        public ReturnAsErrorException(JObject error)
+        public ReturnAsErrorException(JObject error) : base(error.ToString())
             => Error = Result.Err(error);
 
         public ReturnAsErrorException(string message, string className)
+            : base($"[{className}] {message}")
         {
             var result = new
             {
@@ -466,5 +467,7 @@ public ReturnAsErrorException(string message, string className)
                     }
                 }));
         }
+
+        public override string ToString() => $"Error object: {Error}. {base.ToString()}";
     }
 }