[wasm] Download Symbols from microsoft symbol server (#40690)

* Creating a draft to download symbols from microsoft symbol server when there is an exception and exceptions are turned on on debugger.
This is a workaround while VS doesn't work on it, which should be the final solution.

* Fix what lewing suggested.

* Changing what @radical suggested.

* Changed what @radical suggested.

* Update src/mono/wasm/debugger/BrowserDebugProxy/MonoProxy.cs

Co-authored-by: Ankit Jain <radical@gmail.com>

* Update src/mono/wasm/debugger/BrowserDebugProxy/MonoProxy.cs

Co-authored-by: Ankit Jain <radical@gmail.com>

* If it's not available on a URL try in the next one in the list.

* Logging error and adding comment about SDSR

* Returning if we find the method even if we have an exception sending files to browser.

* Logging when we don't find the pdb.

* Changing what @radical suggested.

* Avoiding that we try to load symbols from the same assembly more than once.

* Simplifying and adding more log.

* Update src/mono/wasm/debugger/BrowserDebugProxy/MonoProxy.cs

Co-authored-by: Larry Ewing <lewing@microsoft.com>

* Adding support for receive the urlSymbolServer as a parameter in the request as discussed with Diego:

Diego: The response from vs code is that this shouldn't live in js-debug, so I think that passing it around in inspectUri would be the best approach for us

* Fix @radical suggestion.

* Removing default symbol server URL, like this we can land this on rc 5.0 without any risk of side effect.

* Fix compilation.

* Update src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessStartup.cs

Co-authored-by: Ankit Jain <radical@gmail.com>

* Update src/mono/wasm/debugger/BrowserDebugHost/Startup.cs

Co-authored-by: Ankit Jain <radical@gmail.com>

* Accepting @radical suggestion.

* Adding workaround on dotnet/runtime while PR 686 is not merged on Cecil. When it's merged we will not get any side effect with this workaroud.

Co-authored-by: Ankit Jain <radical@gmail.com>
Co-authored-by: Larry Ewing <lewing@microsoft.com>

src/mono/wasm/debugger/BrowserDebugHost/Startup.cs

@@ -15,6 +15,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Microsoft.Extensions.Primitives;
 
 namespace Microsoft.WebAssembly.Diagnostics
 {
@@ -138,7 +139,10 @@ async Task ConnectProxy(HttpContext context)
                     {
                         using var loggerFactory = LoggerFactory.Create(
                             builder => builder.AddConsole().AddFilter(null, LogLevel.Information));
-                        var proxy = new DebuggerProxy(loggerFactory);
+
+                        context.Request.Query.TryGetValue("urlSymbolServer", out StringValues urlSymbolServerList);
+                        var proxy = new DebuggerProxy(loggerFactory, urlSymbolServerList.ToList());
+                        
                         var ideSocket = await context.WebSockets.AcceptWebSocketAsync();
 
                         await proxy.Run(endpoint, ideSocket);

src/mono/wasm/debugger/BrowserDebugProxy/DebugStore.cs

@@ -382,6 +382,7 @@ class AssemblyInfo
         Dictionary<string, TypeInfo> typesByName = new Dictionary<string, TypeInfo>();
         readonly List<SourceFile> sources = new List<SourceFile>();
         internal string Url { get; }
+        public bool TriedToLoadSymbolsOnDemand { get; set; }
 
         public AssemblyInfo(IAssemblyResolver resolver, string url, byte[] assembly, byte[] pdb)
         {
@@ -438,8 +439,24 @@ public AssemblyInfo(ILogger logger)
         {
             this.logger = logger;
         }
+
+        public ModuleDefinition Image => image;
 
-        void Populate()
+        public void ClearDebugInfo()
+        {
+            foreach (var type in image.GetTypes())
+            {
+                var typeInfo = new TypeInfo(this, type);
+                typesByName[type.FullName] = typeInfo;
+
+                foreach (var method in type.Methods)
+                {
+                    method.DebugInformation = null;
+                }
+            }
+        }
+
+        public void Populate()
         {
             ProcessSourceLink();
 

src/mono/wasm/debugger/BrowserDebugProxy/DebuggerProxy.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
+using System.Collections.Generic;
 using System.Net.WebSockets;
 using System.Threading.Tasks;
 using Microsoft.Extensions.Logging;
@@ -16,9 +17,9 @@ public class DebuggerProxy
     {
         private readonly MonoProxy proxy;
 
-        public DebuggerProxy(ILoggerFactory loggerFactory)
+        public DebuggerProxy(ILoggerFactory loggerFactory, IList<string> urlSymbolServerList)
         {
-            proxy = new MonoProxy(loggerFactory);
+            proxy = new MonoProxy(loggerFactory, urlSymbolServerList);
         }
 
         public Task Run(Uri browserUri, WebSocket ideSocket)

src/mono/wasm/debugger/BrowserDebugProxy/MonoProxy.cs

@@ -11,16 +11,26 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json.Linq;
+using Mono.Cecil.Cil;
+using Mono.Cecil.Pdb;
+using Mono.Cecil;
+using System.Net.Http;
 
 namespace Microsoft.WebAssembly.Diagnostics
 {
 
     internal class MonoProxy : DevToolsProxy
     {
+        IList<string> urlSymbolServerList;
+        static HttpClient client = new HttpClient();
         HashSet<SessionId> sessions = new HashSet<SessionId>();
         Dictionary<SessionId, ExecutionContext> contexts = new Dictionary<SessionId, ExecutionContext>();
 
-        public MonoProxy(ILoggerFactory loggerFactory, bool hideWebDriver = true) : base(loggerFactory) { this.hideWebDriver = hideWebDriver; }
+        public MonoProxy(ILoggerFactory loggerFactory, IList<string> urlSymbolServerList, bool hideWebDriver = true) : base(loggerFactory) 
+        { 
+            this.hideWebDriver = hideWebDriver;
+            this.urlSymbolServerList = urlSymbolServerList ?? new List<string>();
+        }
 
         readonly bool hideWebDriver;
 
@@ -337,6 +347,13 @@ protected override async Task<bool> AcceptCommand(MessageId id, string method, J
                     }
 
                 // Protocol extensions
+                case "DotnetDebugger.addSymbolServerUrl":
+                    {
+                        string url = args["url"]?.Value<string>();
+                        if (!String.IsNullOrEmpty(url) && !urlSymbolServerList.Contains(url))
+                            urlSymbolServerList.Add(url);
+                        return true;
+                    }
                 case "DotnetDebugger.getMethodLocation":
                     {
                         Console.WriteLine("set-breakpoint-by-method: " + id + " " + args);
@@ -539,6 +556,19 @@ async Task<bool> OnPause(SessionId sessionId, JObject args, CancellationToken to
 
                         var method = asm.GetMethodByToken(method_token);
 
+                        if (method == null && !asm.Image.HasSymbols)
+                        {
+                            try
+                            {
+                                method = await LoadSymbolsOnDemand(asm, method_token, sessionId, token);
+                            }
+                            catch (Exception e)
+                            {
+                                Log("info", $"Unable to find il offset: {il_pos} in method token: {method_token} assembly name: {assembly_name} exception: {e.ToString()}");
+                                continue;
+                            }
+                        }
+
                         if (method == null)
                         {
                             Log("info", $"Unable to find il offset: {il_pos} in method token: {method_token} assembly name: {assembly_name}");
@@ -616,6 +646,81 @@ async Task<bool> OnPause(SessionId sessionId, JObject args, CancellationToken to
             return true;
         }
 
+        async Task<MethodInfo> LoadSymbolsOnDemand(AssemblyInfo asm, uint method_token, SessionId sessionId, CancellationToken token)
+        {
+            var context = GetContext(sessionId);
+            if (asm.TriedToLoadSymbolsOnDemand)
+                return null;
+            asm.TriedToLoadSymbolsOnDemand = true;
+            ImageDebugHeader header = asm.Image.GetDebugHeader();
+
+            for (var i = 0; i < header.Entries.Length; i++)
+            {
+                var entry = header.Entries[i];
+                if (entry.Directory.Type != ImageDebugType.CodeView)
+                {
+                    continue;
+                }
+
+                var data = entry.Data;
+
+                if (data.Length < 24)
+                    return null;
+
+                var pdbSignature = (data[0]
+                    | (data[1] << 8)
+                    | (data[2] << 16)
+                    | (data[3] << 24));
+
+                if (pdbSignature != 0x53445352) // "SDSR" mono/metadata/debug-mono-ppdb.c#L101
+                    return null;
+
+                var buffer = new byte[16];
+                Buffer.BlockCopy(data, 4, buffer, 0, 16);
+
+                var pdbAge = (data[20]
+                    | (data[21] << 8)
+                    | (data[22] << 16)
+                    | (data[23] << 24));
+
+                var pdbGuid = new Guid(buffer);
+                var buffer2 = new byte[(data.Length - 24) - 1];
+                Buffer.BlockCopy(data, 24, buffer2, 0, (data.Length - 24) - 1);
+                var pdbName = System.Text.Encoding.UTF8.GetString(buffer2, 0, buffer2.Length);
+                pdbName = Path.GetFileName(pdbName);
+
+                foreach (var urlSymbolServer in urlSymbolServerList) 
+                {
+                    var downloadURL = $"{urlSymbolServer}/{pdbName}/{pdbGuid.ToString("N").ToUpper() + pdbAge}/{pdbName}";
+
+                    try
+                    {
+                        using HttpResponseMessage response = await client.GetAsync(downloadURL);
+                        using Stream streamToReadFrom = await response.Content.ReadAsStreamAsync();
+                        var portablePdbReaderProvider = new PdbReaderProvider();
+                        var symbolReader = portablePdbReaderProvider.GetSymbolReader(asm.Image, streamToReadFrom);
+                        asm.ClearDebugInfo(); //workaround while cecil PR #686 is not merged
+                        asm.Image.ReadSymbols(symbolReader);
+                        asm.Populate();
+                        foreach (var source in asm.Sources)
+                        {
+                            var scriptSource = JObject.FromObject(source.ToScriptSource(context.Id, context.AuxData));
+                            SendEvent(sessionId, "Debugger.scriptParsed", scriptSource, token);
+                        }
+                        return asm.GetMethodByToken(method_token);
+                    }
+                    catch (Exception e)
+                    {
+                        Log("info", $"Unable to load symbols on demand exception: {e.ToString()} url:{downloadURL} assembly: {asm.Name}");
+                    }
+                }
+                break;
+            }
+
+            Log("info", "Unable to load symbols on demand assembly: {asm.Name}");
+            return null;
+        }
+
         async Task OnDefaultContext(SessionId sessionId, ExecutionContext context, CancellationToken token)
         {
             Log("verbose", "Default context created, clearing state and sending events");

src/mono/wasm/debugger/DebuggerTestSuite/TestHarnessStartup.cs

@@ -19,6 +19,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Newtonsoft.Json.Linq;
+using System.Collections.Generic;
 
 namespace Microsoft.WebAssembly.Diagnostics
 {
@@ -122,7 +123,8 @@ public async Task LaunchAndServe(ProcessStartInfo psi, HttpContext context, Func
 
                 using var loggerFactory = LoggerFactory.Create(
                     builder => builder.AddConsole().AddFilter(null, LogLevel.Information));
-                var proxy = new DebuggerProxy(loggerFactory);
+
+                var proxy = new DebuggerProxy(loggerFactory, null);
                 var browserUri = new Uri(con_str);
                 var ideSocket = await context.WebSockets.AcceptWebSocketAsync();