[wasm] event pipe tweaks

src/mono/sample/wasm/Directory.Build.targets

@@ -7,7 +7,18 @@
     <RunScriptInputName Condition="'$(TargetOS)' == 'Browser' and '$(OS)' == 'Windows_NT'">WasmRunnerTemplate.cmd</RunScriptInputName>
 
     <RunScriptOutputPath>$([MSBuild]::NormalizePath('$(WasmAppDir)', '$(RunScriptOutputName)'))</RunScriptOutputPath>
-    <WasmXHarnessArgs Condition="'$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true' or '$(MonoWasmBuildVariant)' == 'perftrace' or '$(MonoWasmBuildVariant)' == 'multithread'">$(WasmXHarnessArgs) --web-server-use-cop</WasmXHarnessArgs>
+
+    <!-- so that SharedArrayBuffer is enabled -->
+    <_MonoWasmThreads Condition="'$(WasmEnableThreads)' == 'true' or '$(WasmEnablePerfTracing)' == 'true' or '$(MonoWasmBuildVariant)' == 'multithread' or '$(MonoWasmBuildVariant)' == 'perftrace'">true</_MonoWasmThreads>
+    <WasmXHarnessArgs Condition="'$(_MonoWasmThreads)' == 'true'">$(WasmXHarnessArgs) --web-server-use-cop</WasmXHarnessArgs>
+    <_ServeHeaders    Condition="'$(_MonoWasmThreads)' == 'true'">$(_ServeHeaders) -h Cross-Origin-Embedder-Policy:require-corp -h Cross-Origin-Opener-Policy:same-origin</_ServeHeaders>
+
+    <!-- For streaming instantiation of WASM module and browser cache -->
+    <_ServeMimeTypes>$(_ServeMimeTypes) --mime .wasm=application/wasm</_ServeMimeTypes>
+    <_ServeMimeTypes>$(_ServeMimeTypes) --mime .json=application/json</_ServeMimeTypes>
+    <_ServeMimeTypes>$(_ServeMimeTypes) --mime .mjs=text/javascript</_ServeMimeTypes>
+    <_ServeMimeTypes>$(_ServeMimeTypes) --mime .cjs=text/javascript</_ServeMimeTypes>
+    <_ServeMimeTypes>$(_ServeMimeTypes) --mime .js=text/javascript</_ServeMimeTypes>
   </PropertyGroup>
 
   <Target Name="BuildSampleInTree"
@@ -26,6 +37,7 @@
       <_Dotnet>$(RepoRoot)dotnet$(_ScriptExt)</_Dotnet>
       <_AOTFlag Condition="'$(RunAOTCompilation)' != ''">/p:RunAOTCompilation=$(RunAOTCompilation)</_AOTFlag>
       <_WasmMainJSFileName>$([System.IO.Path]::GetFileName('$(WasmMainJSPath)'))</_WasmMainJSFileName>
+      <BuildAdditionalArgs Condition="'$(MonoDiagnosticsMock)' != ''">$(BuildAdditionalArgs) /p:MonoDiagnosticsMock=$(MonoDiagnosticsMock) </BuildAdditionalArgs>
     </PropertyGroup>
     <Exec Command="$(_Dotnet) publish -bl /p:Configuration=$(Configuration) /p:TargetArchitecture=wasm /p:TargetOS=Browser $(_AOTFlag) $(_SampleProject) $(BuildAdditionalArgs)" />
   </Target>
@@ -42,12 +54,7 @@
     <Exec Command="dotnet tool install -g dotnet-serve" IgnoreExitCode="true" />
   </Target>
   <Target Name="RunSampleWithBrowser" DependsOnTargets="BuildSampleInTree;CheckServe">
-  <!-- 
-    - we add MIME type for .wasm .mjs .js .cjs .json. Browsers require it for proper and fast execution. For example streaming instantiation of WASM module.
-    - we set `Cross-Origin-Opener-Policy` headers so that SharedArrayBuffer is enabled
-    - we set `Content-Security-Policy` headers to test that the app is able to run in environments with such restrictions.
-  -->
-    <Exec Command="$(_Dotnet) serve -o -d:bin/$(Configuration)/AppBundle -p:8000 --mime .wasm=application/wasm --mime .mjs=text/javascript --mime .js=text/javascript --mime .cjs=text/javascript --mime .json=application/json -h Cross-Origin-Opener-Policy:same-origin -h &quot;Cross-Origin-Embedder-Policy:require-corp&quot; -h &quot;Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval'&quot; " IgnoreExitCode="true" YieldDuringToolExecution="true" />
+    <Exec Command="$(_Dotnet) serve -o -d:bin/$(Configuration)/AppBundle -p:8000 $(_ServeMimeTypes) $(_ServeHeaders)" IgnoreExitCode="true" YieldDuringToolExecution="true" />
   </Target>
   <Target Name="RunSampleWithBrowserAndSimpleServer" DependsOnTargets="BuildSampleInTree">
     <Exec Command="$(_Dotnet) build -c $(Configuration) ..\simple-server\HttpServer.csproj" />

src/mono/sample/wasm/browser-advanced/Wasm.Advanced.Sample.csproj

@@ -6,6 +6,8 @@
     <PublishTrimmed>true</PublishTrimmed>
     <!-- add OpenGL emulation -->
     <EmccExtraLDFlags> -s USE_CLOSURE_COMPILER=1 -s LEGACY_GL_EMULATION=1 -lGL -lSDL</EmccExtraLDFlags>
+    <!-- just to prove we don't do JS eval() -->
+    <_ServeHeaders>$(_ServeHeaders) -h &quot;Content-Security-Policy: default-src 'self' 'wasm-unsafe-eval'&quot;</_ServeHeaders>
   </PropertyGroup>
   <ItemGroup>
     <!-- add export GL object from Module -->

src/mono/sample/wasm/browser-eventpipe/README.md

@@ -0,0 +1,43 @@
+[see also](../../../wasm/runtime/diagnostics/README.md)
+
+To be able to run this sample you need to build the runtime with `/p:WasmEnablePerfTracing=true` and use Chrome browser
+
+# Testing with mock
+
+Build the runtime with `/p:WasmEnablePerfTracing=true /p:MonoDiagnosticsMock=true`
+Run this test with `/p:MonoDiagnosticsMock=true`
+
+It will inject file [mock.js](./mock.js) into the worker thread, which is mocking the `dotnet trace` tool.
+
+The sample will communicate with the mock and start the Fibonacci experiment and start and stop the trace recording automatically.
+It will also download the .nettrace from the browser. You can covert it to other formats, for example
+```
+dotnet trace convert --format Speedscope c:\Downloads\trace.1665653486202.nettrace -o c:\Downloads\trace.1665653486202.speedscope
+```
+
+# Testing with dotnet trace tool
+
+Build the runtime with `/p:WasmEnablePerfTracing=true`
+Build version of dsrouter with WebSockets https://github.com/lambdageek/diagnostics/tree/wasm-server
+
+In console #1 start dsrouter
+```
+c:\Dev\diagnostics\artifacts\bin\dotnet-dsrouter\Debug\net6.0\dotnet-dsrouter.exe server-websocket -ws http://127.0.0.1:8088/diagnostics -ipcs C:\Dev\diagnostics\socket
+```
+
+In console #2 start the sample
+```
+dotnet build /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:Configuration=Debug /t:RunSample src/mono/sample/wasm/browser-eventpipe /p:MonoDiagnosticsMock=false
+```
+
+In console #3 start the dotnet trace
+```
+dotnet trace collect --diagnostic-port C:\Dev\diagnostics\socket,connect --format Chromium
+```
+This will use `cpu-sampling`,
+
+In the browser click `Start Work` button and after it finished
+
+In the #3 console press `Enter`
+
+In the browser open dev tools and on performance tab import file `C:\Dev\socket_XXXXXXXX_YYYYYY.chromium.json` which dotnet trace produced

src/mono/sample/wasm/browser-eventpipe/Wasm.Browser.EventPipe.Sample.csproj

@@ -3,14 +3,11 @@
   <PropertyGroup>
     <WasmBuildNative>true</WasmBuildNative>
     <FeatureWasmPerfTracing>true</FeatureWasmPerfTracing>
+    <WasmEnablePerfTracing>true</WasmEnablePerfTracing>
     <FeatureWasmThreads Condition="false">true</FeatureWasmThreads>
     <NoWarn>$(NoWarn);CA2007</NoWarn> <!-- consider ConfigureAwait() -->
   </PropertyGroup>
 
-  <Target Name="CheckThreadsEnabled" BeforeTargets="Compile" >
-    <Warning Condition="'$(WasmEnableThreads)' != 'true' and '$(WasmEnablePerfTracing)' != 'true'" Text="This sample requires perftracing or threading" />
-  </Target>
-
   <PropertyGroup>
     <MonoDiagnosticsMock Condition="('$(MonoDiagnosticsMock)' == '') and ('$(Configuration)' == 'Debug' or '$(ArchiveTests)' == 'true')">true</MonoDiagnosticsMock>
   </PropertyGroup>

src/mono/sample/wasm/browser-eventpipe/main.js

@@ -26,6 +26,11 @@ async function doWork(startWork, stopWork, getIterationsDone) {
     const ret = await workPromise; // get the answer
     const iterations = getIterationsDone(); // get how many times the loop ran
 
+    INTERNAL.diagnosticServerThread.postMessageToWorker({
+        type: "diagnostic_server_mock",
+        cmd: "fibonacci-done"
+    });
+
     btn = document.getElementById("startWork");
     btn.disabled = false;
     btn.innerText = "Start Work";
@@ -37,12 +42,6 @@ async function doWork(startWork, stopWork, getIterationsDone) {
     return ret;
 }
 
-function getOnClickHandler(startWork, stopWork, getIterationsDone) {
-    return async function () {
-        await doWork(startWork, stopWork, getIterationsDone);
-    }
-}
-
 const isTest = (config) => config.environmentVariables["CI_TEST"] === "true";
 async function runTest({ StartAsyncWork, StopWork, GetIterationsDone }) {
     const result = await doWork(StartAsyncWork, StopWork, GetIterationsDone);
@@ -51,19 +50,47 @@ async function runTest({ StartAsyncWork, StopWork, GetIterationsDone }) {
 }
 
 async function main() {
-    const { MONO, Module, getAssemblyExports, getConfig } = await dotnet
+    const { INTERNAL, MONO, Module, getAssemblyExports, getConfig } = await dotnet
         .withElementOnExit()
         .withExitCodeLogging()
+        .withDiagnosticTracing(false)
         .create();
 
     globalThis.__Module = Module;
     globalThis.MONO = MONO;
+    globalThis.INTERNAL = INTERNAL;
 
     const exports = await getAssemblyExports("Wasm.Browser.EventPipe.Sample.dll");
 
     const btn = document.getElementById("startWork");
     btn.style.backgroundColor = "rgb(192,255,192)";
-    btn.onclick = getOnClickHandler(exports.Sample.Test.StartAsyncWork, exports.Sample.Test.StopWork, exports.Sample.Test.GetIterationsDone);
+    btn.onclick = () => doWork(exports.Sample.Test.StartAsyncWork, exports.Sample.Test.StopWork, exports.Sample.Test.GetIterationsDone);
+
+    INTERNAL.diagnosticServerThread.port.addEventListener("message", (event) => {
+        console.warn("diagnosticServerThread" + event.type)
+
+        if (event.data.cmd === "collecting") {
+            btn.onclick();
+        }
+
+        if (event.data.cmd === "collected") {
+            const buffer = event.data.buffer;
+            const length = event.data.length;
+            console.warn("Downloading trace " + length)
+            const view = new Uint8Array(buffer, 0, length)
+            const blobUrl = URL.createObjectURL(new Blob([view.slice()]));
+            const link = document.createElement("a");
+            link.href = blobUrl;
+            link.download = "trace." + (new Date()).valueOf() + ".nettrace";
+            document.body.appendChild(link);
+
+            link.dispatchEvent(new MouseEvent('click', {
+                bubbles: true, cancelable: true, view: window
+            }));
+
+            document.body.removeChild(link);
+        }
+    });
 
     const config = getConfig();
     if (isTest(config)) {

src/mono/sample/wasm/browser-eventpipe/mock.js

@@ -22,32 +22,79 @@ function script(env) {
     const runtimeResumed = env.createPromiseController();
     /** @type { PromiseAndControllerVoid } */
     const waitForever = env.createPromiseController();
+
+    /** @type { PromiseAndControllerVoid } */
+    const fibonacciDone = env.createPromiseController();
+    env.addEventListenerFromBrowser("fibonacci-done", (event) => {
+        fibonacciDone.promise_control.resolve();
+    });
+
     return [
         async (conn) => {
-            await conn.waitForSend(env.expectAdvertise);
-            conn.reply(env.command.makeEventPipeCollectTracing2({
-                circularBufferMB: 1,
-                format: 1,
-                requestRundown: true,
-                providers: [
-                    {
-                        keywords: [0, 0],
-                        logLevel: 5,
-                        provider_name: "WasmHello",
-                        filter_data: "EventCounterIntervalSec=1"
+            try {
+                await conn.waitForSend(env.expectAdvertise);
+                conn.reply(env.command.makeEventPipeCollectTracing2({
+                    circularBufferMB: 256,
+                    format: 1,
+                    requestRundown: true,
+                    providers: [
+                        {
+                            keywords: [0, 0],
+                            logLevel: 5,
+                            provider_name: "WasmHello",
+                            filter_data: "EventCounterIntervalSec=1"
+                        },
+                        {
+                            keywords: [0, 61440],
+                            logLevel: 4,
+                            provider_name: "Microsoft-DotNETCore-SampleProfiler",
+                            filter_data: null
+                        },
+                        {
+                            keywords: [
+                                -1051734851,
+                                20
+                            ],
+                            logLevel: 4,
+                            provider_name: "Microsoft-Windows-DotNETRuntime",
+                            filter_data: null
+                        }
+                    ]
+                }));
+                let sessionID = undefined;
+                const buffer = new SharedArrayBuffer(2_000_000);
+                const view = new Uint8Array(buffer);
+                let length = 0;
+                await conn.processSend((bytes) => {
+                    if (sessionID === undefined) {
+                        // first block is just a session handshake
+                        if (!env.reply.expectOk(4)) {
+                            throw new Error("bad data");
+                        }
+                        sessionID = env.reply.extractOkSessionID(bytes)
+                        sessionStarted.promise_control.resolve(sessionID);
+                        env.postMessageToBrowser({ cmd: "collecting" });
+                    }
+                    else {
+                        const bytesView = new Uint8Array(bytes)
+                        view.set(bytesView, length);
+                        length += bytesView.byteLength;
                     }
-                ]
-            }));
-            const sessionID = await conn.waitForSend(env.reply.expectOk(4), env.reply.extractOkSessionID);
-            sessionStarted.promise_control.resolve(sessionID);
+                });
+                console.warn("final totalBytesStreamed " + length);
+                env.postMessageToBrowser({ cmd: "collected", buffer, length });
+            }
+            catch (err) {
+                console.error(err)
+            }
         },
         async (conn) => {
             await Promise.all([conn.waitForSend(env.expectAdvertise), sessionStarted.promise]);
             conn.reply(env.command.makeProcessResumeRuntime());
             runtimeResumed.promise_control.resolve();
         },
         async (conn) => {
-            await Promise.all([conn.waitForSend(env.expectAdvertise), runtimeResumed.promise, sessionStarted.promise]);
+            await Promise.all([conn.waitForSend(env.expectAdvertise), runtimeResumed.promise, sessionStarted.promise, fibonacciDone.promise]);
             const sessionID = await sessionStarted.promise;
             await env.delay(5000);
             conn.reply(env.command.makeEventPipeStopTracing({ sessionID }));

src/mono/wasm/runtime/diagnostics-mock.d.ts

@@ -42,14 +42,15 @@ interface EventPipeCollectTracingCommandProvider {
     keywords: [number, number];
     logLevel: number;
     provider_name: string;
-    filter_data: string;
+    filter_data: string | null;
 }
 declare type RemoveCommandSetAndId<T extends ProtocolClientCommandBase> = Omit<T, "command_set" | "command">;
 
 declare type FilterPredicate = (data: ArrayBuffer) => boolean;
 interface MockScriptConnection {
     waitForSend(filter: FilterPredicate): Promise<void>;
     waitForSend<T>(filter: FilterPredicate, extract: (data: ArrayBuffer) => T): Promise<T>;
+    processSend(onMessage: (data: ArrayBuffer) => any): Promise<void>;
     reply(data: ArrayBuffer): void;
 }
 interface MockEnvironmentCommand {
@@ -62,6 +63,8 @@ interface MockEnvironmentReply {
     extractOkSessionID(data: ArrayBuffer): number;
 }
 interface MockEnvironment {
+    postMessageToBrowser(message: any, transferable?: Transferable[]): void;
+    addEventListenerFromBrowser(cmd: string, listener: (data: any) => void): void;
     createPromiseController<T>(): PromiseAndController<T>;
     delay: (ms: number) => Promise<void>;
     command: MockEnvironmentCommand;

src/mono/wasm/runtime/diagnostics-mock.d.ts.sha256

@@ -1 +1 @@
-4a0d47e4e5fabdc42443a54d452be697161a5053d1e24c867200e42742427e3f
+df0de24fd6e67a483f9b3713248d053f912d0cead178e6ec1f537460067495cd

src/mono/wasm/runtime/diagnostics/README.md

@@ -5,10 +5,11 @@ What's in here:
 - `index.ts` toplevel APIs
 - `browser/` APIs for the main thread. The main thread has 2 responsibilities:
   - control the overall diagnostic server `browser/controller.ts`
-- `server_pthread/` A long-running worker that owns the WebSocket connections out of the browser to th ehost and that receives the session payloads from the streaming threads.  The server receives streaming EventPipe data from
-EventPipe streaming threads (that are just ordinary C pthreads) through a shared memory queue and forwards the data to the WebSocket.  The server uses the [DS binary IPC protocol](https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md) which repeatedly opens WebSockets to the host.
+- `server_pthread/` A long-running worker that owns the WebSocket connections out of the browser to the host and that receives the session payloads from the streaming threads.
+  The server receives streaming EventPipe data from EventPipe streaming threads (that are just ordinary C pthreads) through a shared memory queue and forwards the data to the WebSocket.
+  The server uses the [DS binary IPC protocol](https://github.com/dotnet/diagnostics/blob/main/documentation/design-docs/ipc-protocol.md) which repeatedly opens WebSockets to the host.
 - `shared/` type definitions to be shared between the worker and browser main thread
-- `mock/` a utility to fake WebSocket connectings by playing back a script.  Used for prototyping the diagnostic server without hooking up to a real WebSocket.
+- `mock/` a utility to fake WebSocket connections by playing back a script.  Used for prototyping the diagnostic server without hooking up to a real WebSocket.
 
 ## Mocking diagnostics clients
 
@@ -67,30 +68,4 @@ The connection object (of type `MockScriptConnection` defined in [./mock/index.t
 
 ### Mock example
 
-```js
-function script (env) {
-    const sessionStarted = env.createPromiseController(); /* coordinate between the connections */
-    return [
-        async (conn) => {
-            /* first connection.  Expect an ADVR packet */
-            await conn.waitForSend(isAdvertisePacket);
-            conn.reply(makeEventPipeStartCollecting2 ({ "collectRundownEvents": "true", "providers": "WasmHello::5:EventCounterIntervalSec=1" }));
-            /* wait for an "OK" reply with 4 extra bytes of payload, which is the sessionID */
-            const sessionID = await conn.waitForSend(isReplyOK(4), extractSessionID);
-            sessionStarted.promise_control.resolve(sessionID);
-            /* connection kept open. the runtime will send EventPipe data here */
-        },
-        async (conn) => {
-            /* second connection.  Expect an ADVR packet and the sessionStarted sessionID */
-            await Promise.all([conn.waitForSend (isAdvertisePacket); sessionStarted.promise]);
-            /* collect a trace for 5 seconds */
-            await new Promise((resolve) => await new Promise((resolve) => { setTimeout(resolve, 1000); });
-            const sessionID = await sessionStarted.promise;
-            conn.reply(makeEventPipeStopCollecting({sessionID}));
-            /* wait for an "OK" with no payload */
-            await conn.waitForSend(isReplyOK());
-        }
-        /* any further calls to "open" will be an error */
-    ]
-}
-```
+See [browser-eventpipe](../../../sample/wasm/browser-eventpipe/)

src/mono/wasm/runtime/diagnostics/browser/controller.ts

@@ -2,9 +2,11 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 import cwraps from "../../cwraps";
+import { INTERNAL } from "../../imports";
 import { withStackAlloc, getI32 } from "../../memory";
 import { Thread, waitForThread } from "../../pthreads/browser";
 import { isDiagnosticMessage, makeDiagnosticServerControlCommand } from "../shared/controller-commands";
+import monoDiagnosticsMock from "consts:monoDiagnosticsMock";
 
 /// An object that can be used to control the diagnostic server.
 export interface ServerController {
@@ -63,6 +65,9 @@ export async function startDiagnosticServer(websocket_url: string): Promise<Serv
     }
     // have to wait until the message port is created
     const thread = await waitForThread(result);
+    if (monoDiagnosticsMock) {
+        INTERNAL.diagnosticServerThread = thread;
+    }
     if (thread === undefined) {
         throw new Error("unexpected diagnostic server thread not found");
     }