Skip to content

Commit

Permalink
[wasm] write test as single line via base64 (#771)
Browse files Browse the repository at this point in the history 
* save as single base64 line
* check lenght
  • Loading branch information
@pavelsavara
pavelsavara authored Nov 22, 2021
1 parent a584ca9 commit ce11017
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 125 deletions.
62 changes: 20 additions & 42 deletions src/Microsoft.DotNet.XHarness.CLI/Commands/WASM/WasmTestMessagesProcessor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@
using System;
using System.IO;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.Extensions.Logging;

#nullable enable
namespace Microsoft.DotNet.XHarness.CLI.Commands.Wasm
{
public class WasmTestMessagesProcessor
{
private MemoryStream? _xmlResultsMemoryStream;
private StreamWriter? _xmlResultsFileWriter;
private static Regex xmlRx = new Regex(@"^STARTRESULTXML ([0-9]*) ([^ ]*) ENDRESULTXML", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private readonly StreamWriter _stdoutFileWriter;
private readonly string _xmlResultsFilePath;

Expand Down Expand Up @@ -77,16 +76,27 @@ private void InvokeInternal(string message)
line = message.TrimEnd();
}

if (_xmlResultsFileWriter == null)
var match = xmlRx.Match(line);
if (match.Success)
{
if (line.Contains("STARTRESULTXML"))
var expectedLength = Int32.Parse(match.Groups[1].Value);
using (var stream = new FileStream(_xmlResultsFilePath, FileMode.Create))
{
_logger.LogDebug("Reached start of testResults.xml");
_xmlResultsMemoryStream = new MemoryStream();
_xmlResultsFileWriter = new StreamWriter(_xmlResultsMemoryStream);
return;
var bytes = System.Convert.FromBase64String(match.Groups[2].Value);
stream.Write(bytes);
if (bytes.Length == expectedLength)
{
_logger.LogInformation($"Received expected {bytes.Length} of {_xmlResultsFilePath}");
}
else
{
_logger.LogInformation($"Received {bytes.Length} of {_xmlResultsFilePath} but expected {expectedLength}");
}
}
else if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]"))
}
else
{
if (line.StartsWith("[PASS]") || line.StartsWith("[SKIP]"))
{
_logger.LogDebug(line);
}
Expand All @@ -113,38 +123,6 @@ private void InvokeInternal(string message)
if (_stdoutFileWriter.BaseStream.CanWrite)
_stdoutFileWriter.WriteLine(line);
}
else
{
if (line.Contains("ENDRESULTXML"))
{
_logger.LogDebug($"Reached end of {_xmlResultsFilePath}");
_xmlResultsFileWriter.Flush();
_xmlResultsMemoryStream!.Seek(0, SeekOrigin.Begin);
try
{
// we validate it, to make sure it's not corrupted.
var testResults = XElement.Load(_xmlResultsMemoryStream);
using (var testResultsFile = File.CreateText(_xmlResultsFilePath))
{
testResults.Save(testResultsFile);
_logger.LogInformation($"Written {_xmlResultsMemoryStream.Length} original bytes of {_xmlResultsFilePath} as {testResultsFile.BaseStream.Length} formatted bytes");
};
}
catch (Exception ex)
{
_logger.LogError($"Error while saving testResults.xml {ex}");
throw;
}
_xmlResultsFileWriter.Dispose();
_xmlResultsFileWriter = null;
_xmlResultsMemoryStream.Dispose();
_xmlResultsMemoryStream = null;
return;
}

if (_xmlResultsFileWriter?.BaseStream.CanWrite == true)
_xmlResultsFileWriter.WriteLine(line);
}

// the test runner writes this as the last line,
// after the tests have run, and the xml results file
Expand Down
161 changes: 79 additions & 82 deletions src/Microsoft.DotNet.XHarness.TestRunners.Xunit/ThreadlessXunitTestRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@

using System;
using System.Collections.Generic;
using System.IO;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Xml.Linq;
Expand All @@ -21,87 +21,84 @@ namespace Microsoft.DotNet.XHarness.TestRunners.Xunit
{
internal class ThreadlessXunitTestRunner
{
public static async Task<int> Run(string assemblyFileName, bool printXml, XunitFilters filters)
public static async Task<int> Run(string assemblyFileName, bool printXml, XunitFilters filters, bool oneLineResults = false)
{
try
{
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);

discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);

Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");

var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };

testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };

controller.RunTests(testCasesToRun, resultsSink, testOptions);

while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}

if (printXml)
{
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
using (var ms = new MemoryStream())
{
using (var tw = new StreamWriter(ms, Encoding.UTF8, 100*1024, true))
{
// no new line. make it single line, single WS message
resultsXml.Save(tw, SaveOptions.DisableFormatting);
}
ms.Seek(0, SeekOrigin.Begin);
using (var stdout = Console.OpenStandardOutput())
{
using (var twc = new StreamWriter(stdout))
{
twc.WriteLine($"STARTRESULTXML {ms.Length}");
twc.Flush();
ms.CopyTo(stdout, 100 * 1024);
twc.WriteLine();
twc.WriteLine("ENDRESULTXML");
}
}
Console.WriteLine($"Finished writing {ms.Length} bytes of RESULTXML");
}
}

var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return failed ? 1 : 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}");
return 2;
try
{
var configuration = new TestAssemblyConfiguration() { ShadowCopy = false, ParallelizeAssembly = false, ParallelizeTestCollections = false, MaxParallelThreads = 1, PreEnumerateTheories = false };
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
var discoverySink = new TestDiscoverySink();
var diagnosticSink = new ConsoleDiagnosticMessageSink();
var testOptions = TestFrameworkOptions.ForExecution(configuration);
var testSink = new TestMessageSink();
var controller = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyFileName, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false);

discoveryOptions.SetSynchronousMessageReporting(true);
testOptions.SetSynchronousMessageReporting(true);

Console.WriteLine($"Discovering: {assemblyFileName} (method display = {discoveryOptions.GetMethodDisplayOrDefault()}, method display options = {discoveryOptions.GetMethodDisplayOptionsOrDefault()})");
var assembly = Assembly.LoadFrom(assemblyFileName);
var assemblyInfo = new global::Xunit.Sdk.ReflectionAssemblyInfo(assembly);
var discoverer = new ThreadlessXunitDiscoverer(assemblyInfo, new NullSourceInformationProvider(), discoverySink);

discoverer.FindWithoutThreads(includeSourceInformation: false, discoverySink, discoveryOptions);
discoverySink.Finished.WaitOne();
var testCasesToRun = discoverySink.TestCases.Where(filters.Filter).ToList();
Console.WriteLine($"Discovered: {assemblyFileName} (found {testCasesToRun.Count} of {discoverySink.TestCases.Count} test cases)");

var summarySink = new DelegatingExecutionSummarySink(testSink, () => false, (completed, summary) => { Console.WriteLine($"{Environment.NewLine}=== TEST EXECUTION SUMMARY ==={Environment.NewLine}Total: {summary.Total}, Errors: 0, Failed: {summary.Failed}, Skipped: {summary.Skipped}, Time: {TimeSpan.FromSeconds((double)summary.Time).TotalSeconds}s{Environment.NewLine}"); });
var resultsXmlAssembly = new XElement("assembly");
var resultsSink = new DelegatingXmlCreationSink(summarySink, resultsXmlAssembly);

if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null)
{
testSink.Execution.TestStartingEvent += args => { Console.WriteLine($"[STRT] {args.Message.Test.DisplayName}"); };
}
testSink.Execution.TestPassedEvent += args => { Console.WriteLine($"[PASS] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestSkippedEvent += args => { Console.WriteLine($"[SKIP] {args.Message.Test.DisplayName}"); };
testSink.Execution.TestFailedEvent += args => { Console.WriteLine($"[FAIL] {args.Message.Test.DisplayName}{Environment.NewLine}{ExceptionUtility.CombineMessages(args.Message)}{Environment.NewLine}{ExceptionUtility.CombineStackTraces(args.Message)}"); };

testSink.Execution.TestAssemblyStartingEvent += args => { Console.WriteLine($"Starting: {assemblyFileName}"); };
testSink.Execution.TestAssemblyFinishedEvent += args => { Console.WriteLine($"Finished: {assemblyFileName}"); };

controller.RunTests(testCasesToRun, resultsSink, testOptions);

while (!resultsSink.Finished.WaitOne(0))
{
await Task.Delay(1);
}

if (printXml)
{
if (oneLineResults)
{
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
using var sw = new StringWriter();
resultsXml.Save(sw);
var bytes = System.Text.Encoding.UTF8.GetBytes(sw.ToString());
var base64 = Convert.ToBase64String(bytes, Base64FormattingOptions.None);
Console.WriteLine($"STARTRESULTXML {bytes.Length} {base64} ENDRESULTXML");
Console.WriteLine($"Finished writing {bytes.Length} bytes of RESULTXML");
}
else
{
Console.WriteLine($"STARTRESULTXML");
var resultsXml = new XElement("assemblies");
resultsXml.Add(resultsXmlAssembly);
resultsXml.Save(Console.Out);
Console.WriteLine();
Console.WriteLine($"ENDRESULTXML");
}
}

var failed = resultsSink.ExecutionSummary.Failed > 0 || resultsSink.ExecutionSummary.Errors > 0;
return failed ? 1 : 0;
}
catch (Exception ex)
{
Console.Error.WriteLine($"ThreadlessXunitTestRunner failed: {ex}");
return 2;
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/Microsoft.DotNet.XHarness.TestRunners.Xunit/WasmApplicationEntryPoint.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public async Task<int> Run()
foreach (var cl in IncludedClasses) filters.IncludedClasses.Add(cl);
foreach (var me in IncludedMethods) filters.IncludedMethods.Add(me);

var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters);
var result = await ThreadlessXunitTestRunner.Run(TestAssembly, printXml: true, filters, true);

return result;
}
Expand Down

0 comments on commit ce11017

Please sign in to comment.