From f9295ba476336f33d724ec540b17f725d54ef3fe Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 28 Jan 2020 21:53:43 -0500 Subject: [PATCH 01/24] [Xharness] Ignore the first line from the xml logs that is a ping. We ping the tcp listener to know that we have a tcp connection, that is written in the xml logs, which means that parsing will not work. Ignore the ping, parse xml, and make sure that the xml that will be consume by vsts is valid. --- tests/xharness/AppRunner.cs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index f55064cb22e6..fe0a92ec8d28 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -342,13 +342,24 @@ bool IsXml (string filePath) return false; using (var stream = File.OpenText (filePath)) { - var firstLine = stream.ReadLine (); - return firstLine.StartsWith ("<", StringComparison.Ordinal); + string line; + while ((line = stream.ReadLine ()) != null) { + if (line.Contains ("ping")) + continue; + return line.StartsWith ("<", StringComparison.Ordinal); + } } + return false; } bool IsTouchUnitResult (StreamReader stream) { + // more fun, the first like of the stream, is a ping from the application to the tcp server, and that will not be parsable as + // xml, advance the reader one line. + var pingLine = stream.ReadLine(); + if (!pingLine.Contains("ping")) + throw new InvalidDataException("Ping line is missing, unexpected format."); + // TouchUnitTestRun is the very first node in the TouchUnit xml result // which is not preset in the xunit xml, therefore we know the runner // quite quickly @@ -371,6 +382,8 @@ bool IsTouchUnitResult (StreamReader stream) { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; + // ignore the first line + stream.ReadLine (); using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { @@ -399,6 +412,8 @@ bool IsTouchUnitResult (StreamReader stream) { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; + // ignore the first line + stream.ReadLine(); using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { @@ -508,6 +523,8 @@ bool IsTouchUnitResult (StreamReader stream) using (var xmlWriter = new StreamWriter (path)) { string line; while ((line = streamReaderTmp.ReadLine ()) != null) { + if (line.Contains ("ping")) // ignore the ping, because VSTS is going to have issues too. + continue; if (line.Contains (" Date: Wed, 29 Jan 2020 05:38:13 -0500 Subject: [PATCH 02/24] Apply suggestions from code review Co-Authored-By: Rolf Bjarne Kvinge --- tests/xharness/AppRunner.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index fe0a92ec8d28..64a2d8fa2fc8 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -356,9 +356,9 @@ bool IsTouchUnitResult (StreamReader stream) { // more fun, the first like of the stream, is a ping from the application to the tcp server, and that will not be parsable as // xml, advance the reader one line. - var pingLine = stream.ReadLine(); - if (!pingLine.Contains("ping")) - throw new InvalidDataException("Ping line is missing, unexpected format."); + var pingLine = stream.ReadLine (); + if (!pingLine.Contains ("ping")) + throw new InvalidDataException ("Ping line is missing, unexpected format."); // TouchUnitTestRun is the very first node in the TouchUnit xml result // which is not preset in the xunit xml, therefore we know the runner @@ -413,7 +413,7 @@ bool IsTouchUnitResult (StreamReader stream) long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; // ignore the first line - stream.ReadLine(); + stream.ReadLine (); using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { From c7a27d6d03dc63627c987034748e8ff5f13a6083 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 29 Jan 2020 05:42:09 -0500 Subject: [PATCH 03/24] Do not throw exception, got to pos 0. --- tests/xharness/AppRunner.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index fe0a92ec8d28..817a02695e3a 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -358,7 +358,7 @@ bool IsTouchUnitResult (StreamReader stream) // xml, advance the reader one line. var pingLine = stream.ReadLine(); if (!pingLine.Contains("ping")) - throw new InvalidDataException("Ping line is missing, unexpected format."); + stream.BaseStream.Position = 0; // TouchUnitTestRun is the very first node in the TouchUnit xml result // which is not preset in the xunit xml, therefore we know the runner @@ -383,7 +383,9 @@ bool IsTouchUnitResult (StreamReader stream) long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; // ignore the first line - stream.ReadLine (); + var ping = stream.ReadLine (); + if (!ping.Contains ("ping")) + stream.BaseStream.Position = 0; using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { @@ -413,7 +415,9 @@ bool IsTouchUnitResult (StreamReader stream) long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; // ignore the first line - stream.ReadLine(); + var ping = stream.ReadLine(); + if (!ping.Contains ("ping")) + stream.BaseStream.Position = 0; using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { From 12c625ddd624bbcfed4cee4c9eaf97bf4065c338 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Wed, 29 Jan 2020 17:31:01 -0500 Subject: [PATCH 04/24] Add logging when we cannot parse the data. --- tests/xharness/AppRunner.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 319d83f4fba1..229894e27fe7 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -547,6 +547,18 @@ bool IsTouchUnitResult (StreamReader stream) return parseResult; } catch (Exception e) { main_log.WriteLine ("Could not parse xml result file: {0}", e); + // print file for better debugging + main_log.WriteLine ("File data is:"); + main_log.WriteLine (new string ('#', 10)); + using (var streamReaderTmp = new StreamReader (tmpFile)) { + string line; + while ((line = streamReaderTmp.ReadLine ()) != null) { + main_log.WriteLine (line); + } + } + main_log.WriteLine (new string ('#', 10)); + main_log.WriteLine ("End of xml results."); + if (timed_out) { Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
"); From d118a6bd6babb0d4c8c48d4a2a3e9043e3705ce8 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 30 Jan 2020 10:35:52 -0500 Subject: [PATCH 05/24] Copy over Move in case we continue writing to the same fd. Move printing logic closer to the error. --- tests/xharness/AppRunner.cs | 40 +++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 229894e27fe7..f662a1c91b07 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -364,13 +364,27 @@ bool IsTouchUnitResult (StreamReader stream) // which is not preset in the xunit xml, therefore we know the runner // quite quickly bool isTouchUnit = false; - using (var reader = XmlReader.Create (stream)) { - while (reader.Read ()) { - if (reader.NodeType == XmlNodeType.Element && reader.Name == "TouchUnitTestRun") { - isTouchUnit = true; - break; + try { + using (var reader = XmlReader.Create (stream)) { + while (reader.Read ()) { + if (reader.NodeType == XmlNodeType.Element && reader.Name == "TouchUnitTestRun") { + isTouchUnit = true; + break; + } } } + } catch (XmlException e) { + main_log.WriteLine ($"Could not determine if touch unit {e}"); + // print file for better debugging + main_log.WriteLine ("File data is:"); + main_log.WriteLine (new string ('#', 10)); + stream.BaseStream.Position = 0; + string line; + while ((line = stream.ReadLine ()) != null) { + main_log.WriteLine (line); + } + main_log.WriteLine (new string ('#', 10)); + main_log.WriteLine ("End of xml results."); } // we want to reuse the stream (and we are sync) stream.BaseStream.Position = 0; @@ -501,7 +515,8 @@ bool IsTouchUnitResult (StreamReader stream) // write the human readable log var tmpFile = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString ()); - File.Move (listener_log.FullPath, tmpFile); + // copy do not move + File.Copy (listener_log.FullPath, tmpFile, true); crashed = false; try { using (var streamReaderTmp = new StreamReader (tmpFile)) { @@ -547,19 +562,6 @@ bool IsTouchUnitResult (StreamReader stream) return parseResult; } catch (Exception e) { main_log.WriteLine ("Could not parse xml result file: {0}", e); - // print file for better debugging - main_log.WriteLine ("File data is:"); - main_log.WriteLine (new string ('#', 10)); - using (var streamReaderTmp = new StreamReader (tmpFile)) { - string line; - while ((line = streamReaderTmp.ReadLine ()) != null) { - main_log.WriteLine (line); - } - } - main_log.WriteLine (new string ('#', 10)); - main_log.WriteLine ("End of xml results."); - - if (timed_out) { Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
"); return parseResult; From 8d08117904a265a4eb6a62b500b489e91d47aea1 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 30 Jan 2020 10:47:31 -0500 Subject: [PATCH 06/24] Ignore extra nodes from TouchUnit. --- tests/xharness/AppRunner.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index f662a1c91b07..82e2ad7c6793 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -542,9 +542,11 @@ bool IsTouchUnitResult (StreamReader stream) using (var xmlWriter = new StreamWriter (path)) { string line; while ((line = streamReaderTmp.ReadLine ()) != null) { - if (line.Contains ("ping")) // ignore the ping, because VSTS is going to have issues too. + if (line.Contains ("ping") || line.Contains ("") || line.Contains ("")) // ignore the ping, because VSTS is going to have issues too or the Touch unit elements continue; - if (line.Contains ("")) // get out of the loop we are not interested in the rest of the TouchUnit data. + break; + if (line.Contains (" Date: Thu, 30 Jan 2020 22:09:59 -0500 Subject: [PATCH 07/24] Use a socket over a stream. This is an attempt to fix https://github.com/xamarin/maccore/issues/827 by moving away from the stream and too a socket. If we got a 0, the user disconnected and we do not write anything to the log. In theory this should stop writing as soon as the client is out and fix the issue. --- tests/xharness/SimpleTcpListener.cs | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tests/xharness/SimpleTcpListener.cs b/tests/xharness/SimpleTcpListener.cs index 143be06402e6..9f0d988197f0 100644 --- a/tests/xharness/SimpleTcpListener.cs +++ b/tests/xharness/SimpleTcpListener.cs @@ -14,6 +14,7 @@ public class SimpleTcpListener : SimpleListener protected override void Stop () { + // TODO: should we close the accepted socket? server.Stop (); } @@ -33,7 +34,7 @@ protected override void Start () try { do { Log.WriteLine ("Test log server listening on: {0}:{1}", Address, Port); - using (TcpClient client = server.AcceptTcpClient ()) { + using (var client = server.AcceptSocket ()) { processed = Processing (client); } } while (!AutoExit || !processed); @@ -50,18 +51,30 @@ protected override void Start () } } - bool Processing (TcpClient client) + bool Processing (Socket client) { - Connected (client.Client.RemoteEndPoint.ToString ()); - // now simply copy what we receive - int i; - int total = 0; - NetworkStream stream = client.GetStream (); + Connected (client.RemoteEndPoint.ToString ()); var fs = OutputWriter; - while ((i = stream.Read (buffer, 0, buffer.Length)) != 0) { - fs.Write (buffer, 0, i); - fs.Flush (); - total += i; + int total = 0; + // now simply copy what we receive but using the socket, not the stream + // this is due to https://github.com/xamarin/maccore/issues/827 + while (true) { + byte [] buffer = new byte [client.ReceiveBufferSize]; + int read = 0; + + try { + read = client.Receive (buffer); + } catch { // lost connection, bad :/ + break; + } + + if (read > 0) { //Handle data + fs.Write (buffer, 0, read); + fs.Flush (); + total += read; + } else { // read 0 means that client disconnected + break; + } } if (total < 16) { From 1942c11569f11a5cfc94800595ff9b564759aa27 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Thu, 30 Jan 2020 22:16:43 -0500 Subject: [PATCH 08/24] Do not write the line twice, that is a mistake. --- tests/xharness/AppRunner.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 82e2ad7c6793..ed852d8aaf97 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -549,7 +549,6 @@ bool IsTouchUnitResult (StreamReader stream) if (line.Contains (" Date: Thu, 30 Jan 2020 22:23:58 -0500 Subject: [PATCH 09/24] We are not writting the root node on xunit. --- tests/xharness/AppRunner.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index ed852d8aaf97..ba916a91815f 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -551,6 +551,8 @@ bool IsTouchUnitResult (StreamReader stream) xmlWriter.WriteLine (line.Replace ("name=\"\"", $"name=\"{appName + " " + configuration}\"")); } else if (line.Contains ($"name=\"com.xamarin.bcltests.{appName}\"")) { // xunit case xmlWriter.WriteLine (line.Replace ($"name=\"com.xamarin.bcltests.{appName}\"", $"name=\"{appName + " " + configuration}\"")); + } else { + xmlWriter.WriteLine (line); } } else { xmlWriter.WriteLine (line); From acf634c733e4cc7f1ce6f96751c83c4c49c560cd Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 31 Jan 2020 05:09:34 -0500 Subject: [PATCH 10/24] Undo the changees in the listener. --- tests/xharness/SimpleTcpListener.cs | 36 +++++++++-------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/tests/xharness/SimpleTcpListener.cs b/tests/xharness/SimpleTcpListener.cs index 9f0d988197f0..b0fb42915683 100644 --- a/tests/xharness/SimpleTcpListener.cs +++ b/tests/xharness/SimpleTcpListener.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Net; using System.Net.Sockets; @@ -14,7 +14,6 @@ public class SimpleTcpListener : SimpleListener protected override void Stop () { - // TODO: should we close the accepted socket? server.Stop (); } @@ -34,7 +33,7 @@ protected override void Start () try { do { Log.WriteLine ("Test log server listening on: {0}:{1}", Address, Port); - using (var client = server.AcceptSocket ()) { + using (TcpClient client = server.AcceptTcpClient ()) { processed = Processing (client); } } while (!AutoExit || !processed); @@ -51,30 +50,17 @@ protected override void Start () } } - bool Processing (Socket client) + bool Processing (TcpClient client) { - Connected (client.RemoteEndPoint.ToString ()); - var fs = OutputWriter; + Connected (client.Client.RemoteEndPoint.ToString ()); + // now simply copy what we receive int total = 0; - // now simply copy what we receive but using the socket, not the stream - // this is due to https://github.com/xamarin/maccore/issues/827 - while (true) { - byte [] buffer = new byte [client.ReceiveBufferSize]; - int read = 0; - - try { - read = client.Receive (buffer); - } catch { // lost connection, bad :/ - break; - } - - if (read > 0) { //Handle data - fs.Write (buffer, 0, read); - fs.Flush (); - total += read; - } else { // read 0 means that client disconnected - break; - } + NetworkStream stream = client.GetStream (); + var fs = OutputWriter; + while ((i = stream.Read (buffer, 0, buffer.Length)) != 0) { + fs.Write (buffer, 0, i); + fs.Flush (); + total += i; } if (total < 16) { From ec957ea3d61b6877d3844cd911aa8f9a12d1e812 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 31 Jan 2020 05:27:54 -0500 Subject: [PATCH 11/24] Fix NUnit failure parsing. --- tests/xharness/AppRunner.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index ba916a91815f..fa2014d30a74 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -475,6 +475,9 @@ bool IsTouchUnitResult (StreamReader stream) } writer.Write (reader ["name"]); if (status == "Failure" || status == "Error") { // we need to print the message + reader.ReadToDescendant ("message"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + reader.ReadToNextSibling ("stack-trace"); writer.Write ($" : {reader.ReadElementContentAsString ()}"); } // add a new line From 77c1d980166b92a3e7a97ba30b2b4eda2961388c Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 31 Jan 2020 05:46:30 -0500 Subject: [PATCH 12/24] Use two file formats and do not trust the xslt which gives problems in VSTS. --- .../templates/common/TestRunner.xUnit/XUnitTestRunner.cs | 2 +- tests/xharness/AppRunner.cs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs index 7199b5114ba6..e61b1ad8c688 100644 --- a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs @@ -23,7 +23,7 @@ public class XUnitTestRunner : TestRunner List filters = new List (); bool runAssemblyByDefault; - public XUnitResultFileFormat ResultFileFormat { get; set; } = XUnitResultFileFormat.NUnit; + public XUnitResultFileFormat ResultFileFormat { get; set; } = XUnitResultFileFormat.XunitV2; public AppDomainSupport AppDomainSupport { get; set; } = AppDomainSupport.Denied; protected override string ResultsFileName { get; set; } = "TestResults.xUnit.xml"; diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index fa2014d30a74..7feae8c952c7 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -540,6 +540,11 @@ bool IsTouchUnitResult (StreamReader stream) streamReaderTmp.DiscardBufferedData (); var path = listener_log.FullPath; path = Path.ChangeExtension (path, "xml"); + var fileName = Path.GetFileName (path); + if (isTouchUnit) + path = path.Replace (fileName, "nunit-" + fileName); + else + path = path.Replace (fileName, "xunit-" + fileName); // both the nunit and xunit runners are not // setting the test results correctly, lets add them using (var xmlWriter = new StreamWriter (path)) { From 0dda13592b331a10752cfd80eee8e044b31e6d96 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 31 Jan 2020 06:10:51 -0500 Subject: [PATCH 13/24] Fix simpler listener. --- tests/xharness/SimpleTcpListener.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/xharness/SimpleTcpListener.cs b/tests/xharness/SimpleTcpListener.cs index b0fb42915683..b2894ab275f8 100644 --- a/tests/xharness/SimpleTcpListener.cs +++ b/tests/xharness/SimpleTcpListener.cs @@ -54,6 +54,7 @@ bool Processing (TcpClient client) { Connected (client.Client.RemoteEndPoint.ToString ()); // now simply copy what we receive + int i; int total = 0; NetworkStream stream = client.GetStream (); var fs = OutputWriter; From a6d9eb2dd5cd752a037df1081fc78cf99c57b5fb Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Fri, 31 Jan 2020 10:24:50 -0500 Subject: [PATCH 14/24] Renaming the files wont work, we have to be smarter. Leave it for a diff PR. --- tests/xharness/AppRunner.cs | 6 +----- tests/xharness/SimpleTcpListener.cs | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 7feae8c952c7..119248c4462b 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -540,11 +540,6 @@ bool IsTouchUnitResult (StreamReader stream) streamReaderTmp.DiscardBufferedData (); var path = listener_log.FullPath; path = Path.ChangeExtension (path, "xml"); - var fileName = Path.GetFileName (path); - if (isTouchUnit) - path = path.Replace (fileName, "nunit-" + fileName); - else - path = path.Replace (fileName, "xunit-" + fileName); // both the nunit and xunit runners are not // setting the test results correctly, lets add them using (var xmlWriter = new StreamWriter (path)) { @@ -573,6 +568,7 @@ bool IsTouchUnitResult (StreamReader stream) return parseResult; } catch (Exception e) { main_log.WriteLine ("Could not parse xml result file: {0}", e); + if (timed_out) { Harness.LogWrench ($"@MonkeyWrench: AddSummary: {mode} timed out
"); return parseResult; diff --git a/tests/xharness/SimpleTcpListener.cs b/tests/xharness/SimpleTcpListener.cs index b2894ab275f8..143be06402e6 100644 --- a/tests/xharness/SimpleTcpListener.cs +++ b/tests/xharness/SimpleTcpListener.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Net; using System.Net.Sockets; From 361a3af5b57f5154f200f259733c47e115663f4c Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sat, 1 Feb 2020 22:37:20 -0500 Subject: [PATCH 15/24] Fix issues for device tests and xml. --- .../TestRunner.NUnit/NUnitTestRunner.cs | 2 + .../TestRunner.xUnit/XUnitTestRunner.cs | 4 +- tests/xharness/AppRunner.cs | 132 ++++++++++++++++-- 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs index bb94b352ce0a..0af04266639a 100644 --- a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs @@ -274,7 +274,9 @@ public override void WriteResultsToFile (TextWriter writer) if (results == null) return; var resultsXml = new NUnit2XmlOutputWriter (DateTime.UtcNow); + writer.WriteLine (""); resultsXml.WriteResultFile (results, writer); + writer.WriteLine (""); } void AppendFilter (ITestFilter filter) diff --git a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs index e61b1ad8c688..3c06faaf2b11 100644 --- a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs @@ -818,6 +818,7 @@ public override void WriteResultsToFile (TextWriter writer) { if (assembliesElement == null) return; + writer.WriteLine (""); var settings = new XmlWriterSettings { Indent = true }; using (var xmlWriter = XmlWriter.Create (writer, settings)) { switch (ResultFileFormat) { @@ -836,8 +837,9 @@ public override void WriteResultsToFile (TextWriter writer) throw new InvalidOperationException ($"Result output format '{ResultFileFormat}' is not currently supported"); } } + writer.WriteLine (""); } - + void Transform_Results (string xsltResourceName, XElement element, XmlWriter writer) { var xmlTransform = new System.Xml.Xsl.XslCompiledTransform (); diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 119248c4462b..9b879e8d0864 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -34,6 +34,13 @@ public enum Extension TodayExtension, } + public enum XmlResultType + { + TouchUnit, + NUnit, + xUnit, + } + public class AppRunner { public Harness Harness; @@ -352,8 +359,9 @@ bool IsXml (string filePath) return false; } - bool IsTouchUnitResult (StreamReader stream) + XmlResultType GetXmlType (StreamReader stream) { + var resultType = XmlResultType.TouchUnit; // default // more fun, the first like of the stream, is a ping from the application to the tcp server, and that will not be parsable as // xml, advance the reader one line. var pingLine = stream.ReadLine (); @@ -363,14 +371,31 @@ bool IsTouchUnitResult (StreamReader stream) // TouchUnitTestRun is the very first node in the TouchUnit xml result // which is not preset in the xunit xml, therefore we know the runner // quite quickly - bool isTouchUnit = false; try { using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "TouchUnitTestRun") { - isTouchUnit = true; + resultType = XmlResultType.TouchUnit; break; } + if (reader.NodeType == XmlNodeType.Element && reader.Name == "environment") { + // get the enviroment attr + if (reader.HasAttributes) { + var nunitAttr = reader.GetAttribute ("nunit-version"); + if (nunitAttr == null) { + resultType = XmlResultType.NUnit; + break; + } + } + } + if (reader.NodeType == XmlNodeType.Element && reader.Name == "assembly") { + if (reader.HasAttributes) { + var framework = reader.GetAttribute ("test-framework"); + if (framework != null && framework.Contains ("xUnit")) { + resultType = XmlResultType.xUnit; + } + } + } } } } catch (XmlException e) { @@ -389,7 +414,7 @@ bool IsTouchUnitResult (StreamReader stream) // we want to reuse the stream (and we are sync) stream.BaseStream.Position = 0; stream.DiscardBufferedData (); - return isTouchUnit; + return resultType; } (string resultLine, bool failed) ParseTouchUnitXml (StreamReader stream, StreamWriter writer) @@ -493,6 +518,67 @@ bool IsTouchUnitResult (StreamReader stream) return (resultLine, total == 0 | errors != 0 || failed != 0); } + + (string resultLine, bool failed) ParsexUnitXml (StreamReader stream, StreamWriter writer) { + long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; + total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; + // ignore the first line + var ping = stream.ReadLine (); + if (!ping.Contains ("ping")) + stream.BaseStream.Position = 0; + using (var reader = XmlReader.Create (stream)) { + while (reader.Read ()) { + if (reader.NodeType == XmlNodeType.Element && reader.Name == "assembly") { + total += long.Parse (reader ["total"]); + errors += long.Parse (reader ["errors"]); + failed += long.Parse (reader ["failed"]); + skipped += long.Parse (reader ["skipped"]); + } + if (reader.NodeType == XmlNodeType.Element && reader.Name == "collection") { + var testCaseName = reader ["name"].Replace ("Test collection for ", ""); + writer.WriteLine (testCaseName); + var time = reader.GetAttribute ("time") ?? "0"; // some nodes might not have the time :/ + // get the first node and then move in the siblings of the same type + reader.ReadToDescendant ("test"); + do { + if (reader.Name != "test") + break; + // read the test cases in the current node + var status = reader ["result"]; + switch (status) { + case "Pass": + writer.Write ("\t[PASS] "); + break; + case "Skip": + writer.Write ("\t[IGNORED] "); + break; + case "Fail": + writer.Write ("\t[FAIL] "); + break; + default: + writer.Write ("\t[FAIL] "); + break; + } + writer.Write (reader ["name"]); + if (status == "Fail") { // we need to print the message + reader.ReadToDescendant ("message"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + reader.ReadToNextSibling ("stack-trace"); + writer.Write ($" : {reader.ReadElementContentAsString ()}"); + } + // add a new line + writer.WriteLine (); + } while (reader.ReadToNextSibling ("test")); + writer.WriteLine ($"{testCaseName} {time} ms"); + } + } + } + var passed = total - errors - failed - notRun - inconclusive - ignored - skipped - invalid; + string resultLine = $"Tests run: {total} Passed: {passed} Inconclusive: {inconclusive} Failed: {failed + errors} Ignored: {ignored + skipped + invalid}"; + writer.WriteLine (resultLine); + + return (resultLine, total == 0 | errors != 0 || failed != 0); + } (string resultLine, bool failed, bool crashed) ParseResult (Log listener_log, bool timed_out, bool crashed) { @@ -516,30 +602,48 @@ bool IsTouchUnitResult (StreamReader stream) // move the xml to a tmp path, that path will be use to read the xml // in the reader, and the writer will use the stream from the logger to // write the human readable log - var tmpFile = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString ()); + var tmpFile = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString ()); + // copy do not move File.Copy (listener_log.FullPath, tmpFile, true); crashed = false; try { using (var streamReaderTmp = new StreamReader (tmpFile)) { - var isTouchUnit = IsTouchUnitResult (streamReaderTmp); // method resets position + var xmlType = GetXmlType (streamReaderTmp); using (var writer = new StreamWriter (listener_log.FullPath, true)) { // write the human result to the log file - if (isTouchUnit) { - var (resultLine, failed)= ParseTouchUnitXml (streamReaderTmp, writer); - parseResult.resultLine = resultLine; - parseResult.failed = failed; - } else { - var (resultLine, failed)= ParseNUnitXml (streamReaderTmp, writer); - parseResult.resultLine = resultLine; - parseResult.failed = failed; + (string resultLine, bool failed) parseData; + switch (xmlType) { + case XmlResultType.TouchUnit: + parseData = ParseTouchUnitXml (streamReaderTmp, writer); + break; + case XmlResultType.NUnit: + parseData = ParseNUnitXml (streamReaderTmp, writer); + break; + case XmlResultType.xUnit: + parseData = ParsexUnitXml (streamReaderTmp, writer); + break; + default: + throw new Exception ("OMG"); } + parseResult.resultLine = parseData.resultLine; + parseResult.failed = parseData.failed; } // reset pos of the stream streamReaderTmp.BaseStream.Position = 0; streamReaderTmp.DiscardBufferedData (); var path = listener_log.FullPath; path = Path.ChangeExtension (path, "xml"); + var fileName = Path.GetFileName (path); + switch (xmlType) { + case XmlResultType.TouchUnit: + case XmlResultType.NUnit: + path.Replace (fileName, $"nunit-{fileName}"); + break; + case XmlResultType.xUnit: + path.Replace (fileName, $"xunit-{fileName}"); + break; + } // both the nunit and xunit runners are not // setting the test results correctly, lets add them using (var xmlWriter = new StreamWriter (path)) { From 6546e7ce54eaa7d76496c0ad316e61b06946ae00 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sun, 2 Feb 2020 07:35:38 -0500 Subject: [PATCH 16/24] Use the correct path for the upload. --- tests/xharness/AppRunner.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 9b879e8d0864..15a815639592 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -638,10 +638,10 @@ XmlResultType GetXmlType (StreamReader stream) switch (xmlType) { case XmlResultType.TouchUnit: case XmlResultType.NUnit: - path.Replace (fileName, $"nunit-{fileName}"); + path = path.Replace (fileName, $"nunit-{fileName}"); break; case XmlResultType.xUnit: - path.Replace (fileName, $"xunit-{fileName}"); + path = path.Replace (fileName, $"xunit-{fileName}"); break; } // both the nunit and xunit runners are not From 2dbacdc60f486dc785e31fd9eef30758504844df Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sun, 2 Feb 2020 09:59:13 -0500 Subject: [PATCH 17/24] Remove xml junk trying to fix https://github.com/xamarin/maccore/issues/827 --- tests/xharness/AppRunner.cs | 146 ++++++++++++++---------------------- 1 file changed, 57 insertions(+), 89 deletions(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 15a815639592..c129ef4a26fc 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -364,53 +364,20 @@ XmlResultType GetXmlType (StreamReader stream) var resultType = XmlResultType.TouchUnit; // default // more fun, the first like of the stream, is a ping from the application to the tcp server, and that will not be parsable as // xml, advance the reader one line. - var pingLine = stream.ReadLine (); - if (!pingLine.Contains ("ping")) - stream.BaseStream.Position = 0; - - // TouchUnitTestRun is the very first node in the TouchUnit xml result - // which is not preset in the xunit xml, therefore we know the runner - // quite quickly - try { - using (var reader = XmlReader.Create (stream)) { - while (reader.Read ()) { - if (reader.NodeType == XmlNodeType.Element && reader.Name == "TouchUnitTestRun") { - resultType = XmlResultType.TouchUnit; - break; - } - if (reader.NodeType == XmlNodeType.Element && reader.Name == "environment") { - // get the enviroment attr - if (reader.HasAttributes) { - var nunitAttr = reader.GetAttribute ("nunit-version"); - if (nunitAttr == null) { - resultType = XmlResultType.NUnit; - break; - } - } - } - if (reader.NodeType == XmlNodeType.Element && reader.Name == "assembly") { - if (reader.HasAttributes) { - var framework = reader.GetAttribute ("test-framework"); - if (framework != null && framework.Contains ("xUnit")) { - resultType = XmlResultType.xUnit; - } - } - } - } + string line; + while ((line = stream.ReadLine ()) != null) { + if (line.Contains ("TouchUnitTestRun")) { + resultType = XmlResultType.TouchUnit; + break; } - } catch (XmlException e) { - main_log.WriteLine ($"Could not determine if touch unit {e}"); - // print file for better debugging - main_log.WriteLine ("File data is:"); - main_log.WriteLine (new string ('#', 10)); - stream.BaseStream.Position = 0; - string line; - while ((line = stream.ReadLine ()) != null) { - main_log.WriteLine (line); + if (line.Contains ("nunit-version")) + resultType = XmlResultType.NUnit; + if (line.Contains ("xUnit")) { + resultType = XmlResultType.xUnit; + break; } - main_log.WriteLine (new string ('#', 10)); - main_log.WriteLine ("End of xml results."); } + // we want to reuse the stream (and we are sync) stream.BaseStream.Position = 0; stream.DiscardBufferedData (); @@ -421,10 +388,7 @@ XmlResultType GetXmlType (StreamReader stream) { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; - // ignore the first line - var ping = stream.ReadLine (); - if (!ping.Contains ("ping")) - stream.BaseStream.Position = 0; + using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { @@ -453,11 +417,9 @@ XmlResultType GetXmlType (StreamReader stream) { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; - // ignore the first line - var ping = stream.ReadLine (); - if (!ping.Contains ("ping")) - stream.BaseStream.Position = 0; - using (var reader = XmlReader.Create (stream)) { + XmlReaderSettings settings = new XmlReaderSettings (); + settings.ValidationType = ValidationType.None; + using (var reader = XmlReader.Create (stream, settings)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "test-results") { total = long.Parse (reader ["total"]); @@ -522,10 +484,6 @@ XmlResultType GetXmlType (StreamReader stream) (string resultLine, bool failed) ParsexUnitXml (StreamReader stream, StreamWriter writer) { long total, errors, failed, notRun, inconclusive, ignored, skipped, invalid; total = errors = failed = notRun = inconclusive = ignored = skipped = invalid = 0L; - // ignore the first line - var ping = stream.ReadLine (); - if (!ping.Contains ("ping")) - stream.BaseStream.Position = 0; using (var reader = XmlReader.Create (stream)) { while (reader.Read ()) { if (reader.NodeType == XmlNodeType.Element && reader.Name == "assembly") { @@ -604,35 +562,14 @@ XmlResultType GetXmlType (StreamReader stream) // write the human readable log var tmpFile = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString ()); - // copy do not move File.Copy (listener_log.FullPath, tmpFile, true); crashed = false; + string path = listener_log.FullPath; + XmlResultType xmlType; try { using (var streamReaderTmp = new StreamReader (tmpFile)) { - var xmlType = GetXmlType (streamReaderTmp); - using (var writer = new StreamWriter (listener_log.FullPath, true)) { // write the human result to the log file - (string resultLine, bool failed) parseData; - switch (xmlType) { - case XmlResultType.TouchUnit: - parseData = ParseTouchUnitXml (streamReaderTmp, writer); - break; - case XmlResultType.NUnit: - parseData = ParseNUnitXml (streamReaderTmp, writer); - break; - case XmlResultType.xUnit: - parseData = ParsexUnitXml (streamReaderTmp, writer); - break; - default: - throw new Exception ("OMG"); - } - parseResult.resultLine = parseData.resultLine; - parseResult.failed = parseData.failed; - } - // reset pos of the stream - streamReaderTmp.BaseStream.Position = 0; - streamReaderTmp.DiscardBufferedData (); - var path = listener_log.FullPath; + xmlType = GetXmlType (streamReaderTmp); path = Path.ChangeExtension (path, "xml"); var fileName = Path.GetFileName (path); switch (xmlType) { @@ -644,16 +581,15 @@ XmlResultType GetXmlType (StreamReader stream) path = path.Replace (fileName, $"xunit-{fileName}"); break; } - // both the nunit and xunit runners are not - // setting the test results correctly, lets add them + // clean any junk we have using (var xmlWriter = new StreamWriter (path)) { string line; while ((line = streamReaderTmp.ReadLine ()) != null) { - if (line.Contains ("ping") || line.Contains ("") || line.Contains ("")) // ignore the ping, because VSTS is going to have issues too or the Touch unit elements + if (line.Contains ("ping") || line.Contains ("") || line.Contains ("") || line.StartsWith ("")) // get out of the loop we are not interested in the rest of the TouchUnit data. break; - if (line.Contains ("{mode} timed out
"); return parseResult; @@ -1267,4 +1235,4 @@ public override void Write (byte [] buffer, int offset, int count) copy_to.Write (buffer, offset, count); } } -} +} \ No newline at end of file From adc5b0c8906f3f60174e578a0b824471bb13c026 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sun, 2 Feb 2020 12:25:50 -0500 Subject: [PATCH 18/24] Do not trust NUnitLite xml writer. It writes bad formed xml with categoires and we are not interested. --- tests/bcl-test/BCLTests-tv.csproj.in | 3 + .../BCLTests-watchos-extension.csproj.in | 3 + tests/bcl-test/BCLTests.csproj.in | 3 + .../TestRunner.NUnit/NUnitTestRunner.cs | 2 - .../TestRunner.NUnit/XmlOutputWriter.cs | 345 ++++++++++++++++++ 5 files changed, 354 insertions(+), 2 deletions(-) create mode 100644 tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs diff --git a/tests/bcl-test/BCLTests-tv.csproj.in b/tests/bcl-test/BCLTests-tv.csproj.in index 92cc86c1f0b2..a28d298f7e58 100644 --- a/tests/bcl-test/BCLTests-tv.csproj.in +++ b/tests/bcl-test/BCLTests-tv.csproj.in @@ -176,6 +176,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.Core\Extensions.Bool.cs diff --git a/tests/bcl-test/BCLTests-watchos-extension.csproj.in b/tests/bcl-test/BCLTests-watchos-extension.csproj.in index 02b661163cc1..d8334ec83454 100644 --- a/tests/bcl-test/BCLTests-watchos-extension.csproj.in +++ b/tests/bcl-test/BCLTests-watchos-extension.csproj.in @@ -206,6 +206,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.xUnit\XUnitFilter.cs diff --git a/tests/bcl-test/BCLTests.csproj.in b/tests/bcl-test/BCLTests.csproj.in index 1d60c5faae8b..cb5242ecfa47 100644 --- a/tests/bcl-test/BCLTests.csproj.in +++ b/tests/bcl-test/BCLTests.csproj.in @@ -189,6 +189,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.Core\Extensions.Bool.cs diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs index 0af04266639a..2550f22d0cd3 100644 --- a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs @@ -9,10 +9,8 @@ using Foundation; -using NUnitLite.Runner; using NUnit.Framework.Api; using NUnit.Framework.Internal; -using NUnit.Framework.Internal.WorkItems; using NUnit.Framework.Internal.Filters; using NUnitTest = NUnit.Framework.Internal.Test; diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs new file mode 100644 index 000000000000..93d925d38d50 --- /dev/null +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs @@ -0,0 +1,345 @@ +// *********************************************************************** +// Copyright (c) 2011 Charlie Poole +// +// Permission is hereby granted, free of charge, to any person obtaining +// a copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to +// permit persons to whom the Software is furnished to do so, subject to +// the following conditions: +// +// The above copyright notice and this permission notice shall be +// included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +// *********************************************************************** + +using System; +using System.Globalization; +using System.Reflection; +using System.Xml; +using System.IO; +using NUnit.Framework.Api; +using NUnit.Framework.Internal; +using NUnitLite.Runner; +#if CLR_2_0 || CLR_4_0 +using System.Collections.Generic; +#else +using System.Collections.Specialized; +#endif + +namespace Xamarin.iOS.UnitTests.NUnit { + /// + /// NUnit2XmlOutputWriter is able to create an xml file representing + /// the result of a test run in NUnit 2.x format. + /// + public class NUnit2XmlOutputWriter : OutputWriter { + private XmlWriter xmlWriter; + private DateTime startTime; + +#if CLR_2_0 || CLR_4_0 + private static Dictionary resultStates = new Dictionary(); +#else + private static StringDictionary resultStates = new StringDictionary (); +#endif + + static NUnit2XmlOutputWriter () + { + resultStates ["Passed"] = "Success"; + resultStates ["Failed"] = "Failure"; + resultStates ["Failed:Error"] = "Error"; + resultStates ["Failed:Cancelled"] = "Cancelled"; + resultStates ["Inconclusive"] = "Inconclusive"; + resultStates ["Skipped"] = "Skipped"; + resultStates ["Skipped:Ignored"] = "Ignored"; + resultStates ["Skipped:Invalid"] = "NotRunnable"; + } + + public NUnit2XmlOutputWriter (DateTime startTime) + { + this.startTime = startTime; + } + + /// + /// Writes the result of a test run to a specified TextWriter. + /// + /// The test result for the run + /// The TextWriter to which the xml will be written + public override void WriteResultFile (ITestResult result, TextWriter writer) + { + // NOTE: Under .NET 1.1, XmlTextWriter does not implement IDisposable, + // but does implement Close(). Hence we cannot use a 'using' clause. + //using (XmlTextWriter xmlWriter = new XmlTextWriter(writer)) +#if SILVERLIGHT + XmlWriter xmlWriter = XmlWriter.Create(writer); +#else + XmlTextWriter xmlWriter = new XmlTextWriter (writer); + xmlWriter.Formatting = Formatting.Indented; +#endif + + try { + WriteXmlOutput (result, xmlWriter); + } finally { + writer.Close (); + } + } + + private void WriteXmlOutput (ITestResult result, XmlWriter xmlWriter) + { + this.xmlWriter = xmlWriter; + + InitializeXmlFile (result); + WriteResultElement (result); + TerminateXmlFile (); + } + + private void InitializeXmlFile (ITestResult result) + { + ResultSummary summaryResults = new ResultSummary (result); + + xmlWriter.WriteStartDocument (false); + xmlWriter.WriteComment ("This file represents the results of running a test suite"); + + xmlWriter.WriteStartElement ("test-results"); + + xmlWriter.WriteAttributeString ("name", result.FullName); + xmlWriter.WriteAttributeString ("total", summaryResults.TestCount.ToString ()); + xmlWriter.WriteAttributeString ("errors", summaryResults.ErrorCount.ToString ()); + xmlWriter.WriteAttributeString ("failures", summaryResults.FailureCount.ToString ()); + xmlWriter.WriteAttributeString ("not-run", summaryResults.NotRunCount.ToString ()); + xmlWriter.WriteAttributeString ("inconclusive", summaryResults.InconclusiveCount.ToString ()); + xmlWriter.WriteAttributeString ("ignored", summaryResults.IgnoreCount.ToString ()); + xmlWriter.WriteAttributeString ("skipped", summaryResults.SkipCount.ToString ()); + xmlWriter.WriteAttributeString ("invalid", summaryResults.InvalidCount.ToString ()); + + xmlWriter.WriteAttributeString ("date", XmlConvert.ToString (startTime, "yyyy-MM-dd")); + xmlWriter.WriteAttributeString ("time", XmlConvert.ToString (startTime, "HH:mm:ss")); + WriteEnvironment (); + WriteCultureInfo (); + } + + private void WriteCultureInfo () + { + xmlWriter.WriteStartElement ("culture-info"); + xmlWriter.WriteAttributeString ("current-culture", + CultureInfo.CurrentCulture.ToString ()); + xmlWriter.WriteAttributeString ("current-uiculture", + CultureInfo.CurrentUICulture.ToString ()); + xmlWriter.WriteEndElement (); + } + + private void WriteEnvironment () + { + xmlWriter.WriteStartElement ("environment"); + AssemblyName assemblyName = AssemblyHelper.GetAssemblyName (Assembly.GetExecutingAssembly ()); + xmlWriter.WriteAttributeString ("nunit-version", + assemblyName.Version.ToString ()); + xmlWriter.WriteAttributeString ("clr-version", + Environment.Version.ToString ()); + xmlWriter.WriteAttributeString ("os-version", + Environment.OSVersion.ToString ()); + xmlWriter.WriteAttributeString ("platform", + Environment.OSVersion.Platform.ToString ()); +#if !NETCF + xmlWriter.WriteAttributeString ("cwd", + Environment.CurrentDirectory); +#if !SILVERLIGHT + xmlWriter.WriteAttributeString ("machine-name", + Environment.MachineName); + xmlWriter.WriteAttributeString ("user", + Environment.UserName); + xmlWriter.WriteAttributeString ("user-domain", + Environment.UserDomainName); +#endif +#endif + xmlWriter.WriteEndElement (); + } + + private void WriteResultElement (ITestResult result) + { + StartTestElement (result); + + WriteProperties (result); + + switch (result.ResultState.Status) { + case TestStatus.Skipped: + WriteReasonElement (result.Message); + break; + case TestStatus.Failed: + WriteFailureElement (result.Message, result.StackTrace); + break; + } + + if (result.Test is TestSuite) + WriteChildResults (result); + + xmlWriter.WriteEndElement (); // test element + } + + private void TerminateXmlFile () + { + xmlWriter.WriteEndElement (); // test-results + xmlWriter.WriteEndDocument (); + xmlWriter.Flush (); + xmlWriter.Close (); + } + + + #region Element Creation Helpers + + private void StartTestElement (ITestResult result) + { + ITest test = result.Test; + TestSuite suite = test as TestSuite; + + if (suite != null) { + xmlWriter.WriteStartElement ("test-suite"); + xmlWriter.WriteAttributeString ("type", suite.TestType); + xmlWriter.WriteAttributeString ("name", suite.TestType == "Assembly" + ? result.Test.FullName + : result.Test.Name); + } else { + xmlWriter.WriteStartElement ("test-case"); + xmlWriter.WriteAttributeString ("name", result.Name); + } + + if (test.Properties.ContainsKey (PropertyNames.Description)) { + string description = (string)test.Properties.Get (PropertyNames.Description); + xmlWriter.WriteAttributeString ("description", description); + } + + TestStatus status = result.ResultState.Status; + string translatedResult = resultStates [result.ResultState.ToString ()]; + + if (status != TestStatus.Skipped) { + xmlWriter.WriteAttributeString ("executed", "True"); + xmlWriter.WriteAttributeString ("result", translatedResult); + xmlWriter.WriteAttributeString ("success", status == TestStatus.Passed ? "True" : "False"); + xmlWriter.WriteAttributeString ("time", result.Duration.TotalSeconds.ToString ()); + xmlWriter.WriteAttributeString ("asserts", result.AssertCount.ToString ()); + } else { + xmlWriter.WriteAttributeString ("executed", "False"); + xmlWriter.WriteAttributeString ("result", translatedResult); + } + } + + private void WriteProperties (ITestResult result) + { + IPropertyBag properties = result.Test.Properties; + int nprops = 0; + + foreach (string key in properties.Keys) { + if (key != PropertyNames.Category) { + if (nprops++ == 0) + xmlWriter.WriteStartElement ("properties"); + + foreach (object prop in properties [key]) { + xmlWriter.WriteStartElement ("property"); + xmlWriter.WriteAttributeString ("name", key); + xmlWriter.WriteAttributeString ("value", prop.ToString ()); + xmlWriter.WriteEndElement (); + } + } + } + + if (nprops > 0) + xmlWriter.WriteEndElement (); + } + + private void WriteReasonElement (string message) + { + xmlWriter.WriteStartElement ("reason"); + xmlWriter.WriteStartElement ("message"); + xmlWriter.WriteCData (message); + xmlWriter.WriteEndElement (); + xmlWriter.WriteEndElement (); + } + + private void WriteFailureElement (string message, string stackTrace) + { + xmlWriter.WriteStartElement ("failure"); + xmlWriter.WriteStartElement ("message"); + WriteCData (message); + xmlWriter.WriteEndElement (); + xmlWriter.WriteStartElement ("stack-trace"); + if (stackTrace != null) + WriteCData (stackTrace); + xmlWriter.WriteEndElement (); + xmlWriter.WriteEndElement (); + } + + private void WriteChildResults (ITestResult result) + { + xmlWriter.WriteStartElement ("results"); + + foreach (ITestResult childResult in result.Children) + WriteResultElement (childResult); + + xmlWriter.WriteEndElement (); + } + + #endregion + + #region Output Helpers + ///// + ///// Makes string safe for xml parsing, replacing control chars with '?' + ///// + ///// string to make safe + ///// xml safe string + //private static string CharacterSafeString(string encodedString) + //{ + // /*The default code page for the system will be used. + // Since all code pages use the same lower 128 bytes, this should be sufficient + // for finding uprintable control characters that make the xslt processor error. + // We use characters encoded by the default code page to avoid mistaking bytes as + // individual characters on non-latin code pages.*/ + // char[] encodedChars = System.Text.Encoding.Default.GetChars(System.Text.Encoding.Default.GetBytes(encodedString)); + + // System.Collections.ArrayList pos = new System.Collections.ArrayList(); + // for (int x = 0; x < encodedChars.Length; x++) + // { + // char currentChar = encodedChars[x]; + // //unprintable characters are below 0x20 in Unicode tables + // //some control characters are acceptable. (carriage return 0x0D, line feed 0x0A, horizontal tab 0x09) + // if (currentChar < 32 && (currentChar != 9 && currentChar != 10 && currentChar != 13)) + // { + // //save the array index for later replacement. + // pos.Add(x); + // } + // } + // foreach (int index in pos) + // { + // encodedChars[index] = '?';//replace unprintable control characters with ?(3F) + // } + // return System.Text.Encoding.Default.GetString(System.Text.Encoding.Default.GetBytes(encodedChars)); + //} + + private void WriteCData (string text) + { + int start = 0; + while (true) { + int illegal = text.IndexOf ("]]>", start); + if (illegal < 0) + break; + xmlWriter.WriteCData (text.Substring (start, illegal - start + 2)); + start = illegal + 2; + if (start >= text.Length) + return; + } + + if (start > 0) + xmlWriter.WriteCData (text.Substring (start)); + else + xmlWriter.WriteCData (text); + } + + #endregion + } +} \ No newline at end of file From da943bc929eab433c6b7454773b9cb49055230e7 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Sun, 2 Feb 2020 18:52:28 -0500 Subject: [PATCH 19/24] Add missing file to template csproj. --- tests/bcl-test/BCLTests-mac.csproj.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/bcl-test/BCLTests-mac.csproj.in b/tests/bcl-test/BCLTests-mac.csproj.in index 19541ac495c4..1cf56c47ff03 100644 --- a/tests/bcl-test/BCLTests-mac.csproj.in +++ b/tests/bcl-test/BCLTests-mac.csproj.in @@ -99,6 +99,9 @@ TestRunner.NUnit\ClassOrNamespaceFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.NUnit\TestMethodFilter.cs From 52dd4afe21d5547a3a610a40e09f91693352d2b3 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 3 Feb 2020 11:26:19 -0500 Subject: [PATCH 20/24] Fix badly formed xunit xml. --- tests/bcl-test/BCLTests-mac.csproj.in | 8 +- tests/bcl-test/BCLTests-tv.csproj.in | 19 +- .../BCLTests-watchos-extension.csproj.in | 16 +- tests/bcl-test/BCLTests.csproj.in | 16 +- .../TestRunner.NUnit/XmlOutputWriter.cs | 590 +++++++++--------- .../TestRunner.xUnit/XUnitTestRunner.cs | 7 +- tests/xharness/AppRunner.cs | 4 + 7 files changed, 335 insertions(+), 325 deletions(-) diff --git a/tests/bcl-test/BCLTests-mac.csproj.in b/tests/bcl-test/BCLTests-mac.csproj.in index 1cf56c47ff03..70a59e620690 100644 --- a/tests/bcl-test/BCLTests-mac.csproj.in +++ b/tests/bcl-test/BCLTests-mac.csproj.in @@ -48,14 +48,16 @@ - - - + + + + + diff --git a/tests/bcl-test/BCLTests-tv.csproj.in b/tests/bcl-test/BCLTests-tv.csproj.in index a28d298f7e58..0de8f571a10f 100644 --- a/tests/bcl-test/BCLTests-tv.csproj.in +++ b/tests/bcl-test/BCLTests-tv.csproj.in @@ -122,15 +122,15 @@ cjk,mideast,other,rare,west - - - - - + + + + + - - - + + + @@ -176,9 +176,6 @@ TestRunner.NUnit\TestMethodFilter.cs - - TestRunner.NUnit\XmlOutputWriter.cs - TestRunner.Core\Extensions.Bool.cs diff --git a/tests/bcl-test/BCLTests-watchos-extension.csproj.in b/tests/bcl-test/BCLTests-watchos-extension.csproj.in index d8334ec83454..f5e04663a841 100644 --- a/tests/bcl-test/BCLTests-watchos-extension.csproj.in +++ b/tests/bcl-test/BCLTests-watchos-extension.csproj.in @@ -128,15 +128,15 @@ SdkOnly - - - - - + + + + + - - - + + + diff --git a/tests/bcl-test/BCLTests.csproj.in b/tests/bcl-test/BCLTests.csproj.in index cb5242ecfa47..8d66f9a838f5 100644 --- a/tests/bcl-test/BCLTests.csproj.in +++ b/tests/bcl-test/BCLTests.csproj.in @@ -135,15 +135,15 @@ cjk,mideast,other,rare,west - - - - - + + + + + - - - + + + diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs index 93d925d38d50..4476fd90d625 100644 --- a/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/XmlOutputWriter.cs @@ -36,310 +36,314 @@ #endif namespace Xamarin.iOS.UnitTests.NUnit { - /// - /// NUnit2XmlOutputWriter is able to create an xml file representing - /// the result of a test run in NUnit 2.x format. - /// - public class NUnit2XmlOutputWriter : OutputWriter { - private XmlWriter xmlWriter; - private DateTime startTime; + /// + /// NUnit2XmlOutputWriter is able to create an xml file representing + /// the result of a test run in NUnit 2.x format. + /// + public class NUnit2XmlOutputWriter : OutputWriter { + private XmlWriter xmlWriter; + private DateTime startTime; #if CLR_2_0 || CLR_4_0 - private static Dictionary resultStates = new Dictionary(); + private static Dictionary resultStates = new Dictionary(); #else - private static StringDictionary resultStates = new StringDictionary (); + private static StringDictionary resultStates = new StringDictionary (); #endif - static NUnit2XmlOutputWriter () - { - resultStates ["Passed"] = "Success"; - resultStates ["Failed"] = "Failure"; - resultStates ["Failed:Error"] = "Error"; - resultStates ["Failed:Cancelled"] = "Cancelled"; - resultStates ["Inconclusive"] = "Inconclusive"; - resultStates ["Skipped"] = "Skipped"; - resultStates ["Skipped:Ignored"] = "Ignored"; - resultStates ["Skipped:Invalid"] = "NotRunnable"; - } - - public NUnit2XmlOutputWriter (DateTime startTime) - { - this.startTime = startTime; - } - - /// - /// Writes the result of a test run to a specified TextWriter. - /// - /// The test result for the run - /// The TextWriter to which the xml will be written - public override void WriteResultFile (ITestResult result, TextWriter writer) - { - // NOTE: Under .NET 1.1, XmlTextWriter does not implement IDisposable, - // but does implement Close(). Hence we cannot use a 'using' clause. - //using (XmlTextWriter xmlWriter = new XmlTextWriter(writer)) + static NUnit2XmlOutputWriter () + { + resultStates ["Passed"] = "Success"; + resultStates ["Failed"] = "Failure"; + resultStates ["Failed:Error"] = "Error"; + resultStates ["Failed:Cancelled"] = "Cancelled"; + resultStates ["Inconclusive"] = "Inconclusive"; + resultStates ["Skipped"] = "Skipped"; + resultStates ["Skipped:Ignored"] = "Ignored"; + resultStates ["Skipped:Invalid"] = "NotRunnable"; + } + + public NUnit2XmlOutputWriter (DateTime startTime) + { + this.startTime = startTime; + } + + /// + /// Writes the result of a test run to a specified TextWriter. + /// + /// The test result for the run + /// The TextWriter to which the xml will be written + public override void WriteResultFile (ITestResult result, TextWriter writer) + { + // NOTE: Under .NET 1.1, XmlTextWriter does not implement IDisposable, + // but does implement Close(). Hence we cannot use a 'using' clause. + //using (XmlTextWriter xmlWriter = new XmlTextWriter(writer)) #if SILVERLIGHT - XmlWriter xmlWriter = XmlWriter.Create(writer); + XmlWriter xmlWriter = XmlWriter.Create(writer); #else - XmlTextWriter xmlWriter = new XmlTextWriter (writer); - xmlWriter.Formatting = Formatting.Indented; + XmlTextWriter xmlWriter = new XmlTextWriter (writer); + xmlWriter.Formatting = Formatting.Indented; #endif - try { - WriteXmlOutput (result, xmlWriter); - } finally { - writer.Close (); - } - } - - private void WriteXmlOutput (ITestResult result, XmlWriter xmlWriter) - { - this.xmlWriter = xmlWriter; - - InitializeXmlFile (result); - WriteResultElement (result); - TerminateXmlFile (); - } - - private void InitializeXmlFile (ITestResult result) - { - ResultSummary summaryResults = new ResultSummary (result); - - xmlWriter.WriteStartDocument (false); - xmlWriter.WriteComment ("This file represents the results of running a test suite"); - - xmlWriter.WriteStartElement ("test-results"); - - xmlWriter.WriteAttributeString ("name", result.FullName); - xmlWriter.WriteAttributeString ("total", summaryResults.TestCount.ToString ()); - xmlWriter.WriteAttributeString ("errors", summaryResults.ErrorCount.ToString ()); - xmlWriter.WriteAttributeString ("failures", summaryResults.FailureCount.ToString ()); - xmlWriter.WriteAttributeString ("not-run", summaryResults.NotRunCount.ToString ()); - xmlWriter.WriteAttributeString ("inconclusive", summaryResults.InconclusiveCount.ToString ()); - xmlWriter.WriteAttributeString ("ignored", summaryResults.IgnoreCount.ToString ()); - xmlWriter.WriteAttributeString ("skipped", summaryResults.SkipCount.ToString ()); - xmlWriter.WriteAttributeString ("invalid", summaryResults.InvalidCount.ToString ()); - - xmlWriter.WriteAttributeString ("date", XmlConvert.ToString (startTime, "yyyy-MM-dd")); - xmlWriter.WriteAttributeString ("time", XmlConvert.ToString (startTime, "HH:mm:ss")); - WriteEnvironment (); - WriteCultureInfo (); - } - - private void WriteCultureInfo () - { - xmlWriter.WriteStartElement ("culture-info"); - xmlWriter.WriteAttributeString ("current-culture", - CultureInfo.CurrentCulture.ToString ()); - xmlWriter.WriteAttributeString ("current-uiculture", - CultureInfo.CurrentUICulture.ToString ()); - xmlWriter.WriteEndElement (); - } - - private void WriteEnvironment () - { - xmlWriter.WriteStartElement ("environment"); - AssemblyName assemblyName = AssemblyHelper.GetAssemblyName (Assembly.GetExecutingAssembly ()); - xmlWriter.WriteAttributeString ("nunit-version", - assemblyName.Version.ToString ()); - xmlWriter.WriteAttributeString ("clr-version", - Environment.Version.ToString ()); - xmlWriter.WriteAttributeString ("os-version", - Environment.OSVersion.ToString ()); - xmlWriter.WriteAttributeString ("platform", - Environment.OSVersion.Platform.ToString ()); + try { + WriteXmlOutput (result, xmlWriter); + } finally { + writer.Close (); + } + } + + private void WriteXmlOutput (ITestResult result, XmlWriter xmlWriter) + { + this.xmlWriter = xmlWriter; + + InitializeXmlFile (result); + WriteResultElement (result); + TerminateXmlFile (); + } + + private void InitializeXmlFile (ITestResult result) + { + ResultSummary summaryResults = new ResultSummary (result); + + xmlWriter.WriteStartDocument (false); + xmlWriter.WriteComment ("This file represents the results of running a test suite"); + + xmlWriter.WriteStartElement ("test-results"); + + xmlWriter.WriteAttributeString ("name", result.FullName); + xmlWriter.WriteAttributeString ("total", summaryResults.TestCount.ToString ()); + xmlWriter.WriteAttributeString ("errors", summaryResults.ErrorCount.ToString ()); + xmlWriter.WriteAttributeString ("failures", summaryResults.FailureCount.ToString ()); + xmlWriter.WriteAttributeString ("not-run", summaryResults.NotRunCount.ToString ()); + xmlWriter.WriteAttributeString ("inconclusive", summaryResults.InconclusiveCount.ToString ()); + xmlWriter.WriteAttributeString ("ignored", summaryResults.IgnoreCount.ToString ()); + xmlWriter.WriteAttributeString ("skipped", summaryResults.SkipCount.ToString ()); + xmlWriter.WriteAttributeString ("invalid", summaryResults.InvalidCount.ToString ()); + + xmlWriter.WriteAttributeString ("date", XmlConvert.ToString (startTime, "yyyy-MM-dd")); + xmlWriter.WriteAttributeString ("time", XmlConvert.ToString (startTime, "HH:mm:ss")); + WriteEnvironment (); + WriteCultureInfo (); + xmlWriter.Flush (); + } + + private void WriteCultureInfo () + { + xmlWriter.WriteStartElement ("culture-info"); + xmlWriter.WriteAttributeString ("current-culture", + CultureInfo.CurrentCulture.ToString ()); + xmlWriter.WriteAttributeString ("current-uiculture", + CultureInfo.CurrentUICulture.ToString ()); + xmlWriter.WriteEndElement (); + xmlWriter.Flush (); + } + + private void WriteEnvironment () + { + xmlWriter.WriteStartElement ("environment"); + AssemblyName assemblyName = AssemblyHelper.GetAssemblyName (Assembly.GetExecutingAssembly ()); + xmlWriter.WriteAttributeString ("nunit-version", + assemblyName.Version.ToString ()); + xmlWriter.WriteAttributeString ("clr-version", + Environment.Version.ToString ()); + xmlWriter.WriteAttributeString ("os-version", + Environment.OSVersion.ToString ()); + xmlWriter.WriteAttributeString ("platform", + Environment.OSVersion.Platform.ToString ()); #if !NETCF - xmlWriter.WriteAttributeString ("cwd", - Environment.CurrentDirectory); + xmlWriter.WriteAttributeString ("cwd", + Environment.CurrentDirectory); #if !SILVERLIGHT - xmlWriter.WriteAttributeString ("machine-name", - Environment.MachineName); - xmlWriter.WriteAttributeString ("user", - Environment.UserName); - xmlWriter.WriteAttributeString ("user-domain", - Environment.UserDomainName); + xmlWriter.WriteAttributeString ("machine-name", + Environment.MachineName); + xmlWriter.WriteAttributeString ("user", + Environment.UserName); + xmlWriter.WriteAttributeString ("user-domain", + Environment.UserDomainName); #endif #endif - xmlWriter.WriteEndElement (); - } - - private void WriteResultElement (ITestResult result) - { - StartTestElement (result); - - WriteProperties (result); - - switch (result.ResultState.Status) { - case TestStatus.Skipped: - WriteReasonElement (result.Message); - break; - case TestStatus.Failed: - WriteFailureElement (result.Message, result.StackTrace); - break; - } - - if (result.Test is TestSuite) - WriteChildResults (result); - - xmlWriter.WriteEndElement (); // test element - } - - private void TerminateXmlFile () - { - xmlWriter.WriteEndElement (); // test-results - xmlWriter.WriteEndDocument (); - xmlWriter.Flush (); - xmlWriter.Close (); - } - - - #region Element Creation Helpers - - private void StartTestElement (ITestResult result) - { - ITest test = result.Test; - TestSuite suite = test as TestSuite; - - if (suite != null) { - xmlWriter.WriteStartElement ("test-suite"); - xmlWriter.WriteAttributeString ("type", suite.TestType); - xmlWriter.WriteAttributeString ("name", suite.TestType == "Assembly" - ? result.Test.FullName - : result.Test.Name); - } else { - xmlWriter.WriteStartElement ("test-case"); - xmlWriter.WriteAttributeString ("name", result.Name); - } - - if (test.Properties.ContainsKey (PropertyNames.Description)) { - string description = (string)test.Properties.Get (PropertyNames.Description); - xmlWriter.WriteAttributeString ("description", description); - } - - TestStatus status = result.ResultState.Status; - string translatedResult = resultStates [result.ResultState.ToString ()]; - - if (status != TestStatus.Skipped) { - xmlWriter.WriteAttributeString ("executed", "True"); - xmlWriter.WriteAttributeString ("result", translatedResult); - xmlWriter.WriteAttributeString ("success", status == TestStatus.Passed ? "True" : "False"); - xmlWriter.WriteAttributeString ("time", result.Duration.TotalSeconds.ToString ()); - xmlWriter.WriteAttributeString ("asserts", result.AssertCount.ToString ()); - } else { - xmlWriter.WriteAttributeString ("executed", "False"); - xmlWriter.WriteAttributeString ("result", translatedResult); - } - } - - private void WriteProperties (ITestResult result) - { - IPropertyBag properties = result.Test.Properties; - int nprops = 0; - - foreach (string key in properties.Keys) { - if (key != PropertyNames.Category) { - if (nprops++ == 0) - xmlWriter.WriteStartElement ("properties"); - - foreach (object prop in properties [key]) { - xmlWriter.WriteStartElement ("property"); - xmlWriter.WriteAttributeString ("name", key); - xmlWriter.WriteAttributeString ("value", prop.ToString ()); - xmlWriter.WriteEndElement (); - } - } - } - - if (nprops > 0) - xmlWriter.WriteEndElement (); - } - - private void WriteReasonElement (string message) - { - xmlWriter.WriteStartElement ("reason"); - xmlWriter.WriteStartElement ("message"); - xmlWriter.WriteCData (message); - xmlWriter.WriteEndElement (); - xmlWriter.WriteEndElement (); - } - - private void WriteFailureElement (string message, string stackTrace) - { - xmlWriter.WriteStartElement ("failure"); - xmlWriter.WriteStartElement ("message"); - WriteCData (message); - xmlWriter.WriteEndElement (); - xmlWriter.WriteStartElement ("stack-trace"); - if (stackTrace != null) - WriteCData (stackTrace); - xmlWriter.WriteEndElement (); - xmlWriter.WriteEndElement (); - } - - private void WriteChildResults (ITestResult result) - { - xmlWriter.WriteStartElement ("results"); - - foreach (ITestResult childResult in result.Children) - WriteResultElement (childResult); - - xmlWriter.WriteEndElement (); - } - - #endregion - - #region Output Helpers - ///// - ///// Makes string safe for xml parsing, replacing control chars with '?' - ///// - ///// string to make safe - ///// xml safe string - //private static string CharacterSafeString(string encodedString) - //{ - // /*The default code page for the system will be used. - // Since all code pages use the same lower 128 bytes, this should be sufficient - // for finding uprintable control characters that make the xslt processor error. - // We use characters encoded by the default code page to avoid mistaking bytes as - // individual characters on non-latin code pages.*/ - // char[] encodedChars = System.Text.Encoding.Default.GetChars(System.Text.Encoding.Default.GetBytes(encodedString)); - - // System.Collections.ArrayList pos = new System.Collections.ArrayList(); - // for (int x = 0; x < encodedChars.Length; x++) - // { - // char currentChar = encodedChars[x]; - // //unprintable characters are below 0x20 in Unicode tables - // //some control characters are acceptable. (carriage return 0x0D, line feed 0x0A, horizontal tab 0x09) - // if (currentChar < 32 && (currentChar != 9 && currentChar != 10 && currentChar != 13)) - // { - // //save the array index for later replacement. - // pos.Add(x); - // } - // } - // foreach (int index in pos) - // { - // encodedChars[index] = '?';//replace unprintable control characters with ?(3F) - // } - // return System.Text.Encoding.Default.GetString(System.Text.Encoding.Default.GetBytes(encodedChars)); - //} - - private void WriteCData (string text) - { - int start = 0; - while (true) { - int illegal = text.IndexOf ("]]>", start); - if (illegal < 0) - break; - xmlWriter.WriteCData (text.Substring (start, illegal - start + 2)); - start = illegal + 2; - if (start >= text.Length) - return; - } - - if (start > 0) - xmlWriter.WriteCData (text.Substring (start)); - else - xmlWriter.WriteCData (text); - } - - #endregion - } + xmlWriter.WriteEndElement (); + xmlWriter.Flush (); + } + + private void WriteResultElement (ITestResult result) + { + StartTestElement (result); + + WriteProperties (result); + + switch (result.ResultState.Status) { + case TestStatus.Skipped: + WriteReasonElement (result.Message); + break; + case TestStatus.Failed: + WriteFailureElement (result.Message, result.StackTrace); + break; + } + + if (result.Test is TestSuite) + WriteChildResults (result); + + xmlWriter.WriteEndElement (); // test element + xmlWriter.Flush (); + } + + private void TerminateXmlFile () + { + xmlWriter.WriteEndElement (); // test-results + xmlWriter.WriteEndDocument (); + xmlWriter.Flush (); + xmlWriter.Close (); + } + + + #region Element Creation Helpers + + private void StartTestElement (ITestResult result) + { + ITest test = result.Test; + TestSuite suite = test as TestSuite; + + if (suite != null) { + xmlWriter.WriteStartElement ("test-suite"); + xmlWriter.WriteAttributeString ("type", suite.TestType); + xmlWriter.WriteAttributeString ("name", suite.TestType == "Assembly" + ? result.Test.FullName + : result.Test.Name); + } else { + xmlWriter.WriteStartElement ("test-case"); + xmlWriter.WriteAttributeString ("name", result.Name); + } + + if (test.Properties.ContainsKey (PropertyNames.Description)) { + string description = (string)test.Properties.Get (PropertyNames.Description); + xmlWriter.WriteAttributeString ("description", description); + } + + TestStatus status = result.ResultState.Status; + string translatedResult = resultStates [result.ResultState.ToString ()]; + + if (status != TestStatus.Skipped) { + xmlWriter.WriteAttributeString ("executed", "True"); + xmlWriter.WriteAttributeString ("result", translatedResult); + xmlWriter.WriteAttributeString ("success", status == TestStatus.Passed ? "True" : "False"); + xmlWriter.WriteAttributeString ("time", result.Duration.TotalSeconds.ToString ()); + xmlWriter.WriteAttributeString ("asserts", result.AssertCount.ToString ()); + } else { + xmlWriter.WriteAttributeString ("executed", "False"); + xmlWriter.WriteAttributeString ("result", translatedResult); + } + } + + private void WriteProperties (ITestResult result) + { + IPropertyBag properties = result.Test.Properties; + int nprops = 0; + + foreach (string key in properties.Keys) { + if (key != PropertyNames.Category) { + if (nprops++ == 0) + xmlWriter.WriteStartElement ("properties"); + + foreach (object prop in properties [key]) { + xmlWriter.WriteStartElement ("property"); + xmlWriter.WriteAttributeString ("name", key); + xmlWriter.WriteAttributeString ("value", prop.ToString ()); + xmlWriter.WriteEndElement (); + } + } + } + + if (nprops > 0) + xmlWriter.WriteEndElement (); + } + + private void WriteReasonElement (string message) + { + xmlWriter.WriteStartElement ("reason"); + xmlWriter.WriteStartElement ("message"); + xmlWriter.WriteCData (message); + xmlWriter.WriteEndElement (); + xmlWriter.WriteEndElement (); + } + + private void WriteFailureElement (string message, string stackTrace) + { + xmlWriter.WriteStartElement ("failure"); + xmlWriter.WriteStartElement ("message"); + WriteCData (message); + xmlWriter.WriteEndElement (); + xmlWriter.WriteStartElement ("stack-trace"); + if (stackTrace != null) + WriteCData (stackTrace); + xmlWriter.WriteEndElement (); + xmlWriter.WriteEndElement (); + } + + private void WriteChildResults (ITestResult result) + { + xmlWriter.WriteStartElement ("results"); + + foreach (ITestResult childResult in result.Children) + WriteResultElement (childResult); + + xmlWriter.WriteEndElement (); + } + + #endregion + + #region Output Helpers + ///// + ///// Makes string safe for xml parsing, replacing control chars with '?' + ///// + ///// string to make safe + ///// xml safe string + //private static string CharacterSafeString(string encodedString) + //{ + // /*The default code page for the system will be used. + // Since all code pages use the same lower 128 bytes, this should be sufficient + // for finding uprintable control characters that make the xslt processor error. + // We use characters encoded by the default code page to avoid mistaking bytes as + // individual characters on non-latin code pages.*/ + // char[] encodedChars = System.Text.Encoding.Default.GetChars(System.Text.Encoding.Default.GetBytes(encodedString)); + + // System.Collections.ArrayList pos = new System.Collections.ArrayList(); + // for (int x = 0; x < encodedChars.Length; x++) + // { + // char currentChar = encodedChars[x]; + // //unprintable characters are below 0x20 in Unicode tables + // //some control characters are acceptable. (carriage return 0x0D, line feed 0x0A, horizontal tab 0x09) + // if (currentChar < 32 && (currentChar != 9 && currentChar != 10 && currentChar != 13)) + // { + // //save the array index for later replacement. + // pos.Add(x); + // } + // } + // foreach (int index in pos) + // { + // encodedChars[index] = '?';//replace unprintable control characters with ?(3F) + // } + // return System.Text.Encoding.Default.GetString(System.Text.Encoding.Default.GetBytes(encodedChars)); + //} + + private void WriteCData (string text) + { + int start = 0; + while (true) { + int illegal = text.IndexOf ("]]>", start); + if (illegal < 0) + break; + xmlWriter.WriteCData (text.Substring (start, illegal - start + 2)); + start = illegal + 2; + if (start >= text.Length) + return; + } + + if (start > 0) + xmlWriter.WriteCData (text.Substring (start)); + else + xmlWriter.WriteCData (text); + } + + #endregion + } } \ No newline at end of file diff --git a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs index 3c06faaf2b11..1363974c2591 100644 --- a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs @@ -795,9 +795,10 @@ public override string WriteResultsToFile () { if (assembliesElement == null) return String.Empty; - + // remove all the empty nodes + assembliesElement.Descendants ().Where (e => e.Name == "collection" && !e.Descendants ().Any ()).Remove (); string outputFilePath = GetResultsFilePath (); - var settings = new XmlWriterSettings { Indent = true }; + var settings = new XmlWriterSettings { Indent = true}; using (var xmlWriter = XmlWriter.Create (outputFilePath, settings)) { switch (ResultFileFormat) { case XUnitResultFileFormat.XunitV2: @@ -818,6 +819,8 @@ public override void WriteResultsToFile (TextWriter writer) { if (assembliesElement == null) return; + // remove all the empty nodes + assembliesElement.Descendants ().Where (e => e.Name == "collection" && !e.Descendants ().Any ()).Remove (); writer.WriteLine (""); var settings = new XmlWriterSettings { Indent = true }; using (var xmlWriter = XmlWriter.Create (writer, settings)) { diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index c129ef4a26fc..40589ee638d4 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -584,11 +584,14 @@ XmlResultType GetXmlType (StreamReader stream) // clean any junk we have using (var xmlWriter = new StreamWriter (path)) { string line; + string previous = null; while ((line = streamReaderTmp.ReadLine ()) != null) { if (line.Contains ("ping") || line.Contains ("") || line.Contains ("") || line.StartsWith ("")) // get out of the loop we are not interested in the rest of the TouchUnit data. break; + if (previous != null && previous.Contains ("") && line.Contains (" ")) // something funny happens with the xunit results and collections that is priting twice + continue; if (line.Contains (" Date: Mon, 3 Feb 2020 19:03:11 -0500 Subject: [PATCH 21/24] Several parsing fixes: * Do not clean the xml, seems not to be needed and gives issues with threading. * Set the size of the client buffer in the tcp listener. --- .../TestRunner.NUnit/NUnitTestRunner.cs | 2 - .../TestRunner.xUnit/XUnitTestRunner.cs | 2 - tests/xharness/AppRunner.cs | 78 +++++++------------ tests/xharness/SimpleTcpListener.cs | 1 + 4 files changed, 27 insertions(+), 56 deletions(-) diff --git a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs index 2550f22d0cd3..9ce39af84547 100644 --- a/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.NUnit/NUnitTestRunner.cs @@ -272,9 +272,7 @@ public override void WriteResultsToFile (TextWriter writer) if (results == null) return; var resultsXml = new NUnit2XmlOutputWriter (DateTime.UtcNow); - writer.WriteLine (""); resultsXml.WriteResultFile (results, writer); - writer.WriteLine (""); } void AppendFilter (ITestFilter filter) diff --git a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs index 1363974c2591..ffcd4da80e93 100644 --- a/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs +++ b/tests/bcl-test/templates/common/TestRunner.xUnit/XUnitTestRunner.cs @@ -821,7 +821,6 @@ public override void WriteResultsToFile (TextWriter writer) return; // remove all the empty nodes assembliesElement.Descendants ().Where (e => e.Name == "collection" && !e.Descendants ().Any ()).Remove (); - writer.WriteLine (""); var settings = new XmlWriterSettings { Indent = true }; using (var xmlWriter = XmlWriter.Create (writer, settings)) { switch (ResultFileFormat) { @@ -840,7 +839,6 @@ public override void WriteResultsToFile (TextWriter writer) throw new InvalidOperationException ($"Result output format '{ResultFileFormat}' is not currently supported"); } } - writer.WriteLine (""); } void Transform_Results (string xsltResourceName, XElement element, XmlWriter writer) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 40589ee638d4..20bfa86ad170 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -537,10 +537,10 @@ XmlResultType GetXmlType (StreamReader stream) return (resultLine, total == 0 | errors != 0 || failed != 0); } - - (string resultLine, bool failed, bool crashed) ParseResult (Log listener_log, bool timed_out, bool crashed) + + (string resultLine, bool failed, bool crashed) ParseResult (string test_log_path, bool timed_out, bool crashed) { - if (!File.Exists (listener_log.FullPath)) + if (!File.Exists (test_log_path)) return (null, false, true); // if we do not have a log file, the test crashes // parsing the result is different if we are in jenkins or not. @@ -555,61 +555,35 @@ XmlResultType GetXmlType (StreamReader stream) // that case, we cannot do a TCP connection to xharness to get the log, this is a problem since if we did not get the xml // from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only // going to check if we are in CI, but also if the listener_log is valid. - if (Harness.InCI && IsXml (listener_log.FullPath)) { - (string resultLine, bool failed, bool crashed) parseResult = (null, false, false); - // move the xml to a tmp path, that path will be use to read the xml - // in the reader, and the writer will use the stream from the logger to - // write the human readable log - var tmpFile = Path.Combine (Path.GetTempPath (), Guid.NewGuid ().ToString ()); + var path = Path.ChangeExtension (test_log_path, "xml"); + File.Copy (test_log_path, path); - // copy do not move - File.Copy (listener_log.FullPath, tmpFile, true); + if (Harness.InCI && IsXml (test_log_path)) { + (string resultLine, bool failed, bool crashed) parseResult = (null, false, false); crashed = false; - string path = listener_log.FullPath; XmlResultType xmlType; try { - using (var streamReaderTmp = new StreamReader (tmpFile)) { + using (var streamReaderTmp = new StreamReader (path)) { xmlType = GetXmlType (streamReaderTmp); - path = Path.ChangeExtension (path, "xml"); var fileName = Path.GetFileName (path); + var newFilename = ""; switch (xmlType) { case XmlResultType.TouchUnit: case XmlResultType.NUnit: - path = path.Replace (fileName, $"nunit-{fileName}"); + newFilename = path.Replace (fileName, $"nunit-{fileName}"); break; case XmlResultType.xUnit: - path = path.Replace (fileName, $"xunit-{fileName}"); + newFilename = path.Replace (fileName, $"xunit-{fileName}"); break; } - // clean any junk we have - using (var xmlWriter = new StreamWriter (path)) { - string line; - string previous = null; - while ((line = streamReaderTmp.ReadLine ()) != null) { - if (line.Contains ("ping") || line.Contains ("") || line.Contains ("") || line.StartsWith ("")) // get out of the loop we are not interested in the rest of the TouchUnit data. - break; - if (previous != null && previous.Contains ("") && line.Contains (" ")) // something funny happens with the xunit results and collections that is priting twice - continue; - if (line.Contains (" RunAsync () var crashed = false; if (File.Exists (listener_log.FullPath)) { Harness.LogWrench ("@MonkeyWrench: AddFile: {0}", listener_log.FullPath); - success = TestsSucceeded (listener_log, timed_out, crashed); + success = TestsSucceeded (listener_log.FullPath, timed_out, crashed); } else if (timed_out) { Harness.LogWrench ("@MonkeyWrench: AddSummary: {0} never launched
", mode); main_log.WriteLine ("Test run never launched"); diff --git a/tests/xharness/SimpleTcpListener.cs b/tests/xharness/SimpleTcpListener.cs index 143be06402e6..a89fe52a3211 100644 --- a/tests/xharness/SimpleTcpListener.cs +++ b/tests/xharness/SimpleTcpListener.cs @@ -34,6 +34,7 @@ protected override void Start () do { Log.WriteLine ("Test log server listening on: {0}:{1}", Address, Port); using (TcpClient client = server.AcceptTcpClient ()) { + client.ReceiveBufferSize = buffer.Length; processed = Processing (client); } } while (!AutoExit || !processed); From de960c0c8e3e62eac707f0781547bb68a6dd6003 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 3 Feb 2020 22:33:13 -0500 Subject: [PATCH 22/24] Add forgotten file for tvOs tests. --- tests/bcl-test/BCLTests-tv.csproj.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/bcl-test/BCLTests-tv.csproj.in b/tests/bcl-test/BCLTests-tv.csproj.in index 0de8f571a10f..9ad5fb0a1bb0 100644 --- a/tests/bcl-test/BCLTests-tv.csproj.in +++ b/tests/bcl-test/BCLTests-tv.csproj.in @@ -176,6 +176,9 @@ TestRunner.NUnit\TestMethodFilter.cs + + TestRunner.NUnit\XmlOutputWriter.cs + TestRunner.Core\Extensions.Bool.cs From e65cc7a837e947ea13dd6dd02a54eedcf227dfac Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Mon, 3 Feb 2020 22:51:14 -0500 Subject: [PATCH 23/24] We need to not read the ping. --- tests/xharness/AppRunner.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 20bfa86ad170..7f9563bc119b 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -538,6 +538,20 @@ XmlResultType GetXmlType (StreamReader stream) return (resultLine, total == 0 | errors != 0 || failed != 0); } + void CleanXml (string source, string destination) + { + using (var reader = new StreamReader (source)) + using (var writer = new StreamWriter (destination)) { + string line; + while ((line = reader.ReadLine ()) != null) { + if (line.StartsWith ("ping", StringComparison.InvariantCulture)) { + continue; + } + writer.WriteLine (line); + } + } + } + (string resultLine, bool failed, bool crashed) ParseResult (string test_log_path, bool timed_out, bool crashed) { if (!File.Exists (test_log_path)) @@ -556,7 +570,7 @@ XmlResultType GetXmlType (StreamReader stream) // from the TCP connection, we are going to fail when trying to read it and not parse it. Therefore, we are not only // going to check if we are in CI, but also if the listener_log is valid. var path = Path.ChangeExtension (test_log_path, "xml"); - File.Copy (test_log_path, path); + CleanXml (test_log_path, path); if (Harness.InCI && IsXml (test_log_path)) { (string resultLine, bool failed, bool crashed) parseResult = (null, false, false); From 0b823ab48726a55c55421d53afd5ee1e2d279886 Mon Sep 17 00:00:00 2001 From: Manuel de la Pena Date: Tue, 4 Feb 2020 04:13:46 -0500 Subject: [PATCH 24/24] VSTS does not line the touch unit format and will throw the following exception and stop uploading results: ``` Failed to parse result files: System.NotSupportedException: Invalid file format. at Microsoft.TeamFoundation.TestClient.PublishTestResults.NUnitResultParser.ParseTestResultFile(TestRunContext runContext, String filePath) at Microsoft.TeamFoundation.TestClient.PublishTestResults.NUnitResultParser.<>c__DisplayClass1_0.b__0(String file) at System.Linq.Enumerable.SelectListIterator`2.MoveNext() at System.Linq.Enumerable.WhereEnumerableIterator`1.ToList() at System.Linq.Enumerable.ToList[TSource](IEnumerable`1 source) at Microsoft.TeamFoundation.TestClient.PublishTestResults.NUnitResultParser.ParseTestResultFiles(TestRunContext runContext, IList`1 resultFilePaths) at Microsoft.VisualStudio.Services.Agent.Worker.TestResults.Parser.ParseFiles(IExecutionContext executionContext, TestRunContext testRunContext, List`1 testResultsFiles, ITestResultParser testResultParser) ``` --- tests/xharness/AppRunner.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/xharness/AppRunner.cs b/tests/xharness/AppRunner.cs index 7f9563bc119b..733ba7f59106 100644 --- a/tests/xharness/AppRunner.cs +++ b/tests/xharness/AppRunner.cs @@ -544,9 +544,11 @@ void CleanXml (string source, string destination) using (var writer = new StreamWriter (destination)) { string line; while ((line = reader.ReadLine ()) != null) { - if (line.StartsWith ("ping", StringComparison.InvariantCulture)) { + if (line.StartsWith ("ping", StringComparison.InvariantCulture) || line.Contains ("TouchUnitTestRun") || line.Contains ("NUnitOutput") || line.Contains ("