diff --git a/src/Assets/TestProjects/VSTestDataCollectorSample/AttachmentProcessorDataCollector.csproj b/src/Assets/TestProjects/VSTestDataCollectorSample/AttachmentProcessorDataCollector.csproj new file mode 100644 index 000000000000..27aab5399acf --- /dev/null +++ b/src/Assets/TestProjects/VSTestDataCollectorSample/AttachmentProcessorDataCollector.csproj @@ -0,0 +1,16 @@ + + + + netstandard2.0 + + + + + + + + + + + + diff --git a/src/Assets/TestProjects/VSTestDataCollectorSample/SampleDataCollector.cs b/src/Assets/TestProjects/VSTestDataCollectorSample/SampleDataCollector.cs new file mode 100644 index 000000000000..a68628609586 --- /dev/null +++ b/src/Assets/TestProjects/VSTestDataCollectorSample/SampleDataCollector.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AttachmentProcessorDataCollector +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + + internal class ExtensionInfo + { + public const string ExtensionType = "DataCollector"; + public const string ExtensionIdentifier = "my://sample/datacollector"; + } + + [DataCollectorFriendlyName("SampleDataCollector")] + [DataCollectorTypeUri(ExtensionInfo.ExtensionIdentifier)] + [DataCollectorAttachmentProcessor(typeof(SampleDataCollectorAttachmentProcessor))] + public class SampleDataCollectorV2 : SampleDataCollectorV1 { } + + [DataCollectorFriendlyName("SampleDataCollector")] + [DataCollectorTypeUri(ExtensionInfo.ExtensionIdentifier)] + public class SampleDataCollectorV1 : DataCollector + { + private DataCollectionSink _dataCollectionSink; + private DataCollectionEnvironmentContext _context; + private readonly string _tempDirectoryPath = Path.GetTempPath(); + + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + events.SessionEnd += SessionEnded_Handler; + _dataCollectionSink = dataSink; + _context = environmentContext; + } + + private void SessionEnded_Handler(object sender, SessionEndEventArgs e) + { + string tmpAttachment = Path.Combine(_tempDirectoryPath, Guid.NewGuid().ToString("N"), "DataCollectorAttachmentProcessor_1.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(tmpAttachment)); + File.WriteAllText(tmpAttachment, $"SessionEnded_Handler_{Guid.NewGuid():N}"); + _dataCollectionSink.SendFileAsync(_context.SessionDataCollectionContext, tmpAttachment, true); + } + } + + public class SampleDataCollectorAttachmentProcessor : IDataCollectorAttachmentProcessor + { + public bool SupportsIncrementalProcessing => true; + + public IEnumerable GetExtensionUris() + => new List() { new Uri(ExtensionInfo.ExtensionIdentifier) }; + + public Task> ProcessAttachmentSetsAsync(XmlElement configurationElement, ICollection attachments, IProgress progressReporter, IMessageLogger logger, CancellationToken cancellationToken) + { + string finalFileName = configurationElement.FirstChild.InnerText; + StringBuilder stringBuilder = new StringBuilder(); + string finalFolder = null; + foreach (var attachmentSet in attachments) + { + foreach (var attachment in attachmentSet.Attachments.OrderBy(f => f.Uri.AbsolutePath)) + { + if (finalFolder is null) + { + finalFolder = Path.GetDirectoryName(attachment.Uri.AbsolutePath); + } + + stringBuilder.AppendLine(File.ReadAllText(attachment.Uri.AbsolutePath).Trim()); + } + } + + File.WriteAllText(Path.Combine(finalFolder, finalFileName), stringBuilder.ToString()); + + List mergedAttachment = new List(); + var mergedAttachmentSet = new AttachmentSet(new Uri("my://sample/datacollector"), "SampleDataCollector"); + mergedAttachmentSet.Attachments.Add(UriDataAttachment.CreateFrom(Path.Combine(finalFolder, finalFileName), string.Empty)); + mergedAttachment.Add(mergedAttachmentSet); + + return Task.FromResult((ICollection)new Collection(mergedAttachment)); + } + } +} diff --git a/src/Assets/TestProjects/VSTestDataCollectorSample/TestExtensionTypesAttribute.cs b/src/Assets/TestProjects/VSTestDataCollectorSample/TestExtensionTypesAttribute.cs new file mode 100644 index 000000000000..b092e3f79e6e --- /dev/null +++ b/src/Assets/TestProjects/VSTestDataCollectorSample/TestExtensionTypesAttribute.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AttachmentProcessorDataCollector; + +using Microsoft.VisualStudio.TestPlatform; + +[assembly: TestExtensionTypes(typeof(SampleDataCollectorV1))] +[assembly: TestExtensionTypesV2(ExtensionInfo.ExtensionType, ExtensionInfo.ExtensionIdentifier, typeof(SampleDataCollectorV1), 1, "futureUnused")] +[assembly: TestExtensionTypesV2(ExtensionInfo.ExtensionType, ExtensionInfo.ExtensionIdentifier, typeof(SampleDataCollectorV2), 2)] + +namespace Microsoft.VisualStudio.TestPlatform +{ + using System; + + [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = false)] + internal sealed class TestExtensionTypesAttribute : Attribute + { + public TestExtensionTypesAttribute(params Type[] types) + { + Types = types; + } + + public Type[] Types { get; } + } + + [AttributeUsage(AttributeTargets.Assembly, Inherited = false, AllowMultiple = true)] + internal sealed class TestExtensionTypesV2Attribute : Attribute + { + public string ExtensionType { get; } + public string ExtensionIdentifier { get; } + public Type ExtensionImplementation { get; } + public int Version { get; } + + public TestExtensionTypesV2Attribute(string extensionType, string extensionIdentifier, Type extensionImplementation, int version, string _ = null) + { + ExtensionType = extensionType; + ExtensionIdentifier = extensionIdentifier; + ExtensionImplementation = extensionImplementation; + Version = version; + } + } +} diff --git a/src/Assets/TestProjects/VSTestDataCollectorSampleNoMerge/AttachmentProcessorDataCollector.csproj b/src/Assets/TestProjects/VSTestDataCollectorSampleNoMerge/AttachmentProcessorDataCollector.csproj new file mode 100644 index 000000000000..27aab5399acf --- /dev/null +++ b/src/Assets/TestProjects/VSTestDataCollectorSampleNoMerge/AttachmentProcessorDataCollector.csproj @@ -0,0 +1,16 @@ + + + + netstandard2.0 + + + + + + + + + + + + diff --git a/src/Assets/TestProjects/VSTestDataCollectorSampleNoMerge/SampleDataCollector.cs b/src/Assets/TestProjects/VSTestDataCollectorSampleNoMerge/SampleDataCollector.cs new file mode 100644 index 000000000000..a5051c10f069 --- /dev/null +++ b/src/Assets/TestProjects/VSTestDataCollectorSampleNoMerge/SampleDataCollector.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace AttachmentProcessorDataCollector +{ + using System; + using System.Collections.Generic; + using System.Collections.ObjectModel; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading; + using System.Threading.Tasks; + using System.Xml; + + using Microsoft.VisualStudio.TestPlatform.ObjectModel; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection; + using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging; + + internal class ExtensionInfo + { + public const string ExtensionType = "DataCollector"; + public const string ExtensionIdentifier = "my://sample/datacollector"; + } + + [DataCollectorFriendlyName("SampleDataCollector")] + [DataCollectorTypeUri(ExtensionInfo.ExtensionIdentifier)] + public class SampleDataCollectorV1 : DataCollector + { + private DataCollectionSink _dataCollectionSink; + private DataCollectionEnvironmentContext _context; + private readonly string _tempDirectoryPath = Path.GetTempPath(); + + public override void Initialize( + XmlElement configurationElement, + DataCollectionEvents events, + DataCollectionSink dataSink, + DataCollectionLogger logger, + DataCollectionEnvironmentContext environmentContext) + { + events.SessionEnd += SessionEnded_Handler; + _dataCollectionSink = dataSink; + _context = environmentContext; + } + + private void SessionEnded_Handler(object sender, SessionEndEventArgs e) + { + string tmpAttachment = Path.Combine(_tempDirectoryPath, Guid.NewGuid().ToString("N"), "DataCollectorAttachmentProcessor_1.txt"); + Directory.CreateDirectory(Path.GetDirectoryName(tmpAttachment)); + File.WriteAllText(tmpAttachment, $"SessionEnded_Handler_{Guid.NewGuid():N}"); + _dataCollectionSink.SendFileAsync(_context.SessionDataCollectionContext, tmpAttachment, true); + } + } +} diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/sln.sln b/src/Assets/TestProjects/VSTestMultiProjectSolution/sln.sln new file mode 100644 index 000000000000..1c8555438b8f --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/sln.sln @@ -0,0 +1,34 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.30114.105 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test1", "test1\test1.csproj", "{081584FC-1000-4F74-887E-480E08A2FAD4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test2", "test2\test2.csproj", "{F77D1353-8580-46B5-820F-50A899BEA1DD}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test3", "test3\test3.csproj", "{9D8CCF24-5968-4E57-B181-98A026C8ABEA}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {081584FC-1000-4F74-887E-480E08A2FAD4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {081584FC-1000-4F74-887E-480E08A2FAD4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {081584FC-1000-4F74-887E-480E08A2FAD4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {081584FC-1000-4F74-887E-480E08A2FAD4}.Release|Any CPU.Build.0 = Release|Any CPU + {F77D1353-8580-46B5-820F-50A899BEA1DD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F77D1353-8580-46B5-820F-50A899BEA1DD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F77D1353-8580-46B5-820F-50A899BEA1DD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F77D1353-8580-46B5-820F-50A899BEA1DD}.Release|Any CPU.Build.0 = Release|Any CPU + {9D8CCF24-5968-4E57-B181-98A026C8ABEA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D8CCF24-5968-4E57-B181-98A026C8ABEA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D8CCF24-5968-4E57-B181-98A026C8ABEA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D8CCF24-5968-4E57-B181-98A026C8ABEA}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection +EndGlobal diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/test1/UnitTest1.cs b/src/Assets/TestProjects/VSTestMultiProjectSolution/test1/UnitTest1.cs new file mode 100644 index 000000000000..ef905ecd958f --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/test1/UnitTest1.cs @@ -0,0 +1,13 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace test1 +{ + [TestClass] + public class UnitTest1 + { + [TestMethod] + public void TestMethod1() + { + } + } +} diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/test1/test1.csproj b/src/Assets/TestProjects/VSTestMultiProjectSolution/test1/test1.csproj new file mode 100644 index 000000000000..0aa9091b66a5 --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/test1/test1.csproj @@ -0,0 +1,19 @@ + + + + + $(CurrentTargetFramework) + + + + + + + + + + + + + + diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/test2/UnitTest1.cs b/src/Assets/TestProjects/VSTestMultiProjectSolution/test2/UnitTest1.cs new file mode 100644 index 000000000000..64e82189211c --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/test2/UnitTest1.cs @@ -0,0 +1,13 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace test2 +{ + [TestClass] + public class UnitTest1 + { + [TestMethod] + public void TestMethod1() + { + } + } +} diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/test2/test2.csproj b/src/Assets/TestProjects/VSTestMultiProjectSolution/test2/test2.csproj new file mode 100644 index 000000000000..0aa9091b66a5 --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/test2/test2.csproj @@ -0,0 +1,19 @@ + + + + + $(CurrentTargetFramework) + + + + + + + + + + + + + + diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/test3/UnitTest1.cs b/src/Assets/TestProjects/VSTestMultiProjectSolution/test3/UnitTest1.cs new file mode 100644 index 000000000000..8ae2e4739ef1 --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/test3/UnitTest1.cs @@ -0,0 +1,13 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace test3 +{ + [TestClass] + public class UnitTest1 + { + [TestMethod] + public void TestMethod1() + { + } + } +} diff --git a/src/Assets/TestProjects/VSTestMultiProjectSolution/test3/test3.csproj b/src/Assets/TestProjects/VSTestMultiProjectSolution/test3/test3.csproj new file mode 100644 index 000000000000..0aa9091b66a5 --- /dev/null +++ b/src/Assets/TestProjects/VSTestMultiProjectSolution/test3/test3.csproj @@ -0,0 +1,19 @@ + + + + + $(CurrentTargetFramework) + + + + + + + + + + + + + + diff --git a/src/Cli/dotnet/commands/dotnet-test/Program.cs b/src/Cli/dotnet/commands/dotnet-test/Program.cs index 921f62264559..ab1d7c3758c2 100644 --- a/src/Cli/dotnet/commands/dotnet-test/Program.cs +++ b/src/Cli/dotnet/commands/dotnet-test/Program.cs @@ -1,9 +1,11 @@ -// Copyright(c) .NET Foundation and contributors.All rights reserved. +// Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; using System.Collections.Generic; +using System.CommandLine; using System.CommandLine.Parsing; +using System.IO; using System.Linq; using Microsoft.DotNet.Cli; using Microsoft.DotNet.Cli.Utils; @@ -20,7 +22,92 @@ public TestCommand( { } - public static TestCommand FromParseResult(ParseResult result, string[] settings, string msbuildPath = null) + public static int Run(ParseResult parseResult) + { + parseResult.HandleDebugSwitch(); + + FeatureFlag.Default.PrintFlagFeatureState(); + + // We use also current process id for the correlation id for possible future usage in case we need to know the parent process + // from the VSTest side. + string testSessionCorrelationId = $"{Environment.ProcessId}_{Guid.NewGuid()}"; + + string[] args = parseResult.GetArguments(); + + if (VSTestTrace.TraceEnabled) + { + string commandLineParameters = ""; + if (args?.Length > 0) + { + commandLineParameters = args.Aggregate((a, b) => $"{a} | {b}"); + } + VSTestTrace.SafeWriteTrace(() => $"Argument list: '{commandLineParameters}'"); + } + + // settings parameters are after -- (including --), these should not be considered by the parser + string[] settings = args.SkipWhile(a => a != "--").ToArray(); + // all parameters before -- + args = args.TakeWhile(a => a != "--").ToArray(); + + // Fix for https://github.com/Microsoft/vstest/issues/1453 + // Run dll/exe directly using the VSTestForwardingApp + if (ContainsBuiltTestSources(args)) + { + return ForwardToVSTestConsole(parseResult, args, settings, testSessionCorrelationId); + } + + return ForwardToMsbuild(parseResult, settings, testSessionCorrelationId); + } + + private static int ForwardToMsbuild(ParseResult parseResult, string[] settings, string testSessionCorrelationId) + { + // Workaround for https://github.com/Microsoft/vstest/issues/1503 + const string NodeWindowEnvironmentName = "MSBUILDENSURESTDOUTFORTASKPROCESSES"; + string previousNodeWindowSetting = Environment.GetEnvironmentVariable(NodeWindowEnvironmentName); + try + { + Environment.SetEnvironmentVariable(NodeWindowEnvironmentName, "1"); + int exitCode = FromParseResult(parseResult, settings, testSessionCorrelationId).Execute(); + + // We run post processing also if execution is failed for possible partial successful result to post process. + exitCode |= RunArtifactPostProcessingIfNeeded(testSessionCorrelationId, parseResult, FeatureFlag.Default); + + return exitCode; + } + finally + { + Environment.SetEnvironmentVariable(NodeWindowEnvironmentName, previousNodeWindowSetting); + } + } + + private static int ForwardToVSTestConsole(ParseResult parseResult, string[] args, string[] settings, string testSessionCorrelationId) + { + List convertedArgs = new VSTestArgumentConverter().Convert(args, out List ignoredArgs); + if (ignoredArgs.Any()) + { + Reporter.Output.WriteLine(string.Format(LocalizableStrings.IgnoredArgumentsMessage, string.Join(" ", ignoredArgs)).Yellow()); + } + + // merge the args settings, we don't need to escape + // one more time, there is no extra hop via msbuild + convertedArgs.AddRange(settings); + + if (FeatureFlag.Default.IsEnabled(FeatureFlag.ARTIFACTS_POSTPROCESSING)) + { + // Add artifacts processing mode and test session id for the artifact post-processing + convertedArgs.Add("--artifactsProcessingMode-collect"); + convertedArgs.Add($"--testSessionCorrelationId:{testSessionCorrelationId}"); + } + + int exitCode = new VSTestForwardingApp(convertedArgs).Execute(); + + // We run post processing also if execution is failed for possible partial successful result to post process. + exitCode |= RunArtifactPostProcessingIfNeeded(testSessionCorrelationId, parseResult, FeatureFlag.Default); + + return exitCode; + } + + private static TestCommand FromParseResult(ParseResult result, string[] settings, string testSessionCorrelationId, string msbuildPath = null) { result.ShowHelpOrErrorIfAppropriate(); @@ -38,25 +125,32 @@ public static TestCommand FromParseResult(ParseResult result, string[] settings, if (settings.Any()) { // skip '--' and escape every \ to be \\ and every " to be \" to survive the next hop - var escaped = settings.Skip(1).Select(s => s.Replace("\\", "\\\\").Replace("\"", "\\\"")).ToArray(); + string[] escaped = settings.Skip(1).Select(s => s.Replace("\\", "\\\\").Replace("\"", "\\\"")).ToArray(); - var runSettingsArg = string.Join(";", escaped); + string runSettingsArg = string.Join(";", escaped); msbuildArgs.Add($"-property:VSTestCLIRunSettings=\"{runSettingsArg}\""); } - var verbosityArg = result.ForwardedOptionValues>(TestCommandParser.GetCommand(), "verbosity")?.SingleOrDefault() ?? null; + string verbosityArg = result.ForwardedOptionValues>(TestCommandParser.GetCommand(), "verbosity")?.SingleOrDefault() ?? null; if (verbosityArg != null) { - var verbosity = verbosityArg.Split(':', 2); + string[] verbosity = verbosityArg.Split(':', 2); if (verbosity.Length == 2) { msbuildArgs.Add($"-property:VSTestVerbosity={verbosity[1]}"); } } + if (FeatureFlag.Default.IsEnabled(FeatureFlag.ARTIFACTS_POSTPROCESSING)) + { + // Add artifacts processing mode and test session id for the artifact post-processing + msbuildArgs.Add("-property:VSTestArtifactsProcessingMode=collect"); + msbuildArgs.Add($"-property:VSTestSessionCorrelationId={testSessionCorrelationId}"); + } + bool noRestore = result.HasOption(TestCommandParser.NoRestoreOption) || result.HasOption(TestCommandParser.NoBuildOption); - TestCommand testCommand = new TestCommand( + TestCommand testCommand = new( msbuildArgs, noRestore, msbuildPath); @@ -70,57 +164,61 @@ public static TestCommand FromParseResult(ParseResult result, string[] settings, if (!hasRootVariable) { testCommand.EnvironmentVariable(rootVariableName, rootValue); + VSTestTrace.SafeWriteTrace(() => $"Root variable set {rootVariableName}:{rootValue}"); } + VSTestTrace.SafeWriteTrace(() => $"Starting test using MSBuild with arguments '{testCommand.GetArgumentsToMSBuild()}' custom MSBuild path '{msbuildPath}' norestore '{noRestore}'"); return testCommand; } - public static int Run(ParseResult parseResult) + internal static int RunArtifactPostProcessingIfNeeded(string testSessionCorrelationId, ParseResult parseResult, FeatureFlag featureFlag) { - parseResult.HandleDebugSwitch(); - - var args = parseResult.GetArguments(); - - // settings parameters are after -- (including --), these should not be considered by the parser - var settings = args.SkipWhile(a => a != "--").ToArray(); - // all parameters before -- - args = args.TakeWhile(a => a != "--").ToArray(); + if (!featureFlag.IsEnabled(FeatureFlag.ARTIFACTS_POSTPROCESSING)) + { + return 0; + } - // Fix for https://github.com/Microsoft/vstest/issues/1453 - // Try to run dll/exe directly using the VSTestForwardingApp - if (ContainsBuiltTestSources(args)) + // VSTest runner will save artifacts inside a temp folder if needed. + string expectedArtifactDirectory = Path.Combine(Path.GetTempPath(), testSessionCorrelationId); + if (!Directory.Exists(expectedArtifactDirectory)) { - var convertedArgs = new VSTestArgumentConverter().Convert(args, out var ignoredArgs); - if (ignoredArgs.Any()) - { - Reporter.Output.WriteLine(string.Format(LocalizableStrings.IgnoredArgumentsMessage, string.Join(" ", ignoredArgs)).Yellow()); - } + VSTestTrace.SafeWriteTrace(() => "No artifact found, post-processing won't run."); + return 0; + } - // merge the args settings, we don't need to escape - // one more time, there is no extra hop via msbuild - convertedArgs.AddRange(settings); + VSTestTrace.SafeWriteTrace(() => $"Artifacts directory found '{expectedArtifactDirectory}', running post-processing."); - return new VSTestForwardingApp(convertedArgs).Execute(); - } + var artifactsPostProcessArgs = new List { "--artifactsProcessingMode-postprocess", $"--testSessionCorrelationId:{testSessionCorrelationId}" }; - // Workaround for https://github.com/Microsoft/vstest/issues/1503 - const string NodeWindowEnvironmentName = "MSBUILDENSURESTDOUTFORTASKPROCESSES"; - string previousNodeWindowSetting = Environment.GetEnvironmentVariable(NodeWindowEnvironmentName); + if (parseResult.HasOption(TestCommandParser.DiagOption)) + { + artifactsPostProcessArgs.Add($"--diag:{parseResult.GetValueForOption(TestCommandParser.DiagOption)}"); + } try { - Environment.SetEnvironmentVariable(NodeWindowEnvironmentName, "1"); - return FromParseResult(parseResult, settings).Execute(); + return new VSTestForwardingApp(artifactsPostProcessArgs).Execute(); } finally { - Environment.SetEnvironmentVariable(NodeWindowEnvironmentName, previousNodeWindowSetting); + if (Directory.Exists(expectedArtifactDirectory)) + { + VSTestTrace.SafeWriteTrace(() => $"Cleaning artifact directory '{expectedArtifactDirectory}'."); + try + { + Directory.Delete(expectedArtifactDirectory, true); + } + catch (Exception ex) + { + VSTestTrace.SafeWriteTrace(() => $"Exception during artifact cleanup: \n{ex}"); + } + } } } private static bool ContainsBuiltTestSources(string[] args) { - foreach (var arg in args) + foreach (string arg in args) { if (!arg.StartsWith("-") && (arg.EndsWith("dll", StringComparison.OrdinalIgnoreCase) || arg.EndsWith("exe", StringComparison.OrdinalIgnoreCase))) @@ -133,19 +231,19 @@ private static bool ContainsBuiltTestSources(string[] args) private static void SetEnvironmentVariablesFromParameters(TestCommand testCommand, ParseResult parseResult) { - var option = TestCommandParser.EnvOption; + Option> option = TestCommandParser.EnvOption; if (!parseResult.HasOption(option)) { return; } - foreach (var env in parseResult.GetValueForOption(option)) + foreach (string env in parseResult.GetValueForOption(option)) { - var name = env; - var value = string.Empty; + string name = env; + string value = string.Empty; - var equalsIndex = env.IndexOf('='); + int equalsIndex = env.IndexOf('='); if (equalsIndex > 0) { name = env.Substring(0, equalsIndex); diff --git a/src/Cli/dotnet/commands/dotnet-test/TestCommandParser.cs b/src/Cli/dotnet/commands/dotnet-test/TestCommandParser.cs index c9e65ed2f9bb..33bcdc367ae9 100644 --- a/src/Cli/dotnet/commands/dotnet-test/TestCommandParser.cs +++ b/src/Cli/dotnet/commands/dotnet-test/TestCommandParser.cs @@ -64,7 +64,9 @@ internal static class TestCommandParser public static readonly Option DiagOption = new ForwardedOption(new string[] { "-d", "--diag" }, LocalizableStrings.CmdPathTologFileDescription) { ArgumentHelpName = LocalizableStrings.CmdPathToLogFile - }.ForwardAsSingle(o => $"-property:VSTestDiag={SurroundWithDoubleQuotes(CommandDirectoryContext.GetFullPath(o))}"); + } + // Temporarily we don't escape the diag path, issue https://github.com/dotnet/sdk/issues/23970 + .ForwardAsSingle(o => $"-property:VSTestDiag={CommandDirectoryContext.GetFullPath(o)}"); public static readonly Option NoBuildOption = new ForwardedOption("--no-build", LocalizableStrings.CmdNoBuildDescription) .ForwardAs("-property:VSTestNoBuild=true"); diff --git a/src/Cli/dotnet/commands/dotnet-test/VSTestArgumentConverter.cs b/src/Cli/dotnet/commands/dotnet-test/VSTestArgumentConverter.cs index dfdbd86f9ed0..8db8cf3f8d5f 100644 --- a/src/Cli/dotnet/commands/dotnet-test/VSTestArgumentConverter.cs +++ b/src/Cli/dotnet/commands/dotnet-test/VSTestArgumentConverter.cs @@ -49,7 +49,9 @@ public class VSTestArgumentConverter "--output", "--no-build", "--no-restore", - "--interactive" + "--interactive", + "--testSessionId", + "--artifacts-processing-mode" }; /// diff --git a/src/Cli/dotnet/commands/dotnet-test/VSTestFeatureFlag.cs b/src/Cli/dotnet/commands/dotnet-test/VSTestFeatureFlag.cs new file mode 100644 index 000000000000..b15c86c60db8 --- /dev/null +++ b/src/Cli/dotnet/commands/dotnet-test/VSTestFeatureFlag.cs @@ -0,0 +1,45 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; + +namespace Microsoft.DotNet.Tools.Test +{ + // !!! FEATURES MUST BE KEPT IN SYNC WITH https://github.com/microsoft/vstest/blob/main/src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs !!! + internal class FeatureFlag + { + private const string Prefix = "VSTEST_FEATURE_"; + + public Dictionary FeatureFlags { get; } = new(); + + public static FeatureFlag Default { get; } = new FeatureFlag(); + + public FeatureFlag() + { + FeatureFlags.Add(ARTIFACTS_POSTPROCESSING, false); + } + + // Added for artifact porst-processing, it enable/disable the post processing. + // Added in 17.2-preview 7.0-preview + public const string ARTIFACTS_POSTPROCESSING = Prefix + "ARTIFACTS_POSTPROCESSING"; + + // For now we're checking env var. + // We could add it also to some section inside the runsettings. + public bool IsEnabled(string featureName) => + int.TryParse(Environment.GetEnvironmentVariable(featureName), out int enabled) + ? enabled == 1 + : FeatureFlags.TryGetValue(featureName, out bool isEnabled) && isEnabled; + + public void PrintFlagFeatureState() + { + if (VSTestTrace.TraceEnabled) + { + foreach (KeyValuePair flag in FeatureFlags) + { + VSTestTrace.SafeWriteTrace(() => $"Feature {flag.Key}: {IsEnabled(flag.Key)}"); + } + } + } + } +} diff --git a/src/Cli/dotnet/commands/dotnet-test/VSTestForwardingApp.cs b/src/Cli/dotnet/commands/dotnet-test/VSTestForwardingApp.cs index 39639f4d2757..0190ff78a8d4 100644 --- a/src/Cli/dotnet/commands/dotnet-test/VSTestForwardingApp.cs +++ b/src/Cli/dotnet/commands/dotnet-test/VSTestForwardingApp.cs @@ -2,9 +2,11 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using Microsoft.DotNet.Cli.Utils; +using Microsoft.DotNet.Tools.Test; using System; using System.Collections.Generic; using System.IO; +using System.Linq; namespace Microsoft.DotNet.Cli { @@ -19,7 +21,10 @@ public VSTestForwardingApp(IEnumerable argsToForward) if (!hasRootVariable) { WithEnvironmentVariable(rootVariableName, rootValue); + VSTestTrace.SafeWriteTrace(() => $"Root variable set {rootVariableName}:{rootValue}"); } + + VSTestTrace.SafeWriteTrace(() => $"Forwarding to '{GetVSTestExePath()}' with args \"{argsToForward?.Aggregate((a, b) => $"{a} | {b}")}\""); } private static string GetVSTestExePath() diff --git a/src/Cli/dotnet/commands/dotnet-test/VSTestTrace.cs b/src/Cli/dotnet/commands/dotnet-test/VSTestTrace.cs new file mode 100644 index 000000000000..0df075b60032 --- /dev/null +++ b/src/Cli/dotnet/commands/dotnet-test/VSTestTrace.cs @@ -0,0 +1,53 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; + +namespace Microsoft.DotNet.Tools.Test +{ + internal class VSTestTrace + { + public static bool TraceEnabled { get; private set; } + private static readonly string s_traceFilePath; + + static VSTestTrace() + { + TraceEnabled = int.TryParse(Environment.GetEnvironmentVariable("DOTNET_CLI_VSTEST_TRACE"), out int enabled) && enabled == 1; + s_traceFilePath = Environment.GetEnvironmentVariable("DOTNET_CLI_VSTEST_TRACEFILE"); + if (TraceEnabled) + { + Console.WriteLine($"[dotnet test - {DateTime.UtcNow}]Logging to {(!string.IsNullOrEmpty(s_traceFilePath) ? s_traceFilePath : "console")}"); + } + } + + public static void SafeWriteTrace(Func messageLog) + { + if (!TraceEnabled) + { + return; + } + + try + { + string message = $"[dotnet test - {DateTimeOffset.UtcNow}]{messageLog()}"; + if (!string.IsNullOrEmpty(s_traceFilePath)) + { + lock (s_traceFilePath) + { + using StreamWriter logFile = File.AppendText(s_traceFilePath); + logFile.WriteLine(message); + } + } + else + { + Console.WriteLine(message); + } + } + catch (Exception ex) + { + Console.WriteLine($"[dotnet test - {DateTimeOffset.UtcNow}]{ex}"); + } + } + } +} diff --git a/src/Cli/dotnet/commands/dotnet-vstest/Program.cs b/src/Cli/dotnet/commands/dotnet-vstest/Program.cs index 76cd3b56fe54..a10a3f29ad1c 100644 --- a/src/Cli/dotnet/commands/dotnet-vstest/Program.cs +++ b/src/Cli/dotnet/commands/dotnet-vstest/Program.cs @@ -1,12 +1,12 @@ -// Copyright(c) .NET Foundation and contributors.All rights reserved. +// Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using Microsoft.DotNet.Cli.Utils; -using Microsoft.DotNet.Cli; -using System.CommandLine.Parsing; +using System; using System.Collections.Generic; +using System.CommandLine.Parsing; using System.Linq; -using System; +using Microsoft.DotNet.Cli; +using Microsoft.DotNet.Tools.Test; namespace Microsoft.DotNet.Tools.VSTest { @@ -16,9 +16,28 @@ public static int Run(ParseResult parseResult) { parseResult.HandleDebugSwitch(); - VSTestForwardingApp vsTestforwardingApp = new VSTestForwardingApp(GetArgs(parseResult)); + // We use also current process id for the correlation id for possible future usage in case we need to know the parent process + // from the VSTest side. + string testSessionCorrelationId = $"{Environment.ProcessId}_{Guid.NewGuid()}"; + + var args = new List(); + args.AddRange(GetArgs(parseResult)); + + if (FeatureFlag.Default.IsEnabled(FeatureFlag.ARTIFACTS_POSTPROCESSING)) + { + // Add artifacts processing mode and test session id for the artifact post-processing + args.Add("--artifactsProcessingMode-collect"); + args.Add($"--testSessionCorrelationId:{testSessionCorrelationId}"); + } + + VSTestForwardingApp vsTestforwardingApp = new(args); + + int exitCode = vsTestforwardingApp.Execute(); + + // We run post processing also if execution is failed for possible partial successful result to post process. + exitCode |= TestCommand.RunArtifactPostProcessingIfNeeded(testSessionCorrelationId, parseResult, FeatureFlag.Default); - return vsTestforwardingApp.Execute(); + return exitCode; } private static string[] GetArgs(ParseResult parseResult) @@ -28,7 +47,7 @@ private static string[] GetArgs(ParseResult parseResult) if (parseResult.HasOption(CommonOptions.TestLoggerOption)) { // System command line might have mutated the options, reformat test logger option so vstest recognizes it - var loggerValue = parseResult.GetValueForOption(CommonOptions.TestLoggerOption); + string loggerValue = parseResult.GetValueForOption(CommonOptions.TestLoggerOption); args = args.Where(a => !a.Equals(loggerValue) && !CommonOptions.TestLoggerOption.Aliases.Contains(a)); args = args.Prepend($"{CommonOptions.TestLoggerOption.Aliases.First()}:{loggerValue}"); } diff --git a/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsArtifactPostProcessing.cs b/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsArtifactPostProcessing.cs new file mode 100644 index 000000000000..583e22c60fae --- /dev/null +++ b/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsArtifactPostProcessing.cs @@ -0,0 +1,247 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Xml; +using System.Xml.Linq; +using FluentAssertions; +using Microsoft.DotNet.Tools.Test; +using Microsoft.NET.TestFramework; +using Microsoft.NET.TestFramework.Assertions; +using Microsoft.NET.TestFramework.Commands; +using Xunit; +using Xunit.Abstractions; +using CommandResult = Microsoft.DotNet.Cli.Utils.CommandResult; + +namespace Microsoft.DotNet.Cli.Test.Tests +{ + public class GivenDotnetTestBuildsAndRunsArtifactPostProcessing : SdkTest + { + private static object s_dataCollectorInitLock = new(); + private static string s_dataCollectorDll; + private static string s_dataCollectorNoMergeDll; + + public GivenDotnetTestBuildsAndRunsArtifactPostProcessing(ITestOutputHelper log) : base(log) + { + BuildDataCollector(); + BuildDataCollectorNoMerge(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ArtifactPostProcessing_SolutionProjects(bool merge) + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("VSTestMultiProjectSolution", Guid.NewGuid().ToString()) + .WithSource(); + + string runsettings = GetRunsetting(testInstance.Path); + + CommandResult result = new DotnetTestCommand(Log) + .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable(FeatureFlag.ARTIFACTS_POSTPROCESSING, "1") + .Execute( + "--configuration", "release", + "--collect", "SampleDataCollector", + "--test-adapter-path", merge ? Path.GetDirectoryName(s_dataCollectorDll) : Path.GetDirectoryName(s_dataCollectorNoMergeDll), + "--settings", runsettings, + "--diag", testInstance.Path + "/logs/"); + + result.ExitCode.Should().Be(0); + AssertOutput(result.StdOut, merge); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ArtifactPostProcessing_TestContainers(bool merge) + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("VSTestMultiProjectSolution", Guid.NewGuid().ToString()) + .WithSource(); + + string runsettings = GetRunsetting(testInstance.Path); + + new PublishCommand(Log, Path.Combine(testInstance.Path, "sln.sln")).Execute("/p:Configuration=Release").Should().Pass(); + + CommandResult result = new DotnetTestCommand(Log) + .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable(FeatureFlag.ARTIFACTS_POSTPROCESSING, "1") + .WithEnvironmentVariable("DOTNET_CLI_VSTEST_TRACE", "1") + .Execute( + Directory.GetFiles(testInstance.Path, "test1.dll", SearchOption.AllDirectories).SingleOrDefault(x => x.Contains("publish")), + Directory.GetFiles(testInstance.Path, "test2.dll", SearchOption.AllDirectories).SingleOrDefault(x => x.Contains("publish")), + Directory.GetFiles(testInstance.Path, "test3.dll", SearchOption.AllDirectories).SingleOrDefault(x => x.Contains("publish")), + "--collect:SampleDataCollector", + $"--test-adapter-path:{(merge ? Path.GetDirectoryName(s_dataCollectorDll) : Path.GetDirectoryName(s_dataCollectorNoMergeDll))}", + $"--settings:{runsettings}", + "--diag:" + testInstance.Path + "/logs/"); + + result.ExitCode.Should().Be(0); + AssertOutput(result.StdOut, merge); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ArtifactPostProcessing_VSTest_TestContainers(bool merge) + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("VSTestMultiProjectSolution", Guid.NewGuid().ToString()) + .WithSource(); + + string runsettings = GetRunsetting(testInstance.Path); + + new PublishCommand(Log, Path.Combine(testInstance.Path, "sln.sln")).Execute("/p:Configuration=Release").Should().Pass(); + + CommandResult result = new DotnetVSTestCommand(Log) + .WithWorkingDirectory(testInstance.Path) + .WithEnvironmentVariable(FeatureFlag.ARTIFACTS_POSTPROCESSING, "1") + .Execute( + Directory.GetFiles(testInstance.Path, "test1.dll", SearchOption.AllDirectories).SingleOrDefault(x => x.Contains("publish")), + Directory.GetFiles(testInstance.Path, "test2.dll", SearchOption.AllDirectories).SingleOrDefault(x => x.Contains("publish")), + Directory.GetFiles(testInstance.Path, "test3.dll", SearchOption.AllDirectories).SingleOrDefault(x => x.Contains("publish")), + "--collect:SampleDataCollector", + $"--testAdapterPath:{(merge ? Path.GetDirectoryName(s_dataCollectorDll) : Path.GetDirectoryName(s_dataCollectorNoMergeDll))}", + $"--settings:{runsettings}", + $"--diag:{testInstance.Path}/logs/"); + + result.ExitCode.Should().Be(0); + AssertOutput(result.StdOut, merge); + } + + private static void AssertOutput(string stdOut, bool merge) + { + List output = new(); + using StringReader reader = new(stdOut); + while (true) + { + string line = reader.ReadLine()?.Trim(); + if (line is null) break; + output.Add(line); + } + + if (merge) + { + output[^3].Trim().Should().BeEmpty(); + output[^2].Trim().Should().Be("Attachments:"); + string mergedFile = output[^1].Trim(); + + var fileContent = new HashSet(); + using var streamReader = new StreamReader(mergedFile); + LoadLines(streamReader, fileContent); + fileContent.Count.Should().Be(3); + } + else + { + output[^5].Trim().Should().BeEmpty(); + output[^4].Trim().Should().Be("Attachments:"); + + int currentLine = 0; + for (int i = 3; i > 0; i--) + { + currentLine = output.Count - i; + string file = output[currentLine].Trim(); + var fileContent = new HashSet(); + using var streamReader = new StreamReader(file); + LoadLines(streamReader, fileContent); + fileContent.Count.Should().Be(1); + } + + output.Count.Should().Be(currentLine + 1); + } + + static void LoadLines(StreamReader stream, HashSet fileContent) + { + while (!stream.EndOfStream) + { + string line = stream.ReadLine(); + line.Should().StartWith("SessionEnded_Handler_"); + fileContent.Add(line); + } + } + } + + private void BuildDataCollector() + => LazyInitializer.EnsureInitialized(ref s_dataCollectorDll, ref s_dataCollectorInitLock, () => + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("VSTestDataCollectorSample").WithSource(); + + string testProjectDirectory = testInstance.Path; + + new BuildCommand(testInstance) + .Execute("/p:Configuration=Release") + .Should() + .Pass(); + + return Directory.GetFiles(testProjectDirectory, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Single(x => x.Contains("bin")); + }); + + private void BuildDataCollectorNoMerge() + => LazyInitializer.EnsureInitialized(ref s_dataCollectorNoMergeDll, ref s_dataCollectorInitLock, () => + { + TestAsset testInstance = _testAssetsManager.CopyTestAsset("VSTestDataCollectorSampleNoMerge").WithSource(); + + string testProjectDirectory = testInstance.Path; + + new BuildCommand(testInstance) + .Execute("/p:Configuration=Release") + .Should() + .Pass(); + + return Directory.GetFiles(testProjectDirectory, "AttachmentProcessorDataCollector.dll", SearchOption.AllDirectories).Single(x => x.Contains("bin")); + }); + + private static string GetRunsetting(string directory) + { + string runSettings = GetRunsettingsFilePath(directory); + // Set datacollector parameters + XElement runSettingsXml = XElement.Load(runSettings); + runSettingsXml.Element("DataCollectionRunSettings") + .Element("DataCollectors") + .Element("DataCollector") + .Add(new XElement("Configuration", new XElement("MergeFile", "MergedFile.txt"))); + runSettingsXml.Save(runSettings); + return runSettings; + } + + private static string GetRunsettingsFilePath(string resultsDir) + { + string runsettingsPath = Path.Combine(resultsDir, "test_" + Guid.NewGuid() + ".runsettings"); + var dataCollectionAttributes = new Dictionary + { + { "friendlyName", "SampleDataCollector" }, + { "uri", "my://sample/datacollector" } + }; + + CreateDataCollectionRunSettingsFile(runsettingsPath, dataCollectionAttributes); + return runsettingsPath; + } + + private static void CreateDataCollectionRunSettingsFile(string destinationRunsettingsPath, Dictionary dataCollectionAttributes) + { + var doc = new XmlDocument(); + XmlNode xmlDeclaration = doc.CreateNode(XmlNodeType.XmlDeclaration, string.Empty, string.Empty); + + doc.AppendChild(xmlDeclaration); + XmlElement runSettingsNode = doc.CreateElement("RunSettings"); + doc.AppendChild(runSettingsNode); + XmlElement dcConfigNode = doc.CreateElement("DataCollectionRunSettings"); + runSettingsNode.AppendChild(dcConfigNode); + XmlElement dataCollectorsNode = doc.CreateElement("DataCollectors"); + dcConfigNode.AppendChild(dataCollectorsNode); + XmlElement dataCollectorNode = doc.CreateElement("DataCollector"); + dataCollectorsNode.AppendChild(dataCollectorNode); + + foreach (KeyValuePair kvp in dataCollectionAttributes) + { + dataCollectorNode.SetAttribute(kvp.Key, kvp.Value); + } + + using var stream = new FileStream(destinationRunsettingsPath, FileMode.Create); + doc.Save(stream); + } + } +} diff --git a/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs b/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs index 2c8d3ba1212f..6868d1938181 100644 --- a/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs +++ b/src/Tests/dotnet-test.Tests/GivenDotnetTestBuildsAndRunsTestfromCsproj.cs @@ -728,7 +728,8 @@ public void FilterPropertyCorrectlyHandlesComma(string filter, string folderSuff [Theory] [InlineData("--output")] - [InlineData("--diag")] + // Temporarily we don't escape the diag path, issue https://github.com/dotnet/sdk/issues/23970 + // [InlineData("--diag")] [InlineData("--results-directory")] public void EnsureOutputPathEscaped(string flag) {