From cfca759294095b723ad2bd39ff469c2f266fb324 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 7 Sep 2022 18:42:47 -0700 Subject: [PATCH 1/3] Enable the fundamentals behind libraries tests compiled via crossgen2 - Document how to use the special modes of libraries compilation - Add the TestCrossgen2 flag to libraries testing - Unlike TestNativeAot and TestSingleFile, use a self-contained strategy instead of actual single-file to improve debuggability of the produced output - Tweak the SingleFileTestRunner.cs to avoid the fork bomb problem introduced by RemoteTestExecutor in a self-contained, but not singlefile scenario - Add support for a set of flags to SingleFileTestRunner to respect the command line options I needed when debugging why the tests were not working. --- docs/workflow/testing/libraries/testing.md | 19 ++++++- eng/testing/tests.props | 1 + eng/testing/tests.singlefile.targets | 11 +++- .../SingleFileTestRunner.cs | 51 +++++++++++++++++++ 4 files changed, 79 insertions(+), 3 deletions(-) diff --git a/docs/workflow/testing/libraries/testing.md b/docs/workflow/testing/libraries/testing.md index cb17c48b6cc7a..140e2f77dc7c2 100644 --- a/docs/workflow/testing/libraries/testing.md +++ b/docs/workflow/testing/libraries/testing.md @@ -101,4 +101,21 @@ dotnet build /t:Test /p:Outerloop=true ### Running tests on a different target framework -Each test project can potentially have multiple target frameworks. There are some tests that might be OS-specific, or might be testing an API that is available only on some target frameworks, so the `TargetFrameworks` property specifies the valid target frameworks. \ No newline at end of file +Each test project can potentially have multiple target frameworks. There are some tests that might be OS-specific, or might be testing an API that is available only on some target frameworks, so the `TargetFrameworks` property specifies the valid target frameworks. + +### Running tests in custom compilation modes + +There are several custom compilation modes for tests. These are enabled by setting a switch during the config. + +| Mode | Description | Prerequisites +| --- | --- | --- | +| TestSingleFile | Test compilation using the single file compilation mode | libs+clr | +| TestNativeAot | Test by compiling the test using NativeAOT | libs+clr.aot | +| TestCrossgen2 | Test compilation of the test/libraries into R2R binaries | libs+clr | + +To run a test in specific mode, simply build the tests after building the prerequisite subsets, and specify the test mode on a command line such as: +```cmd +dotnet build /t:Test /p:Configuration=Release /p:TestCrossgen2=true +``` + +These tests do not use the standard XUnit test runner, instead they use [SingleFileTestRunner.cs](../../../../src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs). The set of available command line options is limited to `-xml`, `-notrait`, `-class`, `-class-`, `-noclass`, `-method`, `-method-`, `-nomethod`, `-namespace`, `-namespace-`, `-nonamespace`, and `-parallel`. diff --git a/eng/testing/tests.props b/eng/testing/tests.props index 7ed95b94326e3..2cb609dac0552 100644 --- a/eng/testing/tests.props +++ b/eng/testing/tests.props @@ -8,6 +8,7 @@ true $(MSBuildThisFileDirectory)ILLinkDescriptors\ true + true diff --git a/eng/testing/tests.singlefile.targets b/eng/testing/tests.singlefile.targets index d307e837e6235..3b67ffac44c61 100644 --- a/eng/testing/tests.singlefile.targets +++ b/eng/testing/tests.singlefile.targets @@ -2,8 +2,6 @@ Exe - $(DefineConstants);SINGLE_FILE_TEST_RUNNER - $([MSBuild]::NormalizeDirectory('$(OutDir)', 'publish')) $([MSBuild]::NormalizePath('$(BundleDir)', '$(RunScriptOutputName)')) $(PackageRID) @@ -18,6 +16,11 @@ true + + true + false + + $(CoreCLRILCompilerDir) $(CoreCLRCrossILCompilerDir) @@ -35,6 +38,10 @@ true + + $(DefineConstants);SINGLE_FILE_TEST_RUNNER + + diff --git a/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs b/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs index 260b2643d0c0d..4b71b4ef68537 100644 --- a/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs +++ b/src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs @@ -27,6 +27,9 @@ public static int Main(string[] args) var asm = typeof(SingleFileTestRunner).Assembly; Console.WriteLine("Running assembly:" + asm.FullName); + // The current RemoteExecutor implementation is not compatible with the SingleFileTestRunner + Environment.SetEnvironmentVariable("DOTNET_REMOTEEXECUTOR_SUPPORTED", "0"); + var diagnosticSink = new ConsoleDiagnosticMessageSink(); var testsFinished = new TaskCompletionSource(); var testSink = new TestMessageSink(); @@ -74,10 +77,58 @@ public static int Main(string[] args) noTraits.Add(traitKeyValue[0], values = new List()); } values.Add(traitKeyValue[1]); + i++; } if (args[i].Equals("-xml", StringComparison.OrdinalIgnoreCase)) { xmlResultFileName=args[i + 1].Trim(); + i++; + } + if (args[i].Equals("-class", StringComparison.OrdinalIgnoreCase)) + { + filters.IncludedClasses.Add(args[i + 1].Trim()); + i++; + } + if (args[i].Equals("-noclass", StringComparison.OrdinalIgnoreCase) || args[i].Equals("-class-", StringComparison.OrdinalIgnoreCase)) + { + filters.ExcludedClasses.Add(args[i + 1].Trim()); + i++; + } + if (args[i].Equals("-method", StringComparison.OrdinalIgnoreCase)) + { + filters.IncludedMethods.Add(args[i + 1].Trim()); + i++; + } + if (args[i].Equals("-nomethod", StringComparison.OrdinalIgnoreCase) || args[i].Equals("-method-", StringComparison.OrdinalIgnoreCase)) + { + filters.ExcludedMethods.Add(args[i + 1].Trim()); + i++; + } + if (args[i].Equals("-namespace", StringComparison.OrdinalIgnoreCase)) + { + filters.IncludedNamespaces.Add(args[i + 1].Trim()); + i++; + } + if (args[i].Equals("-nonamespace", StringComparison.OrdinalIgnoreCase) || args[i].Equals("-namespace-", StringComparison.OrdinalIgnoreCase)) + { + filters.ExcludedNamespaces.Add(args[i + 1].Trim()); + i++; + } + if (args[i].Equals("-parallel", StringComparison.OrdinalIgnoreCase)) + { + string parallelismArg = args[i + 1].ToLower(); + var (parallelizeAssemblies, parallelizeTestCollections) = parallelismArg switch + { + "all" => (true, true), + "assemblies" => (true, false), + "collections" => (false, true), + "none" => (false, false), + _ => throw new ArgumentException("unknown parallelism option") + }; + + assemblyConfig.ParallelizeAssembly = parallelizeAssemblies; + assemblyConfig.ParallelizeTestCollections = parallelizeTestCollections; + i++; } } From 6d7a7f3440c1fafaff3b4618be85f71935332ba2 Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 8 Sep 2022 09:18:26 -0700 Subject: [PATCH 2/3] Fix issues noted by code reviewers, and CI --- docs/workflow/testing/libraries/testing.md | 4 ++-- eng/testing/tests.props | 2 +- eng/testing/tests.singlefile.targets | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/workflow/testing/libraries/testing.md b/docs/workflow/testing/libraries/testing.md index 140e2f77dc7c2..374b6eee8f159 100644 --- a/docs/workflow/testing/libraries/testing.md +++ b/docs/workflow/testing/libraries/testing.md @@ -111,11 +111,11 @@ There are several custom compilation modes for tests. These are enabled by setti | --- | --- | --- | | TestSingleFile | Test compilation using the single file compilation mode | libs+clr | | TestNativeAot | Test by compiling the test using NativeAOT | libs+clr.aot | -| TestCrossgen2 | Test compilation of the test/libraries into R2R binaries | libs+clr | +| TestReadyToRun | Test compilation of the test/libraries into R2R binaries | libs+clr | To run a test in specific mode, simply build the tests after building the prerequisite subsets, and specify the test mode on a command line such as: ```cmd -dotnet build /t:Test /p:Configuration=Release /p:TestCrossgen2=true +dotnet build /t:Test /p:Configuration=Release /p:TestReadyToRun=true ``` These tests do not use the standard XUnit test runner, instead they use [SingleFileTestRunner.cs](../../../../src/libraries/Common/tests/SingleFileTestRunner/SingleFileTestRunner.cs). The set of available command line options is limited to `-xml`, `-notrait`, `-class`, `-class-`, `-noclass`, `-method`, `-method-`, `-nomethod`, `-namespace`, `-namespace-`, `-nonamespace`, and `-parallel`. diff --git a/eng/testing/tests.props b/eng/testing/tests.props index 2cb609dac0552..fbff925bd613a 100644 --- a/eng/testing/tests.props +++ b/eng/testing/tests.props @@ -8,7 +8,7 @@ true $(MSBuildThisFileDirectory)ILLinkDescriptors\ true - true + true diff --git a/eng/testing/tests.singlefile.targets b/eng/testing/tests.singlefile.targets index 3b67ffac44c61..a814f2250b0bf 100644 --- a/eng/testing/tests.singlefile.targets +++ b/eng/testing/tests.singlefile.targets @@ -16,7 +16,7 @@ true - + true false @@ -38,7 +38,7 @@ true - + $(DefineConstants);SINGLE_FILE_TEST_RUNNER From 4d9c0dee5f249f0be07060c968af1c38a11fafdd Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Thu, 8 Sep 2022 12:27:38 -0700 Subject: [PATCH 3/3] Actually use live built crossgen2, and workaround issues around project references as content --- eng/testing/tests.singlefile.targets | 14 ++++++++++++++ ...System.Diagnostics.FileVersionInfo.Tests.csproj | 2 ++ .../tests/System.Diagnostics.Process.Tests.csproj | 2 ++ 3 files changed, 18 insertions(+) diff --git a/eng/testing/tests.singlefile.targets b/eng/testing/tests.singlefile.targets index a814f2250b0bf..86f7277aae032 100644 --- a/eng/testing/tests.singlefile.targets +++ b/eng/testing/tests.singlefile.targets @@ -90,6 +90,20 @@ + + + $([MSBuild]::NormalizeDirectory('$(CoreCLRArtifactsPath)','crossgen2'))crossgen2$(ExeSuffix) + + + + + + + + + + + PreserveNewest + + diff --git a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj index e23bb4a94a71c..d2ff9a91c113a 100644 --- a/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj +++ b/src/libraries/System.Diagnostics.Process/tests/System.Diagnostics.Process.Tests.csproj @@ -68,5 +68,7 @@ Content PreserveNewest + +