Additional changes to merged test runners

Directory.Build.props

@@ -93,6 +93,7 @@
     <WorkloadBuildTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'WorkloadBuildTasks', 'Debug', '$(NetCoreAppToolCurrent)', 'publish'))</WorkloadBuildTasksDir>
     <MonoAOTCompilerDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoAOTCompiler', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoAOTCompilerDir>
     <MonoTargetsTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'MonoTargetsTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</MonoTargetsTasksDir>
+    <TestExclusionListTasksDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'TestExclusionListTasks', 'Debug', '$(NetCoreAppToolCurrent)'))</TestExclusionListTasksDir>
     <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
     <InstallerTasksAssemblyPath Condition="'$(MSBuildRuntimeType)' != 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'installer.tasks', 'Debug', '$(NetFrameworkToolCurrent)', 'installer.tasks.dll'))</InstallerTasksAssemblyPath>
     <Crossgen2SdkOverridePropsPath Condition="'$(MSBuildRuntimeType)' == 'Core'">$([MSBuild]::NormalizePath('$(ArtifactsBinDir)', 'Crossgen2Tasks', 'Debug', '$(NetCoreAppToolCurrent)', 'Microsoft.NET.CrossGen.props'))</Crossgen2SdkOverridePropsPath>
@@ -104,6 +105,7 @@
     <WorkloadBuildTasksAssemblyPath>$([MSBuild]::NormalizePath('$(WorkloadBuildTasksDir)', 'WorkloadBuildTasks.dll'))</WorkloadBuildTasksAssemblyPath>
     <MonoAOTCompilerTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoAOTCompilerDir)', 'MonoAOTCompiler.dll'))</MonoAOTCompilerTasksAssemblyPath>
     <MonoTargetsTasksAssemblyPath>$([MSBuild]::NormalizePath('$(MonoTargetsTasksDir)', 'MonoTargetsTasks.dll'))</MonoTargetsTasksAssemblyPath>
+    <TestExclusionListTasksAssemblyPath>$([MSBuild]::NormalizePath('$(TestExclusionListTasksDir)', 'TestExclusionListTasks.dll'))</TestExclusionListTasksAssemblyPath>
     <ILAsmToolPath Condition="'$(DotNetBuildFromSource)' == 'true'">$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'coreclr', '$(TargetOS).$(TargetArchitecture).$(Configuration)'))</ILAsmToolPath>
   </PropertyGroup>
 

eng/pipelines/common/templates/runtimes/android-runtime-and-send-to-helix.yml

@@ -13,7 +13,6 @@ parameters:
   runtimeVariant: ''
   variables: {}
   pool: ''
-  runtimeFlavorDisplayName: 'Mono'
   dependsOn: []
   #arcade-specific parameters
   condition: always()
@@ -38,8 +37,8 @@ steps:
       osGroup: ${{ parameters.osGroup }}
       osSubgroup: ${{ parameters.osSubgroup}}
       coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr
-      runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
       shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      runtimeFlavor: ${{ parameters.runtimeFlavor }}
       runtimeVariant: ${{ parameters.runtimeVariant }}
 
       ${{ if eq(variables['System.TeamProject'], 'public') }}:

eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -11,7 +11,6 @@ parameters:
   variables: {}
   pool: ''
   runtimeFlavor: 'coreclr'
-  runtimeFlavorDisplayName: 'CoreCLR'
   runtimeVariant: ''
   dependsOn: []
   dependOnEvaluatePaths: false
@@ -52,10 +51,10 @@ jobs:
     # Compute job name from template parameters
     ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
       name: '${{ parameters.runtimeFlavor }}_common_test_build_p0_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavor }} Common Pri0 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
     ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
       name: '${{ parameters.runtimeFlavor }}_common_test_build_p1_AnyOS_AnyCPU_${{ parameters.buildConfig }}'
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavor }} Common Pri1 Test Build AnyOS AnyCPU ${{ parameters.buildConfig }}'
 
     # Since the condition is being altered, merge the default with the additional conditions.
     # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -21,7 +21,6 @@ parameters:
   variables: {}
   pool: ''
   runtimeFlavor: 'coreclr'
-  runtimeFlavorDisplayName: 'CoreCLR'
   shouldContinueOnError: false
   dependsOn: []
   dependOnEvaluatePaths: false
@@ -81,10 +80,10 @@ jobs:
     # Compute job name from template parameters
     ${{ if in(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
       name: 'run_test_p0_${{ parameters.runtimeFlavor }}${{ parameters.runtimeVariant }}_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant}} Pri0 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
     ${{ if notIn(parameters.testGroup, 'innerloop', 'clrinterpreter') }}:
       name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
-      displayName: '${{ parameters.runtimeFlavorDisplayName }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavor }} ${{ parameters.runtimeVariant }} Pri1 Runtime Tests Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
 
     variables:
 
@@ -335,7 +334,7 @@ jobs:
         archType: ${{ parameters.archType }}
         osGroup: ${{ parameters.osGroup }}
         osSubgroup: ${{ parameters.osSubgroup}}
-        runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
+        runtimeFlavor: ${{ parameters.runtimeFlavor }}
         shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
         runtimeVariant: ${{ parameters.runtimeVariant }}
 

eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -24,7 +24,7 @@ parameters:
   tieringTest: ''
   longRunningGcTests: ''
   gcSimulatorTests: ''
-  runtimeFlavorDisplayName: 'CoreCLR'
+  runtimeFlavor: 'CoreCLR'
   runtimeVariant: ''
   shouldContinueOnError: false
 
@@ -57,7 +57,7 @@ steps:
       _PALTestsDir: ${{ parameters.runPALTestsDir }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
-      runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
+      RuntimeFlavor: ${{ parameters.runtimeFlavor }}
       _RuntimeVariant: ${{ parameters.runtimeVariant }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

eng/pipelines/common/templates/runtimes/wasm-runtime-and-send-to-helix.yml

@@ -18,8 +18,6 @@ parameters:
   runtimeVariant: ''
   variables: {}
   pool: ''
-  runtimeFlavor: 'mono'
-  runtimeFlavorDisplayName: 'Mono'
   dependsOn: []
   #arcade-specific parameters
   condition: always()
@@ -32,7 +30,7 @@ parameters:
 
 
 steps:
-  - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=monointerpreter /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper)
+  - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) /p:RuntimeVariant=${{ parameters.runtimeVariant }} /p:LibrariesConfiguration=${{ parameters.buildConfig }} -ci -mono os Browser wasm $(buildConfigUpper)
     displayName: Build Tests
 
   #  Send tests to Helix
@@ -44,8 +42,9 @@ steps:
       osGroup: ${{ parameters.osGroup }}
       osSubgroup: ${{ parameters.osSubgroup}}
       coreClrRepoRoot: $(Build.SourcesDirectory)/src/coreclr
-      runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
       shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
+      runtimeFlavor: ${{ parameters.runtimeFlavor }}
+      runtimeVariant: ${{ parameters.runtimeVariant }}
 
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
         creator: $(Build.DefinitionName)

eng/pipelines/common/templates/wasm-runtime-tests.yml

@@ -28,6 +28,7 @@ jobs:
       testGroup: innerloop
       isExtraPlatforms: ${{ parameters.isExtraPlatformsBuild }}
       nameSuffix: AllSubsets_Mono_RuntimeTests
+      runtimeVariant: monointerpreter
       buildArgs: -s mono+libs -c $(_BuildConfig)
       timeoutInMinutes: 180
       condition: >-

eng/pipelines/common/xplat-setup.yml

@@ -14,11 +14,6 @@ parameters:
 jobs:
 - template: ${{ coalesce(parameters.helixQueuesTemplate, parameters.jobTemplate) }}
   parameters:
-    ${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}:
-      runtimeFlavorDisplayName: 'CoreCLR'
-    ${{ if eq(parameters.jobParameters.runtimeFlavor, 'mono') }}:
-      runtimeFlavorDisplayName: 'Mono'
-
     shouldContinueOnError: ${{ and(endsWith(variables['Build.DefinitionName'], 'staging'), eq(variables['Build.Reason'], 'PullRequest')) }}
 
     # keep in sync with /eng/pipelines/common/variables.yml

eng/pipelines/coreclr/templates/helix-queues-setup.yml

@@ -7,7 +7,6 @@ parameters:
   container: ''
   pool: ''
   platform: ''
-  runtimeFlavorDisplayName: ''
   shouldContinueOnError: false
   dependOnEvaluatePaths: false
   jobParameters: {}
@@ -23,8 +22,7 @@ jobs:
     pool: ${{ parameters.pool }}
     platform: ${{ parameters.platform }}
     shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
-    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}}
-    runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
+    dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }}
     helixQueues:
 
     # iOS/tvOS simulator x64/x86

eng/pipelines/libraries/helix-queues-setup.yml

@@ -7,7 +7,6 @@ parameters:
   container: ''
   pool: ''
   platform: ''
-  runtimeFlavorDisplayName: ''
   shouldContinueOnError: false
   dependOnEvaluatePaths: false
   jobParameters: {}
@@ -24,7 +23,6 @@ jobs:
     platform: ${{ parameters.platform }}
     shouldContinueOnError: ${{ parameters.shouldContinueOnError }}
     dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths}}
-    runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
     helixQueues:
 
     # Linux arm

src/tasks/AndroidAppBuilder/ApkBuilder.cs

@@ -463,7 +463,7 @@ public ApkBuilder(TaskLoggingHelper logger)
         // 4. Align APK
 
         string alignedApk = Path.Combine(OutputDir, "bin", $"{ProjectName}.apk");
-        Utils.RunProcess(logger, zipalign, $"-v 4 {apkFile} {alignedApk}", workingDir: OutputDir);
+        AlignApk(apkFile, alignedApk, zipalign);
         // we don't need the unaligned one any more
         File.Delete(apkFile);
 
@@ -475,6 +475,11 @@ public ApkBuilder(TaskLoggingHelper logger)
         return (alignedApk, packageId);
     }
 
+    private void AlignApk(string unalignedApkPath, string apkOutPath, string zipalign)
+    {
+        Utils.RunProcess(logger, zipalign, $"-v 4 {unalignedApkPath} {apkOutPath}", workingDir: OutputDir);
+    }
+
     private void SignApk(string apkPath, string apksigner)
     {
         string defaultKey = Path.Combine(OutputDir, "debug.keystore");
@@ -495,6 +500,31 @@ private void SignApk(string apkPath, string apksigner)
             $"--ks-pass pass:android --key-pass pass:android {apkPath}", workingDir: OutputDir);
     }
 
+    public void ZipAndSignApk(string apkPath)
+    {
+        if (string.IsNullOrEmpty(AndroidSdk))
+            AndroidSdk = Environment.GetEnvironmentVariable("ANDROID_SDK_ROOT");
+
+        if (string.IsNullOrEmpty(AndroidSdk) || !Directory.Exists(AndroidSdk))
+            throw new ArgumentException($"Android SDK='{AndroidSdk}' was not found or incorrect (can be set via ANDROID_SDK_ROOT envvar).");
+
+        if (string.IsNullOrEmpty(BuildToolsVersion))
+            BuildToolsVersion = GetLatestBuildTools(AndroidSdk);
+
+        if (string.IsNullOrEmpty(MinApiLevel))
+            MinApiLevel = DefaultMinApiLevel;
+
+        string buildToolsFolder = Path.Combine(AndroidSdk, "build-tools", BuildToolsVersion);
+        string zipalign = Path.Combine(buildToolsFolder, "zipalign");
+        string apksigner = Path.Combine(buildToolsFolder, "apksigner");
+
+        string alignedApkPath = $"{apkPath}.aligned";
+        AlignApk(apkPath, alignedApkPath, zipalign);
+        logger.LogMessage(MessageImportance.High, $"\nMoving '{alignedApkPath}' to '{apkPath}'.\n");
+        File.Move(alignedApkPath, apkPath, overwrite: true);
+        SignApk(apkPath, apksigner);
+    }
+
     public void ReplaceFileInApk(string file)
     {
         if (string.IsNullOrEmpty(AndroidSdk))

src/tasks/TestExclusionListTasks/PatchExclusionListInApks.cs

@@ -0,0 +1,56 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+namespace TestExclusionListTasks;
+
+public class PatchExclusionListInApks : Task
+{
+    [Required]
+    public ITaskItem[]? ApkPaths { get; set; }
+
+    [Required]
+    public ITaskItem[]? ExcludedTests { get; set; }
+
+    public string? AndroidSdk { get; set; }
+
+    public string? MinApiLevel { get; set; }
+
+    public string? BuildToolsVersion { get; set; }
+
+    public string? KeyStorePath { get; set; }
+
+    public override bool Execute()
+    {
+        var apkBuilder = new ApkBuilder(Log);
+        apkBuilder.AndroidSdk = AndroidSdk;
+        apkBuilder.MinApiLevel = MinApiLevel;
+        apkBuilder.BuildToolsVersion = BuildToolsVersion;
+        apkBuilder.KeyStorePath = KeyStorePath;
+
+        string testExclusionList = string.Join(
+            '\n',
+            (ExcludedTests ?? Enumerable.Empty<ITaskItem>()).Select(t => t.ItemSpec));
+        foreach (ITaskItem apk in ApkPaths ?? Enumerable.Empty<ITaskItem>())
+        {
+            string apkPath = apk.GetMetadata("FullPath")!;
+            apkBuilder.OutputDir = Path.GetDirectoryName(apkPath)!;
+            using (ZipArchive apkArchive = ZipFile.Open(apkPath, ZipArchiveMode.Update))
+            {
+                ZipArchiveEntry assetsZipEntry = apkArchive.GetEntry("assets/assets.zip")!;
+                using ZipArchive assetsArchive = new ZipArchive(assetsZipEntry.Open(), ZipArchiveMode.Update);
+                ZipArchiveEntry testExclusionListEntry = assetsArchive.GetEntry("TestExclusionList.txt")!;
+                using StreamWriter textExclusionListWriter = new StreamWriter(testExclusionListEntry.Open());
+                textExclusionListWriter.WriteLine(testExclusionList);
+            }
+            apkBuilder.ZipAndSignApk(apkPath);
+        }
+        return true;
+    }
+}

src/tasks/TestExclusionListTasks/TestExclusionListTasks.csproj

@@ -0,0 +1,19 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>$(TargetFrameworkForNETCoreTasks)</TargetFramework>
+    <OutputType>Library</OutputType>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+    <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
+    <Nullable>enable</Nullable>
+    <NoWarn>$(NoWarn),CA1050</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="PatchExclusionListInApks.cs" />
+    <Compile Include="../AndroidAppBuilder/ApkBuilder.cs" />
+    <Compile Include="../Common/Utils.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Build.Tasks.Core" Version="$(MicrosoftBuildTasksCoreVersion)" />
+  </ItemGroup>
+</Project>

src/tests/Common/CoreCLRTestLibrary/OutOfProcessTest.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System;
@@ -42,6 +42,12 @@ static OutOfProcessTest()
             }
         }
 
+        public static bool OutOfProcessTestsSupported =>
+            !OperatingSystem.IsIOS()
+            && !OperatingSystem.IsTvOS()
+            && !OperatingSystem.IsAndroid()
+            && !OperatingSystem.IsBrowser();
+
         public static void RunOutOfProcessTest(string basePath, string assemblyPath)
         {
             int ret = -100;
@@ -65,6 +71,12 @@ public static void RunOutOfProcessTest(string basePath, string assemblyPath)
                     testExecutable = Path.Combine(baseDir, Path.ChangeExtension(assemblyPath.Replace("\\", "/"), ".sh"));
                 }
 
+                if (!File.Exists(testExecutable))
+                {
+                    // Skip platform-specific test when running on the excluded platform
+                    return;
+                }
+
                 System.IO.Directory.CreateDirectory(outputDir);
 
                 ret = wrapper.RunTest(testExecutable, outputFile, errorFile, Assembly.GetEntryAssembly()!.FullName!, testBinaryBase, outputDir);

src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs

@@ -16,6 +16,11 @@ public static async Task<int> RunTests(
         string? filter,
         HashSet<string> testExclusionList)
     {
+        // If an exclusion list is passed as a filter, treat it as though no filter is provided here.
+        if (filter?.StartsWith("--exclusion-list=") == true)
+        {
+            filter = null;
+        }
         ApplicationEntryPoint? entryPoint = null;
         if (OperatingSystem.IsAndroid())
         {

src/tests/Common/XUnitWrapperGenerator/ITestInfo.cs

@@ -249,7 +249,12 @@ public OutOfProcessTest(string displayName, string relativeAssemblyPath)
         Method = displayName;
         DisplayNameForFiltering = displayName;
         TestNameExpression = $"@\"{displayName}\"";
-        ExecutionStatement = $@"TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");";
+        ExecutionStatement = $@"
+if (TestLibrary.OutOfProcessTest.OutOfProcessTestsSupported)
+{{
+TestLibrary.OutOfProcessTest.RunOutOfProcessTest(typeof(Program).Assembly.Location, @""{relativeAssemblyPath}"");
+}}
+";
     }
 
     public string TestNameExpression { get; }