Enable CI to run CoreCLR tests on Mono

eng/pipelines/coreclr/templates/build-test-job.yml → eng/pipelines/common/templates/runtimes/build-test-job.yml

@@ -5,6 +5,7 @@ parameters:
   osSubgroup: ''
   container: ''
   testGroup: ''
+  liveRuntimeBuildConfig: ''
 
   # When set to a non-empty value (Debug / Release), it determines libraries
   # build configuration to use for the tests. Setting this property implies
@@ -18,6 +19,7 @@ parameters:
   stagedBuild: false
   variables: {}
   pool: ''
+  runtimeFlavorDisplayName: 'CoreCLR'
 
 ### Build managed test components (native components are getting built as part
 ### of the the product build job).
@@ -27,8 +29,9 @@ parameters:
 ### over to its reference assembly we should be able to remove this dependency and
 ### run managed test builds in parallel with the product build job.
 
+
 jobs:
-- template: xplat-pipeline-job.yml
+- template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -40,21 +43,25 @@ jobs:
     testGroup: ${{ parameters.testGroup }}
     stagedBuild: ${{ parameters.stagedBuild }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
-    variables: ${{ parameters.variables }}
+    variables:
+    - runtimeFlavorArgs: ''
+    - ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
+      - runtimeFlavorArgs: '-excludemonofailures'
+    - ${{ each variable in parameters.variables }}:
+      - ${{insert}}: ${{ variable }}	
     pool: ${{ parameters.pool }}
 
     # Test jobs should continue on error for internal builds
     ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       continueOnError: true
 
-    # Compute job name from template parameters
     ${{ if eq(parameters.testGroup, 'innerloop') }}:
-      name: 'coreclr_test_build_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
-      displayName: 'CoreCLR Pri0 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      name: '${{ parameters.runtimeFlavor }}_test_build_p0_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Pri0 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
 
     ${{ if ne(parameters.testGroup, 'innerloop') }}:
-      name: 'coreclr_test_build_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
-      displayName: 'CoreCLR Pri1 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      name: '${{ parameters.runtimeFlavor }}_test_build_p1_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'      
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Pri1 Test Build ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
 
     # Since the condition is being altered, merge the default with the additional conditions.
     # See https://docs.microsoft.com/azure/devops/pipelines/process/conditions
@@ -65,7 +72,7 @@ jobs:
     # by switching over to using reference assembly.
     ${{ if ne(parameters.stagedBuild, true) }}:
       dependsOn:
-      - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+      - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, coalesce(parameters.liveRuntimeBuildConfig, parameters.buildConfig)) }}
       - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
         - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
 
@@ -98,6 +105,15 @@ jobs:
           artifactName: '$(librariesBuildArtifactName)'
           displayName: 'live-built libraries'
 
+    - ${{ if eq(parameters.runtimeFlavor, 'mono') }}:  
+      # We need to explictly download CoreCLR for Mono because the CoreCLR tests depend on it
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(coreClrProductRootFolderPath)
+          artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)'
+          artifactName: '$(coreClrProductArtifactName)'
+          displayName: 'CoreCLR product build for Mono'
+
 
     # Download product binaries directory
     - template: /eng/pipelines/common/download-artifact-step.yml
@@ -109,7 +125,7 @@ jobs:
 
 
     # Build managed test components
-    - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(runtimeFlavorArgs) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
       displayName: Build managed test components
 
 

eng/pipelines/coreclr/templates/run-test-job.yml → eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -21,14 +21,15 @@ parameters:
   runInUnloadableContext: false
   variables: {}
   pool: ''
+  runtimeFlavorDisplayName: 'CoreCLR'
 
 ### Test run job
 
 ### Each test run job depends on a corresponding test build job with the same
 ### buildConfig and archType.
 
 jobs:
-- template: xplat-pipeline-job.yml
+- template: /eng/pipelines/${{ parameters.runtimeFlavor }}/templates/xplat-pipeline-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -53,22 +54,22 @@ jobs:
     dependsOn:
     - ${{ if ne(parameters.corefxTests, true) }}:
       - ${{ if eq(parameters.testGroup, 'innerloop') }}:
-        - 'coreclr_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - '${{ parameters.runtimeFlavor }}_test_build_p0_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
       - ${{ if ne(parameters.testGroup, 'innerloop') }}:
-        - 'coreclr_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
+        - '${{ parameters.runtimeFlavor }}_test_build_p1_${{ parameters.managedTestBuildOsGroup }}${{ parameters.managedTestBuildOsSubgroup }}_${{ parameters.archType }}_${{parameters.buildConfig }}'
     - ${{ if ne(parameters.stagedBuild, true) }}:
-      - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+      - ${{ format('{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeFlavor, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
       - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
         - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }}
 
     # Compute job name from template parameters
     ${{ if eq(parameters.testGroup, 'innerloop') }}:
-      name: 'run_test_p0_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
-      displayName: 'CoreCLR Pri0 Test Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      name: 'run_test_p0_${{ parameters.runtimeFlavor }}_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavorDisplayName }}  Pri0 Test Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
 
     ${{ if ne(parameters.testGroup, 'innerloop') }}:
       name: 'run_test_p1_${{ parameters.displayNameArgs }}_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}'
-      displayName: 'CoreCLR Pri1 Test Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
+      displayName: '${{ parameters.runtimeFlavorDisplayName }} Pri1 Test Run ${{ parameters.displayNameArgs }} ${{ parameters.osGroup }}${{ parameters.osSubgroup }} ${{ parameters.archType }} ${{ parameters.buildConfig }}'
 
     variables:
     - name: testhostArg
@@ -77,6 +78,13 @@ jobs:
       - name: testhostArg
         value: 'buildtesthostonly'
 
+    - name: runtimeFlavorArgs
+    - value: ''
+
+    - ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
+      - name: runtimeFlavorArgs
+      - value: 'mono'
+
     - name: crossgenArg
       value: ''
     - name: LogNamePrefix
@@ -198,6 +206,22 @@ jobs:
         displayName: 'product build'
 
 
+    - ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
+      # We need to explictly download CoreCLR for Mono
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: $(coreClrProductRootFolderPath)
+          artifactFileName: '$(coreClrProductArtifactName)$(archiveExtension)'
+          artifactName: '$(coreClrProductArtifactName)'
+          displayName: 'CoreCLR product download for Mono'
+
+      - script: $(_msbuildCommand)
+                $(Build.SourcesDirectory)/src/mono/mono.proj
+                /t:PatchCoreClrCoreRoot
+                /p:Configuration=$(buildConfig);BuildArchitecture=$(archType)
+        displayName: "Patch dotnet with mono"
+
+
     # Download and unzip the Microsoft.NET.Sdk.IL package needed for traversing
     # ilproj test projects during copynativeonly.
     - ${{ if ne(parameters.corefxTests, true) }}:
@@ -235,8 +259,8 @@ jobs:
         displayName: Generate test host
 
 
-    # Generate test wrappers
-    - script: $(coreClrRepoRootDir)build-test$(scriptExt) buildtestwrappersonly $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
+    # Generate test wrappers. This is the step that examines issues.targets to exclude tests.
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) buildtestwrappersonly $(runtimeFlavorArgs) $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
       displayName: Generate test wrappers
 
 
@@ -247,13 +271,14 @@ jobs:
 
 
     # Send tests to Helix
-    - template: /eng/pipelines/coreclr/templates/send-to-helix-step.yml
+    - template: /eng/pipelines/common/templates/runtimes/send-to-helix-step.yml
       parameters:
         displayName: Send tests to Helix
         buildConfig: $(buildConfigUpper)
         archType: ${{ parameters.archType }}
         osGroup: ${{ parameters.osGroup }}
         coreClrRepoRoot: $(coreClrRepoRoot)
+        runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
 
         ${{ if eq(variables['System.TeamProject'], 'public') }}:
           creator: $(Build.DefinitionName)

eng/pipelines/coreclr/templates/send-to-helix-step.yml → eng/pipelines/common/templates/runtimes/send-to-helix-step.yml

@@ -22,6 +22,7 @@ parameters:
   longRunningGcTests: ''
   gcSimulatorTests: ''
   coreClrRepoRoot: ''
+  runtimeFlavorDisplayName: 'CoreCLR'
 
 steps:
 - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
@@ -55,6 +56,7 @@ steps:
       _Scenarios: ${{ join(',', parameters.scenarios) }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
+      runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed
@@ -95,6 +97,7 @@ steps:
       _Scenarios: ${{ join(',', parameters.scenarios) }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
+      runtimeFlavorDisplayName: ${{ parameters.runtimeFlavorDisplayName }}
       ${{ if eq(parameters.publishTestResults, 'true') }}:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)
       # TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed

eng/pipelines/common/xplat-setup.yml

@@ -11,6 +11,10 @@ parameters:
 jobs:
 - template: ${{ coalesce(parameters.helixQueuesTemplate, parameters.jobTemplate) }}
   parameters:
+    ${{ if eq(parameters.runtimeFlavor, 'coreclr') }}:
+      runtimeFlavorDisplayName: 'CoreCLR'
+    ${{ if eq(parameters.runtimeFlavor, 'mono') }}:
+      runtimeFlavorDisplayName: 'Mono'
     variables:
       # Disable component governance in our CI builds. These builds are not shipping nor
       # are they a service. Also the component governance jobs issue lots of inconsequential

eng/pipelines/coreclr/ci.yml

@@ -94,7 +94,7 @@ jobs:
 #
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: checked
     platforms:
     - Linux_arm
@@ -113,7 +113,7 @@ jobs:
 #
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platformGroup: all
     helixQueueGroup: ci
@@ -127,7 +127,7 @@ jobs:
 #
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platforms:
     - Linux_arm64

eng/pipelines/coreclr/corefx-jitstress.yml

@@ -33,7 +33,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platforms:
     # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this.

eng/pipelines/coreclr/corefx-jitstress2-jitstressregs.yml

@@ -33,7 +33,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platforms:
     # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this.

eng/pipelines/coreclr/corefx-jitstressregs.yml

@@ -33,7 +33,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platforms:
     # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this.

eng/pipelines/coreclr/corefx.yml

@@ -33,7 +33,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platforms:
     # TODO: add Windows_NT_arm64, when we have hardware available. Note: platform-matrix.yml needs to enable a Helix queue for this.

eng/pipelines/coreclr/crossgen2-outerloop.yml

@@ -40,7 +40,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     buildConfig: checked
     platforms:

eng/pipelines/coreclr/crossgen2.yml

@@ -29,7 +29,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: checked
     platforms:
     - OSX_x64
@@ -40,7 +40,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
     buildConfig: checked
     platforms:

eng/pipelines/coreclr/gc-longrunning.yml

@@ -43,7 +43,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: release
     platforms:
     - Linux_x64

eng/pipelines/coreclr/jitstress-isas-x86.yml

@@ -30,7 +30,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/build-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     buildConfig: checked
     platforms:
     - OSX_x64
@@ -42,7 +42,7 @@ jobs:
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: checked
     platforms:
     - Linux_x64

eng/pipelines/coreclr/release-tests.yml

@@ -44,7 +44,7 @@ jobs:
 #
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: release
     platformGroup: all
     helixQueueGroup: ci
@@ -58,7 +58,7 @@ jobs:
 #
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
-    jobTemplate: /eng/pipelines/coreclr/templates/run-test-job.yml
+    jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml
     buildConfig: release
     platformGroup: all
     helixQueueGroup: ci

eng/pipelines/coreclr/templates/test-job.yml

@@ -27,7 +27,7 @@ parameters:
 
 jobs:
 - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.osSubgroup, parameters.managedTestBuildOsSubgroup), eq(parameters.osGroup, parameters.managedTestBuildOsGroup)) }}:
-  - template: /eng/pipelines/coreclr/templates/build-test-job.yml
+  - template: /eng/pipelines/common/templates/runtimes/build-test-job.yml
     parameters:
       buildConfig: ${{ parameters.buildConfig }}
       liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
@@ -42,7 +42,7 @@ jobs:
       variables: ${{ parameters.variables }}
       pool: ${{ parameters.pool }}
 
-- template: /eng/pipelines/coreclr/templates/run-test-job.yml
+- template: /eng/pipelines/common/templates/runtimes/run-test-job.yml
   parameters:
     buildConfig: ${{ parameters.buildConfig }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}