[ci] Add test job for .NET APK instrumentation (dotnet#5862)

The Mono.Android.NET_Tests suite has been moved to a new smoke test job. Additional test configurations have been added for the interpreter and the .aab package format. Timing data for these tests has been moved to a new .csv file that will be processed separately from the non .NET 6 Mono.Android_Tests variants. Additionally, .apk / .aab artifact uploading will now only occur when a previous test step has failed. The xaprepare `Required` scenario has been removed from the apk test jobs as they should only rely on preparing Java.Interop, which can be done separately.
grendello · Apr 27, 2021 · af7f7f5 · af7f7f5
1 parent a916ee0
commit af7f7f5
Show file tree

Hide file tree

Showing 9 changed files with 239 additions and 120 deletions.
diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml
diff --git a/build-tools/automation/yaml-templates/apk-instrumentation.yaml b/build-tools/automation/yaml-templates/apk-instrumentation.yaml
@@ -6,31 +6,43 @@ parameters:
   extraBuildArgs: ""
   testResultsFormat: NUnit
   packageType: Apk
-  artifactName: ""
+  artifactSource: ""
   artifactFolder: ""
+  useDotNet: false
   condition: succeeded()
 
 steps:
-- task: MSBuild@1
-  displayName: run ${{ parameters.testName }}
-  inputs:
-    solution: ${{ parameters.project }}
-    configuration: ${{ parameters.configuration }}
-    msbuildArguments: >-
-      /restore
-      /t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
-      /bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog
-      ${{ parameters.extraBuildArgs }}
-  condition: ${{ parameters.condition }}
-  continueOnError: true
+- ${{ if eq(parameters.useDotNet, false) }}:
+  - task: MSBuild@1
+    displayName: run ${{ parameters.testName }}
+    inputs:
+      solution: ${{ parameters.project }}
+      configuration: ${{ parameters.configuration }}
+      msbuildArguments: >-
+        /restore
+        /t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
+        /bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog
+        ${{ parameters.extraBuildArgs }}
+    condition: ${{ parameters.condition }}
+    continueOnError: true
+
+- ${{ if eq(parameters.useDotNet, true) }}:
+  - template: run-dotnet-preview.yaml
+    parameters:
+      displayName: run ${{ parameters.testName }}
+      project: ${{ parameters.project }}
+      arguments: >-
+        -t:AcquireAndroidTarget,SignAndroidPackage,DeployTest${{ parameters.packageType }}s,CheckAndRecordApkSizes,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures
+        -bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog
+        -v:n -c ${{ parameters.configuration }} ${{ parameters.extraBuildArgs }}
+      condition: ${{ parameters.condition }}
 
 - script: >
-    SOURCE="$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/${{ parameters.artifactName }}" &&
     DEST="$(Build.ArtifactStagingDirectory)/${{ parameters.artifactFolder }}/" &&
     mkdir -p "$DEST" &&
-    cp "$SOURCE" "$DEST"
+    cp "${{ parameters.artifactSource }}" "$DEST"
   displayName: copy apk/aab
-  condition: ${{ parameters.condition }}
+  condition: eq(variables['agent.jobstatus'], 'SucceededWithIssues')
   continueOnError: true
 
 - task: PublishTestResults@2

diff --git a/build-tools/automation/yaml-templates/run-dotnet-preview.yaml b/build-tools/automation/yaml-templates/run-dotnet-preview.yaml
@@ -0,0 +1,28 @@
+parameters:
+  command: build
+  project: ''
+  arguments: ''
+  useExitCodeForErrors: false
+  displayName: run dotnet preview
+  condition: succeeded()
+  continueOnError: true
+
+steps:
+- powershell: |
+    if ([Environment]::OSVersion.Platform -eq "Unix") {
+        $dotnetPath = "$env:HOME/Library/Android/dotnet/dotnet"
+        if (-not (Test-Path $dotnetPath)) {
+            $dotnetPath = "$env:HOME/android-toolchain/dotnet/dotnet"
+        }
+    } else {
+        $dotnetPath = "$env:USERPROFILE\android-toolchain\dotnet\dotnet.exe"
+    }
+    & $dotnetPath ${{ parameters.command }} ${{ parameters.project }} ${{ parameters.arguments }}
+    if ([System.Convert]::ToBoolean("${{ parameters.useExitCodeForErrors }}") -and $LASTEXITCODE -ne 0) {
+        Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)."
+        Write-Host "##vso[task.complete result=Failed;]"
+        exit 0
+    }
+  displayName: ${{ parameters.displayName }}
+  condition: ${{ parameters.condition }}
+  continueOnError: ${{ parameters.continueOnError }}
diff --git a/build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-device-tests.yaml
@@ -16,8 +16,6 @@ jobs:
     cancelTimeoutInMinutes: 5
     workspace:
       clean: all
-    variables:
-      UseDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
     steps:
     - template: setup-test-environment.yaml
       parameters:
@@ -57,7 +55,7 @@ jobs:
 
     - template: run-nunit-tests.yaml
       parameters:
-        useDotNet: $(UseDotNet)
+        useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
         testRunTitle: MSBuildDeviceIntegration On Device - macOS - ${{ parameters.job_suffix }}
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/MSBuildDeviceIntegration/${{ parameters.target_framework }}/MSBuildDeviceIntegration.dll
         nunitConsoleExtraArgs: --where "cat != SystemApplication && cat != TimeZoneInfo && cat != SmokeTests ${{ parameters.nunit_categories }}"

diff --git a/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-mac-tests.yaml
@@ -18,8 +18,6 @@ jobs:
     cancelTimeoutInMinutes: 5
     workspace:
       clean: all
-    variables:
-      UseDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
     steps:
     - template: setup-test-environment.yaml
       parameters:
@@ -38,7 +36,7 @@ jobs:
 
     - template: run-nunit-tests.yaml
       parameters:
-        useDotNet: $(UseDotNet)
+        useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
         testRunTitle: Xamarin.Android.Build.Tests - macOS-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
         testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/${{ parameters.target_framework }}/Xamarin.Android.Build.Tests.dll
         nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} ${{ parameters.nunit_categories }}"
@@ -49,7 +47,7 @@ jobs:
     - ${{ if eq(parameters.run_extra_tests, true) }}:
       - template: run-nunit-tests.yaml
         parameters:
-          useDotNet: $(UseDotNet)
+          useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
           testRunTitle: Xamarin.Android.Build.Tests - macOS - No Node
           testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/${{ parameters.target_framework }}/Xamarin.Android.Build.Tests.dll
           nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
@@ -60,7 +58,7 @@ jobs:
     - ${{ if eq(parameters.run_extra_tests, true) }}:
       - template: run-nunit-tests.yaml
         parameters:
-          useDotNet: $(UseDotNet)
+          useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
           testRunTitle: Xamarin.Android.Tools.Aidl-Tests - macOS
           testAssembly: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration)/${{ parameters.target_framework }}/Xamarin.Android.Tools.Aidl-Tests.dll
           testResultsFile: TestResult-Aidl-Tests-macOS-$(XA.Build.Configuration).xml

diff --git a/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml b/build-tools/automation/yaml-templates/run-msbuild-win-tests.yaml
@@ -17,7 +17,6 @@ jobs:
     cancelTimeoutInMinutes: 5
     variables:
       VSINSTALLDIR: C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise
-      UseDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
     steps:
     - script: netsh int ipv4 set global sourceroutingbehavior=drop
 
@@ -45,7 +44,7 @@ jobs:
     # Using all available cores seems to occasionally bog down our machines and cause parallel test execution to slow down dramatically.
     - template: run-nunit-tests.yaml
       parameters:
-        useDotNet: $(UseDotNet)
+        useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
         testRunTitle: Xamarin.Android.Build.Tests - Windows-${{ parameters.node_id }} - ${{ parameters.job_suffix }}
         testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\${{ parameters.target_framework }}\Xamarin.Android.Build.Tests.dll
         nunitConsoleExtraArgs: --where "cat == Node-${{ parameters.node_id }} ${{ parameters.nunit_categories }}"
@@ -56,7 +55,7 @@ jobs:
     - ${{ if eq(parameters.run_extra_tests, true) }}:
       - template: run-nunit-tests.yaml
         parameters:
-          useDotNet: $(UseDotNet)
+          useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
           testRunTitle: Xamarin.Android.Build.Tests - Windows - No Node
           testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\${{ parameters.target_framework }}\Xamarin.Android.Build.Tests.dll
           nunitConsoleExtraArgs: --where "cat != Node-1 && cat != Node-2 && cat != Node-3"
@@ -67,7 +66,7 @@ jobs:
     - ${{ if eq(parameters.run_extra_tests, true) }}:
       - template: run-nunit-tests.yaml
         parameters:
-          useDotNet: $(UseDotNet)
+          useDotNet: ${{ eq(parameters.target_framework, 'netcoreapp3.1') }}
           testRunTitle: Xamarin.Android.Tools.Aidl-Tests - Windows
           testAssembly: $(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\${{ parameters.target_framework }}\Xamarin.Android.Tools.Aidl-Tests.dll
           testResultsFile: TestResult-Aidl-Tests-Windows-$(XA.Build.Configuration).xml

diff --git a/build-tools/automation/yaml-templates/run-nunit-tests.yaml b/build-tools/automation/yaml-templates/run-nunit-tests.yaml
@@ -1,7 +1,7 @@
 parameters:
   nunitConsole: $(System.DefaultWorkingDirectory)/build-tools/scripts/nunit3-console
   testRunTitle: Xamarin Android Tests
-  testAssembly: []
+  testAssembly: ''
   testResultsFile: TestResult.xml
   nunitConsoleExtraArgs: ''
   dotNetTestExtraArgs: ''
@@ -10,36 +10,35 @@ parameters:
   condition: succeeded()
 
 steps:
-- powershell: |
-    if ([Environment]::OSVersion.Platform -eq "Unix") {
-        if ("${{ parameters.useDotNet }}" -eq $true) {
-            Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
-            $dotnetPath = "$env:HOME/Library/Android/dotnet/dotnet"
-            if (-not (Test-Path $dotnetPath)) {
-                $dotnetPath = "$env:HOME/android-toolchain/dotnet/dotnet"
-            }
-            & $dotnetPath test ${{ parameters.testAssembly }} --results-directory . --logger "trx;LogFileName=${{ parameters.testResultsFile }}" ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
-        } else {
-            Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
-            & ${{ parameters.nunitConsole }} ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
-        }
-    } else {
-        if ("${{ parameters.useDotNet }}" -eq $true) {
-           Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
-           & $env:USERPROFILE\android-toolchain\dotnet\dotnet.exe test ${{ parameters.testAssembly }} --results-directory . --logger "trx;LogFileName=${{ parameters.testResultsFile }}" ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
-        } else {
-           Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
-           & cmd /c '${{ parameters.nunitConsole }}.cmd' ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
-        }
-    }
-    if ($LASTEXITCODE -ne 0) {
-        Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)."
-        Write-Host "##vso[task.complete result=Failed;]"
-        exit 0
-    }
-  displayName: run ${{ parameters.testRunTitle }}
-  condition: ${{ parameters.condition }}
-  continueOnError: true
+- ${{ if eq(parameters.useDotNet, false) }}:
+  - powershell: |
+      Write-Host '##vso[task.setvariable variable=TestResultsFormat]NUnit'
+      if ([Environment]::OSVersion.Platform -eq "Unix") {
+          & ${{ parameters.nunitConsole }} ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
+      } else {
+          & cmd /c '${{ parameters.nunitConsole }}.cmd' ${{ parameters.testAssembly }} --result ${{ parameters.testResultsFile }} --workers=${{ parameters.workers }} ${{ parameters.nunitConsoleExtraArgs }}
+      }
+      if ($LASTEXITCODE -ne 0) {
+          Write-Host "##vso[task.logissue type=error]Test suite had $LASTEXITCODE failure(s)."
+          Write-Host "##vso[task.complete result=Failed;]"
+          exit 0
+      }
+    displayName: run ${{ parameters.testRunTitle }}
+    condition: ${{ parameters.condition }}
+    continueOnError: true
+
+- ${{ if eq(parameters.useDotNet, true) }}:
+  - powershell: Write-Host '##vso[task.setvariable variable=TestResultsFormat]VSTest'
+  - template: run-dotnet-preview.yaml
+    parameters:
+      command: test
+      project: ${{ parameters.testAssembly }}
+      useExitCodeForErrors: true
+      arguments: >-
+        --results-directory . --logger "trx;LogFileName=${{ parameters.testResultsFile }}"
+        ${{ parameters.dotNetTestExtraArgs }} -- NUnit.NumberOfTestWorkers=${{ parameters.workers }}
+      displayName: run ${{ parameters.testRunTitle }}
+      condition: ${{ parameters.condition }}
 
 - template: kill-processes.yaml
 

diff --git a/tests/Mono.Android-Tests/Directory.Build.targets b/tests/Mono.Android-Tests/Directory.Build.targets
@@ -59,14 +59,4 @@
     />
   </Target>
 
-  <Target Name="RunDotNetInstrumentation"
-      DependsOnTargets="GenerateNuGetConfig" >
-    <ItemGroup>
-      <_RuntimeTestArgs Include="-p:Configuration=$(Configuration)" />
-      <_RuntimeTestArgs Include="-t:AcquireAndroidTarget,SignAndroidPackage,DeployTestApks,RunTestApks,UndeployTestApks,RenameApkTestCases,ReportComponentFailures" />
-      <_RuntimeTestArgs Include="-bl:$(XamarinAndroidSourcePath)bin\Test$(Configuration)\run-Mono.Android.NET-Tests.binlog" />
-    </ItemGroup>
-    <Exec Command="$(DotNetPreviewTool) build /v:n @(_RuntimeTestArgs, ' ') &quot;$(MSBuildThisFileDirectory)Runtime-Microsoft.Android.Sdk\Mono.Android.NET-Tests.csproj&quot;" />
-  </Target>
-
 </Project>
diff --git a/tests/Mono.Android-Tests/Runtime-Microsoft.Android.Sdk/Mono.Android.NET-Tests.csproj b/tests/Mono.Android-Tests/Runtime-Microsoft.Android.Sdk/Mono.Android.NET-Tests.csproj
@@ -18,7 +18,7 @@
     <EnableDefaultAndroidResourceItems>false</EnableDefaultAndroidResourceItems>
     <EnableDefaultAndroidAssetItems>false</EnableDefaultAndroidAssetItems>
     <_MonoAndroidTestPackage>Mono.Android.NET_Tests</_MonoAndroidTestPackage>
-    <PlotDataLabelSuffix>-NET6</PlotDataLabelSuffix>
+    <PlotDataLabelSuffix>-$(TestsFlavor)NET6</PlotDataLabelSuffix>
     <!-- TODO: Fix excluded tests -->
     <ExcludeCategories>DotNetIgnore</ExcludeCategories>
   </PropertyGroup>
@@ -34,20 +34,32 @@
     <Compile Remove="..\Mono.Data.Sqlite\SqliteTests.cs" />
   </ItemGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition=" '$(AndroidPackageFormat)' != 'aab' ">
     <TestApk Include="$(OutputPath)$(_MonoAndroidTestPackage)-Signed.apk">
       <Package>$(_MonoAndroidTestPackage)</Package>
       <InstrumentationType>xamarin.android.runtimetests.NUnitInstrumentation</InstrumentationType>
       <ResultsPath>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage).xml</ResultsPath>
       <TimingDefinitionsFilename>$(XamarinAndroidSourcePath)build-tools\scripts\TimingDefinitions.txt</TimingDefinitionsFilename>
-      <!-- Use default package name to write all Mono.Android-Tests* timing data to the same .csv -->
-      <TimingResultsFilename>$(XamarinAndroidSourcePath)TestResult-Mono.Android_Tests-times.csv</TimingResultsFilename>
+      <TimingResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-times.csv</TimingResultsFilename>
       <ApkSizesInputFilename>apk-sizes-$(_MonoAndroidTestPackage)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
       <ApkSizesDefinitionFilename>$(XamarinAndroidSourcePath)build-tools\scripts\ApkSizesDefinitions.txt</ApkSizesDefinitionFilename>
       <ApkSizesResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-values-$(Configuration).csv</ApkSizesResultsFilename>
     </TestApk>
   </ItemGroup>
 
+  <ItemGroup Condition=" '$(AndroidPackageFormat)' == 'aab' ">
+    <TestAab Include="$(OutputPath)$(_MonoAndroidTestPackage)-Signed.aab">
+      <Package>$(_MonoAndroidTestPackage)</Package>
+      <InstrumentationType>xamarin.android.runtimetests.NUnitInstrumentation</InstrumentationType>
+      <ResultsPath>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage).xml</ResultsPath>
+      <TimingDefinitionsFilename>$(XamarinAndroidSourcePath)build-tools\scripts\TimingDefinitions.txt</TimingDefinitionsFilename>
+      <TimingResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-times.csv</TimingResultsFilename>
+      <ApkSizesInputFilename>apk-sizes-$(_MonoAndroidTestPackage)-$(Configuration)$(TestsFlavor).txt</ApkSizesInputFilename>
+      <ApkSizesDefinitionFilename>$(XamarinAndroidSourcePath)build-tools\scripts\ApkSizesDefinitions.txt</ApkSizesDefinitionFilename>
+      <ApkSizesResultsFilename>$(XamarinAndroidSourcePath)TestResult-$(_MonoAndroidTestPackage)-values-$(Configuration).csv</ApkSizesResultsFilename>
+    </TestAab>
+  </ItemGroup>
+
   <Import Project="$(XamarinAndroidSourcePath)build-tools\scripts\TestApks.targets" />
 
 </Project>