Add APK test job to azure-pipelines.yaml

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/Adb.cs

@@ -43,6 +43,7 @@ protected class CommandInfo
 		public                  bool            IgnoreExitCode        { get; set; }
 		public                  int             Timeout               { get; set; } = -1;
 		public                  string[]        EnvironmentVariables  { get; set; }
+		public                  bool            WriteOutputAsMessage  { get; set; } = false;
 
 		[Required]
 		public                  string          ToolPath              { get; set; }
@@ -147,7 +148,7 @@ void OnOutput (string line, bool isStdout, CommandInfo info)
 			lock (linesLock) lines.Add (line);
 
 			if (!info.SuppressMSbuildLog) {
-				if (isStdout)
+				if (isStdout || WriteOutputAsMessage)
 					Log.LogMessage (MessageImportance.Low, line);
 				else
 					Log.LogWarning (line);

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RenameTestCases.cs

@@ -74,6 +74,7 @@ string GetFixedUpPath (string source, string testNameSuffix)
 		{
 			var destFilename = Path.GetFileNameWithoutExtension (source) +
 				(string.IsNullOrWhiteSpace (Configuration) ? "" : "-" + Configuration) +
+				(string.IsNullOrWhiteSpace (TestsFlavor) ? "" : TestsFlavor) +
 				Path.GetExtension (source);
 			var dest = Path.Combine (DestinationFolder, destFilename);
 			return dest;

build-tools/Xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks/RunInstrumentationTests.cs

@@ -14,6 +14,7 @@ public class RunInstrumentationTests : Adb
 		const                   string              TestResultsPathResult       = "INSTRUMENTATION_RESULT: nunit2-results-path=";
 		internal const          string              AdbRestartText              = "daemon not running; starting now at tcp:";
 		internal const          string              AdbCrashErrorText           = "The adb might have crashed and was restarted. ";
+		const                   string              InstrumentationExitCodeName = "INSTRUMENTATION_CODE: ";
 		const                   int                 StateRunInstrumentation     = 0;
 		const                   int                 StateGetLogcat              = 1;
 		const                   int                 StateClearLogcat            = 2;
@@ -41,6 +42,7 @@ public class RunInstrumentationTests : Adb
 		public                  string              LogLevel                    { get; set; }
 
 		int                     currentState = -1;
+		int                     instrumentationExitCode = 99;
 		string                  targetTestResultsPath;
 
 		bool                    adbRestarted;
@@ -68,6 +70,14 @@ public override bool Execute ()
 				return false;
 			}
 
+			if (instrumentationExitCode != -1) {
+				FailedToRun = Component;
+				Log.LogError (
+					$"Instrumentation for component `{Component}` did not exit successfully. " +
+					"Process crashed or test failures occurred!");
+				return false;
+			}
+
 			return !Log.HasLoggedErrors;
 		}
 
@@ -134,11 +144,15 @@ protected override void ProcessStdout (string line)
 			if (currentState != StateRunInstrumentation || String.IsNullOrEmpty (line))
 				return;
 
-			int i = line.IndexOf (TestResultsPathResult, StringComparison.OrdinalIgnoreCase);
-			if (i < 0)
-				return;
+			int testResultIndex = line.IndexOf (TestResultsPathResult, StringComparison.OrdinalIgnoreCase);
+			int exitCodeIndex = line.IndexOf (InstrumentationExitCodeName, StringComparison.OrdinalIgnoreCase);
 
-			targetTestResultsPath = line.Substring (i + TestResultsPathResult.Length).Trim ();
+			if (testResultIndex < 0 && exitCodeIndex < 0)
+				return;
+			else if (testResultIndex >= 0)
+				targetTestResultsPath = line.Substring (testResultIndex + TestResultsPathResult.Length).Trim ();
+			else if (exitCodeIndex >= 0)
+				instrumentationExitCode = int.Parse (line.Substring (exitCodeIndex + InstrumentationExitCodeName.Length).Trim ());
 		}
 	}
 }

build-tools/Xamarin.Android.Tools.BootstrapTasks/result-packaging.targets

@@ -6,8 +6,7 @@
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)Configuration.OperatingSystem.props" Condition="Exists('$(XamarinAndroidSourcePath)Configuration.OperatingSystem.props')" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)Configuration.Override.props" Condition="Exists('$(XamarinAndroidSourcePath)Configuration.Override.props')" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\XABuildConfig.cs" Condition="Exists('$(XamarinAndroidSourcePath)bin\Build$(Configuration)\XABuildConfig.cs')" />
-    <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\msbuild*.binlog" />
-    <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\bootstrap*.binlog" />
+    <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\*.binlog" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\prepare*.log" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\*.mk" />
     <_BuildStatusFiles Include="$(XamarinAndroidSourcePath)bin\Build$(Configuration)\*.projitems" />
@@ -27,7 +26,7 @@
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\*.log" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\compatibility\*" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\logcat*" />
-    <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\msbuild*.binlog*" />
+    <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\*.binlog" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\**\*.binlog" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\temp\**\*.log" />
     <_TestResultFiles Include="$(XamarinAndroidSourcePath)bin\Test$(Configuration)\EmbeddedDSO\EmbeddedDSO.build\**\*" />

build-tools/automation/azure-pipelines.yaml

@@ -9,6 +9,7 @@ trigger:
 # Global variables
 variables:
   BundleArtifactName: bundle
+  InstallerArtifactName: unsigned-installers
   AutoProvisionArgs: /p:AutoProvision=True /p:AutoProvisionUsesSudo=True /p:IgnoreMaxMonoVersion=False
   AndroidTargetAbiArgs: >-
     /p:AndroidSupportedTargetJitAbis=armeabi-v7a:arm64-v8a:x86:x86_64
@@ -122,7 +123,7 @@ stages:
     - task: PublishPipelineArtifact@0
       displayName: upload unsigned installers
       inputs:
-        artifactName: unsigned
+        artifactName: $(InstallerArtifactName)
         targetPath: bin/Build$(XA.Build.Configuration)/unsigned-installers
 
     - task: MSBuild@1
@@ -215,7 +216,6 @@ stages:
         testResultsFormat: NUnit
         testResultsFiles: TestResult-*.xml
         testRunTitle: xamarin-android
-        failTaskOnFailedTests: true
       condition: succeededOrFailed()
 
     - task: MSBuild@1
@@ -232,3 +232,206 @@ stages:
         artifactName: win-build-test-results
         targetPath: $(Build.ArtifactStagingDirectory)
       condition: always()
+
+- stage: test
+  displayName: Test
+  dependsOn: mac_build
+  jobs:
+  - job: mac_apk_tests
+    displayName: APK Instrumentation
+    pool: $(XA.Build.Mac.Pool)
+    timeoutInMinutes: 240
+    cancelTimeoutInMinutes: 5
+    workspace:
+      clean: all
+    variables:
+      ApkTestConfiguration: Release
+    steps:
+    - task: DownloadPipelineArtifact@1
+      inputs:
+        artifactName: $(InstallerArtifactName)
+        itemPattern: "*.pkg"
+        downloadPath: $(System.DefaultWorkingDirectory)
+
+    - template: yaml-templates/run-installer.yaml
+
+    - task: MSBuild@1
+      displayName: build xaprepare
+      inputs:
+        solution: build-tools/xaprepare/xaprepare.sln
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: /t:Restore,Build
+
+    - script: |
+        mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=UpdateMono --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
+        mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=Required --auto-provision=yes --auto-provision-uses-sudo=yes --no-emoji --run-mode=CI
+        mono build-tools/xaprepare/xaprepare/bin/$(ApkTestConfiguration)/xaprepare.exe --s=AndroidToolchain --no-emoji --run-mode=CI
+      displayName: provision dependencies
+
+    - task: NuGetCommand@2
+      displayName: nuget restore Xamarin.Android.Tools.sln
+      inputs:
+        restoreSolution: external/xamarin-android-tools/Xamarin.Android.Tools.sln
+
+    - task: MSBuild@1
+      displayName: build Xamarin.Android.Tools.BootstrapTasks.csproj
+      inputs:
+        solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/BootstrapTasks.binlog
+
+    - task: NuGetCommand@2
+      displayName: nuget restore Xamarin.Android-Tests.sln
+      inputs:
+        restoreSolution: Xamarin.Android-Tests.sln
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests-Aot
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration)-Aot.xml
+        extraBuildArgs: /p:AotAssemblies=True /p:EnableLlvm=True
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests-Profiled
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration)-Profiled.xml
+        extraBuildArgs: /p:AndroidEnableProfiledAot=true
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_Tests-Bundle
+        project: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        testResultsFiles: TestResult-Mono.Android_Tests-$(ApkTestConfiguration)-Bundle.xml
+        extraBuildArgs: /p:BundleAssemblies=true
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_TestsAppBundle
+        project: tests/Runtime-AppBundle/Mono.Android-TestsAppBundle.csproj
+        testResultsFiles: TestResult-Mono.Android_TestsAppBundle-$(ApkTestConfiguration).xml
+        packageType: Aab
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Mono.Android_TestsMultiDex
+        project: tests/Runtime-MultiDex/Mono.Android-TestsMultiDex.csproj
+        testResultsFiles: TestResult-Mono.Android_TestsMultiDex-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.JcwGen_Tests
+        project: tests/CodeGen-Binding/Xamarin.Android.JcwGen-Tests/Xamarin.Android.JcwGen-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.JcwGen_Tests-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.Locale_Tests
+        project: tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration).xml
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.Locale_Tests-Aot
+        project: tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration)-Aot.xml
+        extraBuildArgs: /p:AotAssemblies=True
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.Locale_Tests-Profiled
+        project: tests/locales/Xamarin.Android.Locale-Tests/Xamarin.Android.Locale-Tests.csproj
+        testResultsFiles: TestResult-Xamarin.Android.Locale_Tests-$(ApkTestConfiguration)-Profiled.xml
+        extraBuildArgs: /p:AndroidEnableProfiledAot=true
+
+    - template: yaml-templates/apk-instrumentation.yaml
+      parameters:
+        configuration: $(ApkTestConfiguration)
+        testName: Xamarin.Android.EmbeddedDSO_Test
+        project: tests/EmbeddedDSOs/EmbeddedDSO/EmbeddedDSO.csproj
+        testResultsFiles: TestResult-Xamarin.Android.EmbeddedDSO_Test.nunit-$(ApkTestConfiguration).xml
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf.binlog
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration-Aot
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf-Aot.binlog
+          /p:AotAssemblies=true
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration-Profiled
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf-Profiled.binlog
+          /p:AndroidEnableProfiledAot=true
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: run Xamarin.Forms-Performance-Integration-Bundle
+      inputs:
+        solution: tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTestApks,RunTestApks,ReportComponentFailures
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/XamarinFormsPerf-Bundle.binlog
+          /p:BundleAssemblies=true
+      condition: succeededOrFailed()
+
+    - task: MSBuild@1
+      displayName: shut down emulator
+      inputs:
+        solution: src/Mono.Android/Test/Mono.Android-Tests.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: >
+          /t:AcquireAndroidTarget,ReleaseAndroidTarget
+          /bl:$(System.DefaultWorkingDirectory)/bin/Test$(ApkTestConfiguration)/shutdown-emulator.binlog
+      condition: always()
+
+    - task: MSBuild@1
+      displayName: package results
+      inputs:
+        solution: build-tools/xamarin.Android.Tools.BootstrapTasks/Xamarin.Android.Tools.BootstrapTasks.csproj
+        configuration: $(ApkTestConfiguration)
+        msbuildArguments: /t:ZipBuildStatus;ZipTestResults /p:BuildStatusZipOutputPath=$(Build.ArtifactStagingDirectory) /p:TestResultZipOutputPath=$(Build.ArtifactStagingDirectory)
+      condition: always()
+
+    - task: PublishPipelineArtifact@0
+      displayName: upload artifacts
+      inputs:
+        artifactName: mac-apk-test-results
+        targetPath: $(Build.ArtifactStagingDirectory)
+      condition: always()

build-tools/automation/yaml-templates/apk-instrumentation.yaml

@@ -0,0 +1,28 @@
+parameters:
+  configuration: []
+  testName: []
+  project: []
+  testResultsFiles: []
+  extraBuildArgs: ""
+  testResultsFormat: NUnit
+  packageType: Apk
+
+steps:
+- task: MSBuild@1
+  displayName: run ${{ parameters.testName }}
+  inputs:
+    solution: ${{ parameters.project }}
+    configuration: ${{ parameters.configuration }}
+    msbuildArguments: >
+      /t:AcquireAndroidTarget,SignAndroidPackage,UndeployTestApks,DeployTest${{ parameters.packageType }}s,RunTestApks,RenameApkTestCases,ReportComponentFailures
+      /bl:$(System.DefaultWorkingDirectory)/bin/Test${{ parameters.configuration }}/run${{ parameters.testName }}.binlog
+      ${{ parameters.extraBuildArgs }}
+  condition: succeededOrFailed()
+
+- task: PublishTestResults@2
+  displayName: publish ${{ parameters.testName }} results
+  inputs:
+    testResultsFormat: ${{ parameters.testResultsFormat }}
+    testResultsFiles: ${{ parameters.testResultsFiles }}
+    testRunTitle: ${{ parameters.testName }}
+  condition: succeededOrFailed()

build-tools/automation/yaml-templates/run-installer.yaml

@@ -0,0 +1,22 @@
+parameters:
+  artifactDirectory: $(System.DefaultWorkingDirectory)
+
+steps:
+- powershell: |
+    if ([Environment]::OSVersion.Platform -eq "Unix") {
+        $installer = Get-ChildItem -Path "${{ parameters.artifactDirectory }}/*" -Include *.pkg -File
+    } else {
+        $installer = Get-ChildItem -Path "${{ parameters.artifactDirectory }}\*" -Include *.vsix -File
+    }
+    if (![System.IO.File]::Exists($installer)) {
+        throw [System.IO.FileNotFoundException] "Installer not found in $artifactDirectory."
+    }
+    Write-Host "##vso[task.setvariable variable=XA.Provisionator.Args]$installer"
+  displayName: find installer and set provisionator variable
+
+- task: provisionator@2
+  inputs:
+    provisionator_uri: $(provisionator-uri)
+    github_token: $(GitHub.Token)
+    provisioning_script: $(XA.Provisionator.Args)
+    provisioning_extra_args: -vv

build-tools/scripts/Jar.targets

@@ -7,6 +7,8 @@
       <_JavacSourceVersion Condition=" '$(_JavacSourceVersion)' == '' ">1.5</_JavacSourceVersion>
       <_JavacTargetVersion Condition="$(_JdkVersion.StartsWith ('9'))">1.8</_JavacTargetVersion>
       <_JavacTargetVersion Condition=" '$(_JavacTargetVersion)' == '' ">1.6</_JavacTargetVersion>
+      <JarPath Condition=" '$(JarPath)' == '' ">$(JavaSdkDirectory)\bin\jar</JarPath>
+      <JavaCPath Condition=" '$(JavaCPath)' == '' ">$(JavaSdkDirectory)\bin\javac</JavaCPath>
     </PropertyGroup>
   </Target>
   <Target Name="BuildTestJarFile"