[testing] Enable a way to test on real devices

eng/devices/ios.cake

@@ -34,7 +34,9 @@ string DEVICE_OS = "";
 
 // other
 string PLATFORM = TEST_DEVICE.ToLower().Contains("simulator") ? "iPhoneSimulator" : "iPhone";
-string DOTNET_PLATFORM = TEST_DEVICE.ToLower().Contains("simulator") ? "iossimulator-x64" : "ios-arm64";
+string DOTNET_PLATFORM = TEST_DEVICE.ToLower().Contains("simulator") ? 
+	$"iossimulator-{System.Runtime.InteropServices.RuntimeInformation.OSArchitecture.ToString().ToLower()}"
+  : $"ios-arm64";
 string CONFIGURATION = Argument("configuration", "Debug");
 bool DEVICE_CLEANUP = Argument("cleanup", true);
 string TEST_FRAMEWORK = "net472";
@@ -63,6 +65,9 @@ Setup(context =>
 
 	Information($"DOTNET_TOOL_PATH {DOTNET_TOOL_PATH}");
 
+	Information("Host OS System Arch: {0}", System.Runtime.InteropServices.RuntimeInformation.OSArchitecture);
+	Information("Host Processor System Arch: {0}", System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture);
+
 	// only grab attached iOS devices if we are trying to test on device
 	if (TEST_DEVICE.Contains("device")) 
 	{ 
@@ -93,7 +98,16 @@ void Cleanup()
 		Information("No simulators found to delete.");
 		return;
 	}
-	var xharness = sims.Where(s => s.Name.Contains("XHarness")).ToArray();
+	var simulatorName = "XHarness";
+	if(iosVersion.Contains("17"))
+		simulatorName = "iPhone 15";	
+	Information("Looking for simulator: {0} iosversion {1}", simulatorName, iosVersion);
+	var xharness = sims.Where(s => s.Name.Contains(simulatorName))?.ToArray();
+	if(xharness == null || xharness.Length == 0)
+	{
+		Information("No XHarness simulators found to delete.");
+		return;
+	}
 	foreach (var sim in xharness) {
 		Information("Deleting XHarness simulator {0} ({1})...", sim.Name, sim.UDID);
 		StartProcess("xcrun", "simctl shutdown " + sim.UDID);
@@ -185,8 +199,8 @@ Task("Test")
 			TEST_APP = apps.First().FullPath;
 		}
 		else {
-			Error("Error: Couldn't find .app file");
-			throw new Exception("Error: Couldn't find .app file");
+			Error($"Error: Couldn't find *.app file in {binDir}");
+			throw new Exception($"Error: Couldn't find *.app file in {binDir}");
 		}
 	}
 	if (string.IsNullOrEmpty(TEST_RESULTS)) {
@@ -219,12 +233,25 @@ Task("Test")
 	var settings = new DotNetToolSettings {
 		ToolPath = DOTNET_TOOL_PATH,
 		DiagnosticOutput = true,
-		ArgumentCustomization = args => args.Append("run xharness apple test " +
-		$"--app=\"{TEST_APP}\" " +
-		$"--targets=\"{TEST_DEVICE}\" " +
-		$"--output-directory=\"{TEST_RESULTS}\" " +
-		xcode_args +
-		$"--verbosity=\"Debug\" ")
+		ArgumentCustomization = args => 
+		{
+			args.Append("run xharness apple test " +
+				$"--app=\"{TEST_APP}\" " +
+				$"--targets=\"{TEST_DEVICE}\" " +
+				$"--output-directory=\"{TEST_RESULTS}\" " +
+				xcode_args +
+				$"--verbosity=\"Debug\" ");
+
+			if (TEST_DEVICE.Contains("device"))
+			{
+				if(string.IsNullOrEmpty(DEVICE_UDID))
+				{
+					throw new Exception("No device was found to install the app on. See the Setup method for more details.");
+				}
+				args.Append($"--device=\"{DEVICE_UDID}\" ");
+			}
+			return args;	
+		}
 	};
 
 	bool testsFailed = true;
@@ -468,35 +495,45 @@ void GetDevices(string version)
 						};
 				}
 	});
+	Information("List count: {0}", list.Count);	
 	//this removes the extra lines from output
+	if(list.Count == 0)
+	{
+		throw new Exception($"No devices found");
+		return;
+	}
 	list.Remove(list.Last());
 	list.Remove(list.Last());
 	foreach (var item in list)
 	{
-		Information("Device: {0}", item.Trim());
-		var parts = item.Trim().Split(" ",StringSplitOptions.RemoveEmptyEntries);
-		if(item.Contains("iPhone"))
+		var stringToTest = $"Device: {item.Trim()}";
+		Information(stringToTest);
+		var regex = new System.Text.RegularExpressions.Regex(@"Device:\s+((?:[^\s]+(?:\s+[^\s]+)*)?)\s+([0-9A-Fa-f-]+)\s+([\d.]+)\s+(iPhone|iPad)\s+iOS");
+		var match = regex.Match(stringToTest);
+		if(match.Success)
 		{
-			deviceOS = "iPhone";
-		}
-		if(item.Contains("iPad iOS"))
-		{
-			deviceOS = "iPad iOS";
+			deviceName = match.Groups[1].Value;
+			deviceUdid = match.Groups[2].Value;
+			deviceVersion = match.Groups[3].Value;
+			deviceOS = match.Groups[4].Value;
+			Information("DeviceName:{0} udid:{1} version:{2} os:{3}", deviceName, deviceUdid, deviceVersion, deviceOS);
+			if(version.Contains(deviceVersion.Split(".")[0]))
+			{
+				Information("We want this device: {0} {1} because it matches {2}", deviceName, deviceVersion, version);
+				DEVICE_UDID = deviceUdid;
+				DEVICE_VERSION = deviceVersion;
+				DEVICE_NAME = deviceName;
+				DEVICE_OS = deviceOS;
+				break;
+			}
 		}
-
-		deviceName = parts[0].Trim();
-		deviceUdid = parts[1].Trim();
-		deviceVersion = parts[2].Trim();
-		Information("DeviceName:{0} udid:{1} version:{2} os:{3}", deviceName, deviceUdid, deviceVersion, deviceOS);
-
-		if(version.Contains(deviceVersion.Split(".")[0]))
-		{
-			Information("We want this device: {0} {1} because it matches {2}", deviceName, deviceVersion, version);
-			DEVICE_UDID = deviceUdid;
-			DEVICE_VERSION = deviceVersion;
-			DEVICE_NAME = deviceName;
-			DEVICE_OS = deviceOS;
-			break;
+        else
+        {
+            Information("No match found for {0}", stringToTest);
 		}
 	}
+	if(string.IsNullOrEmpty(DEVICE_UDID))
+	{
+		throw new Exception($"No devices found for version {version}");
+	}
 }

eng/pipelines/common/device-tests-steps.yml

@@ -2,13 +2,15 @@ parameters:
   platform: '' # [ android, ios, catalyst, windows ]
   path: '' # path to csproj
   device: '' # the xharness device to use
+  apiversion: '' # the iOS device api version to use
   cakeArgs: '' # additional cake args
   provisionatorChannel: 'latest'
   agentPoolAccessToken: ''
   artifactName: 'nuget'
   artifactItemPattern: '**/*.nupkg'
   checkoutDirectory: $(System.DefaultWorkingDirectory)
   useArtifacts: false
+  rebootAgent: true
 
 steps:
   - template: provision.yml
@@ -18,7 +20,7 @@ steps:
       ${{ if or(eq(parameters.platform, 'ios'), eq(parameters.platform, 'catalyst'), eq(parameters.platform, 'android'))}}:
         platform: macos
       skipXcode: ${{ or(eq(parameters.platform, 'android'), eq(parameters.platform, 'windows')) }}
-      skipProvisioning: ${{ eq(parameters.platform, 'windows') }}
+      skipProvisioning: ${{ or(eq(parameters.platform, 'windows'),eq(parameters.platform, 'ios')) }}
       provisionatorChannel: ${{ parameters.provisionatorChannel }}
 
   - pwsh: ./build.ps1 --target=dotnet --configuration="Release" --verbosity=diagnostic
@@ -67,15 +69,15 @@ steps:
     displayName: 'Set Platform.Name'
 
   - pwsh: |
-      ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --project="${{ parameters.path }}" --device=${{ parameters.device }} --packageid=${{ parameters.windowsPackageId }} --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }}
+      ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --project="${{ parameters.path }}" --device=${{ parameters.device }} --apiversion=${{ parameters.apiversion }} --packageid=${{ parameters.windowsPackageId }} --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }}
     displayName: $(Agent.JobName)
     workingDirectory: ${{ parameters.checkoutDirectory }}
     condition: and(succeeded(), ne(variables['Platform.Name'], 'Mac'))
     retryCountOnTaskFailure: 2
 
   - bash: |
       # Execute the powershell script from a bash shell on Mac to avoid interference between powershell processes that lead to this error: The STDIO streams did not close within 10 seconds of the exit event from process '/usr/local/bin/pwsh'. This may indicate a child process inherited the STDIO streams and has not yet exited.
-      pwsh ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --project="${{ parameters.path }}" --device=${{ parameters.device }} --packageid=${{ parameters.windowsPackageId }} --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }}
+      pwsh ./build.ps1 -Script eng/devices/${{ parameters.platform }}.cake --project="${{ parameters.path }}" --device=${{ parameters.device }} --apiversion=${{ parameters.apiversion }} --packageid=${{ parameters.windowsPackageId }} --results="$(TestResultsDirectory)" --binlog="$(LogDirectory)" ${{ parameters.cakeArgs }}
     displayName: $(Agent.JobName)
     workingDirectory: ${{ parameters.checkoutDirectory }}
     condition: and(succeeded(), eq(variables['Platform.Name'], 'Mac'))
@@ -95,7 +97,8 @@ steps:
     inputs:
       artifactName: '$(Agent.JobName) (attempt $(System.JobAttempt))'
 
-  # This must always be placed as the last step in the job
-  - template: agent-rebooter/mac.v1.yml@yaml-templates
-    parameters:
-      AgentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}
+  - ${{ if eq(parameters.rebootAgent, true) }}:
+    # This must always be placed as the last step in the job
+    - template: agent-rebooter/mac.v1.yml@yaml-templates
+      parameters:
+        AgentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}

eng/pipelines/common/device-tests.yml

@@ -5,6 +5,7 @@ parameters:
   windowsPool: { }
   androidApiLevels: [ 33 ]
   iosVersions: [ 'latest' ]
+  iosDeviceVersions: [ '15' ]
   catalystVersions: [ 'latest' ]
   provisionatorChannel: 'latest'
   agentPoolAccessToken: ''
@@ -42,14 +43,17 @@ stages:
                     PROJECT_PATH: ${{ project.android }}
                     ${{ if eq(api, 27) }}:
                       DEVICE: android-emulator-32_${{ api }}
+                      APIVERSION: ${{ api }}
                     ${{ if not(eq(api, 27)) }}:
                       DEVICE: android-emulator-64_${{ api }}
+                      APIVERSION: ${{ api }}
       steps:
         - template: device-tests-steps.yml
           parameters:
             platform: android
             path: $(PROJECT_PATH) 
             device: $(DEVICE)
+            apiVersion: $(APIVERSION)
             windowsPackageId: android # Only needed for Windows, will be ignored
             provisionatorChannel: ${{ parameters.provisionatorChannel }}
             agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}
@@ -65,9 +69,9 @@ stages:
     - job: ios_device_tests
       workspace:
         clean: all
-      displayName: "iOS simulator tests"
+      displayName: "iOS tests"
       pool: ${{ parameters.iosPool }}
-      timeoutInMinutes: 240
+      timeoutInMinutes: 140
       strategy:
         matrix:
           # create all the variables used for the matrix
@@ -78,16 +82,19 @@ stages:
                   ${{ replace(coalesce(project.desc, project.name), ' ', '_') }}_V_${{ version }}:
                     REQUIRED_XCODE: $(DEVICETESTS_REQUIRED_XCODE)
                     PROJECT_PATH: ${{ project.ios }}
-                    ${{ if eq(version, 'latest') }}:
-                      DEVICE: ios-simulator-64
+                    ${{ if contains(version, 'device') }}:
+                      DEVICE: ios-device
+                      APIVERSION: ${{ replace(version, 'device-', '') }}
                     ${{ else }}:
-                      DEVICE: ios-simulator-64_${{ version }}
+                      DEVICE: ios-simulator-64_${{ replace(version, 'simulator-', '') }}
+                      APIVERSION: ${{ replace(version, 'simulator-', '') }}
       steps:
         - template: device-tests-steps.yml
           parameters:
             platform: ios
             path: $(PROJECT_PATH)
             device: $(DEVICE)
+            apiVersion: $(APIVERSION)
             windowsPackageId: ios # Only needed for Windows, will be ignored
             provisionatorChannel: ${{ parameters.provisionatorChannel }}
             agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}
@@ -126,6 +133,7 @@ stages:
             platform: catalyst
             path: $(PROJECT_PATH)
             device: $(DEVICE)
+            apiVersion: macos # Only needed for iOS, will be ignored
             windowsPackageId: catalyst # Only needed for Windows, will be ignored
             provisionatorChannel: ${{ parameters.provisionatorChannel }}
             agentPoolAccessToken: ${{ parameters.agentPoolAccessToken }}
@@ -159,6 +167,7 @@ stages:
           parameters:
             platform: windows
             path: $(PROJECT_PATH)
+            apiVersion: 10.0.19041
             windowsPackageId: $(PACKAGE_ID)
             device: $(DEVICE) # For Windows this switches between packaged and unpackaged
             provisionatorChannel: ${{ parameters.provisionatorChannel }}

eng/pipelines/common/provision.yml

@@ -1,5 +1,6 @@
 parameters:
   poolName: ''
+  clearCaches: true
   skipXcode: false
   skipProvisioning: $(skipProvisionator)
   skipAndroidSdks: false
@@ -134,8 +135,9 @@ steps:
       dotnet --list-sdks
     displayName: 'Show .NET SDK info'
 
-  - pwsh: dotnet nuget locals all --clear
-    displayName: 'Clear all NuGet caches'
+  - ${{ if eq(parameters.clearCaches, 'true') }}:
+    - pwsh: dotnet nuget locals all --clear
+      displayName: 'Clear all NuGet caches'
 
   - ${{ if eq(variables['System.TeamProject'], 'devdiv') }}:
     - task: PowerShell@2

eng/pipelines/device-tests.yml

@@ -107,13 +107,13 @@ stages:
       agentPoolAccessToken: $(AgentPoolAccessToken)
       ${{ if or(parameters.BuildEverything, and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'devdiv'))) }}:
         androidApiLevels: [ 33, 30, 29, 28, 27, 26, 25, 24, 23 ]
-        iosVersions: [ '16.4', '15.5', '14.5']
+        iosVersions: [ 'simulator-16.4','simulator-15.5','simulator-14.5' ]
         catalystVersions: [ 'latest' ]
         windowsVersions: ['packaged', 'unpackaged']
         provisionatorChannel: ${{ parameters.provisionatorChannel }}
       ${{ if not(or(parameters.BuildEverything, and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'devdiv')))) }}:
         androidApiLevels: [ 30, 23 ]
-        iosVersions: [ '16.4' ]
+        iosVersions: [ 'simulator-16.4' ]
         catalystVersions: [ 'latest' ]
         windowsVersions: ['packaged', 'unpackaged']
         provisionatorChannel: ${{ parameters.provisionatorChannel }}
@@ -144,7 +144,7 @@ stages:
           windows: $(System.DefaultWorkingDirectory)/src/Core/tests/DeviceTests/Core.DeviceTests.csproj
         - name: controls
           desc: Controls
-          androidApiLevelsExclude: [25] # Ignore for now API25 since the runs's are not stable
+          androidApiLevelsExclude: [27, 25] # Ignore for now API25 since the runs's are not stable
           windowsPackageId: 'com.microsoft.maui.controls.devicetests'
           android: $(System.DefaultWorkingDirectory)/src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj
           ios: $(System.DefaultWorkingDirectory)/src/Controls/tests/DeviceTests/Controls.DeviceTests.csproj