dotnet · ivanpovazan · Sep 26, 2024 · Sep 26, 2024
diff --git a/.gitignore b/.gitignore
@@ -360,3 +360,5 @@ MigrationBackup/
 
 # some ide stuff
 .idea
+
+.vscode/
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -15,7 +15,7 @@
     <PackageVersion Include="Mono.Options" Version="6.12.0.148" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Selenium.WebDriver" Version="4.0.0-alpha05" />
-    <PackageVersion Include="Microsoft.Tools.Mlaunch" Version="1.0.96" />
+    <PackageVersion Include="Microsoft.Tools.Mlaunch" Version="1.0.256" />
     <PackageVersion Include="NUnit" Version="3.13.0" />
     <PackageVersion Include="NUnit.Engine" Version="3.13.0" />
     <PackageVersion Include="xunit.extensibility.execution" Version="$(XUnitVersion)" />

diff --git a/src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/Simulators/InstallCommand.cs b/src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/Simulators/InstallCommand.cs
@@ -121,6 +121,30 @@ protected override async Task<ExitCode> InvokeInternal(ILogger logger)
 
     private async Task<bool> Install(Simulator simulator)
     {
+        if (simulator.Source is null)
+        {
+            var xcodeVersion = await GetXcodeVersion();
+            if (!simulator.IsCryptexDiskImage || Version.Parse(xcodeVersion).Major < 16)
+            {
+                throw new Exception($"Cannot download simulator: {simulator.Name} from source nor through Xcode: {xcodeVersion}");
+            }
+            else
+            {
+                Logger.LogInformation($"Downloading and installing simulator: {simulator.Name} through xcodebuild with Xcode: {xcodeVersion}");
+                var (succeeded, stdout) = await ExecuteCommand("xcodebuild", TimeSpan.FromMinutes(15), "-downloadPlatform", simulator.Platform, "-verbose");
+                if (!succeeded)
+                {
+                    Logger.LogError($"Download and installation failed through xcodebuild for simulator: {simulator.Name} with Xcode: {xcodeVersion}!" + Environment.NewLine + stdout);
+                    return false;
+                }
+                else
+                {
+                    Logger.LogDebug(stdout);
+                    return true;
+                }
+            }
+        }
+
         var filename = Path.GetFileName(simulator.Source);
         var downloadPath = Path.Combine(TempDirectory, filename);
         var download = true;

diff --git a/src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/Simulators/Simulator.cs b/src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/Simulators/Simulator.cs
@@ -9,21 +9,25 @@ namespace Microsoft.DotNet.XHarness.CLI.Commands.Apple.Simulators;
 internal class Simulator
 {
     public string Name { get; }
+    public string Platform { get; }
     public string Identifier { get; }
     public string Version { get; }
-    public string Source { get; }
+    public string? Source { get; }
     public string InstallPrefix { get; }
     public long FileSize { get; }
     public bool IsDmgFormat { get; }
+    public bool IsCryptexDiskImage { get; }
 
-    public Simulator(string name, string identifier, string version, string source, string installPrefix, long fileSize)
+    public Simulator(string name, string platform, string identifier, string version, string? source, string installPrefix, long fileSize, bool isCryptexDiskImage)
     {
         Name = name;
+        Platform = platform;
         Identifier = identifier;
         Version = version;
         Source = source;
         InstallPrefix = installPrefix;
         FileSize = fileSize;
         IsDmgFormat = Identifier.StartsWith("com.apple.dmg.", StringComparison.OrdinalIgnoreCase);
+        IsCryptexDiskImage = isCryptexDiskImage;
     }
 }
diff --git a/src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/Simulators/SimulatorsCommand.cs b/src/Microsoft.DotNet.XHarness.CLI/Commands/Apple/Simulators/SimulatorsCommand.cs
@@ -108,13 +108,6 @@ protected async Task<IEnumerable<Simulator>> GetAvailableSimulators()
             var versionNode = downloadable.SelectSingleNode("key[text()='version']/following-sibling::string") ?? throw new Exception("Version node not found");
             var identifierNode = downloadable.SelectSingleNode("key[text()='identifier']/following-sibling::string") ?? throw new Exception("Identifier node not found");
             var sourceNode = downloadable.SelectSingleNode("key[text()='source']/following-sibling::string");
-            if (sourceNode is null)
-            {
-                // It seems that Apple can list beta simulators in the index file, but they do not provide a source for downloading them (eg: iOS 18.0 beta Simulator Runtime).
-                // In such cases log a warning and skip trying to download such simulator as they are not publicly available.
-                Logger.LogWarning($"Simulator with name: '{nameNode.InnerText}' version: '{versionNode.InnerText}' identifier: '{identifierNode.InnerText}' has no source for download, skipping...");
-                continue;
-            }
 
             var fileSizeNode = downloadable.SelectSingleNode("key[text()='fileSize']/following-sibling::integer|key[text()='fileSize']/following-sibling::real");
             var installPrefixNode = downloadable.SelectSingleNode("key[text()='userInfo']/following-sibling::dict/key[text()='InstallPrefix']/following-sibling::string");
@@ -144,13 +137,51 @@ protected async Task<IEnumerable<Simulator>> GetAvailableSimulators()
                 installPrefix = $"/Library/Developer/CoreSimulator/Profiles/Runtimes/{simRuntimeName}";
             }
 
+            var platform = name.Split(' ').FirstOrDefault();
+            if (platform is null)
+            {
+                Logger.LogWarning($"Platform name could not be parsed from simulator name: '{nameNode.InnerText}' version: '{versionNode.InnerText}' identifier: '{identifierNode.InnerText}' skipping...");
+                continue;
+            }
+
+            var source = ReplaceStringUsingKey(sourceNode?.InnerText, dict);
+            var isCryptexDiskImage = false;
+            if (source is null)
+            {
+                // We allow source to be missing for newer simulators (e.g., iOS 18+ available from Xcode 16) that use cryptographically-sealed archives.
+                // Eg.:
+                // <dict>
+                //     <key>category</key>
+                //     <string>simulator</string>
+                //     <key>contentType</key>
+                //     <string>cryptexDiskImage</string>
+                //     ...
+                // These images are downloaded and installed through xcodebuild instead.
+                // https://developer.apple.com/documentation/xcode/installing-additional-simulator-runtimes#Install-and-manage-Simulator-runtimes-from-the-command-line
+                var contentTypeNode = downloadable.SelectSingleNode("key[text()='contentType']/following-sibling::string") ?? throw new Exception("ContentType node not found");
+                var contentType = contentTypeNode.InnerText;
+                if (contentType.Equals("cryptexDiskImage", StringComparison.OrdinalIgnoreCase))
+                {
+                    isCryptexDiskImage = true;
+                    Logger.LogInformation($"Simulator with name: '{nameNode.InnerText}' version: '{versionNode.InnerText}' identifier: '{identifierNode.InnerText}' has no source but it is a cryptex disk image which can be downloaded through xcodebuild.");
+                }
+                else
+                {
+                    Logger.LogWarning($"Simulator with name: '{nameNode.InnerText}' version: '{versionNode.InnerText}' identifier: '{identifierNode.InnerText}' has no source for download nor it is a cryptex disk image, skipping...");
+                    continue;
+                }
+            }
+
             simulators.Add(new Simulator(
                 name: name,
+                platform: platform,
                 identifier: ReplaceStringUsingKey(identifierNode.InnerText, dict),
                 version: versionNode.InnerText,
-                source: ReplaceStringUsingKey(sourceNode.InnerText, dict),
+                source: source,
                 installPrefix: installPrefix,
-                fileSize: (long)parsedFileSize));
+                fileSize: (long)parsedFileSize,
+                isCryptexDiskImage: isCryptexDiskImage
+                ));
         }
 
         return simulators;
@@ -182,6 +213,8 @@ protected async Task<IEnumerable<Simulator>> GetAvailableSimulators()
             {
                 return null;
             }
+            Logger.LogDebug($"Listing runtime disk images via returned: {json}");
+
             string simulatorRuntime = "";
             string simulatorVersion = "";
 
@@ -385,7 +418,7 @@ private async Task<bool> DownloadFile(string url, string destinationPath)
         return false;
     }
 
-    private async Task<string> GetXcodeVersion()
+    protected async Task<string> GetXcodeVersion()
     {
         if (_xcodeVersion is not null)
         {

diff --git a/tests/integration-tests/Apple/Simulator.Commands.Tests.proj b/tests/integration-tests/Apple/Simulator.Commands.Tests.proj
@@ -4,6 +4,10 @@
     <HelixTargetQueue Include="osx.13.amd64.open"/>
   </ItemGroup>
 
+  <PropertyGroup>
+    <iOSSimulatorVersionUnderTest>16.4</iOSSimulatorVersionUnderTest>
+  </PropertyGroup>
+
   <PropertyGroup>
     <TestAppBundleName>System.Numerics.Vectors.Tests</TestAppBundleName>
     <XHarnessTestAppBundleUrl>$(AssetsBaseUri)/ios/test-app/ios-simulator-64/$(TestAppBundleName).app.zip</XHarnessTestAppBundleUrl>
@@ -21,13 +25,15 @@
 
     <ItemGroup>
       <XHarnessAppBundleToTest Include="$(TestAppDestinationDir)\$(TestAppBundleName).app">
-        <TestTarget>ios-simulator-64</TestTarget>
+        <TestTarget>ios-simulator-64_$(iOSSimulatorVersionUnderTest)</TestTarget>
         <WorkItemTimeout>00:20:00</WorkItemTimeout>
         <TestTimeout>00:07:00</TestTimeout>
         <LaunchTimeout>00:03:30</LaunchTimeout>
         <CustomCommands>
         <![CDATA[
           set -ex
+          xharness apple simulators install $target --verbosity=Debug
+          xharness apple simulators reset-simulator --output-directory="$output_directory" --target=$target --verbosity=Debug
           deviceId=`xharness apple device $target`
           xharness apple install -t=$target --device="$deviceId" -o="$output_directory" --app="$app" --timeout=$launch_timeout -v
           set +e

diff --git a/tests/integration-tests/Apple/Simulator.Scouting.Commands.Tests.proj b/tests/integration-tests/Apple/Simulator.Scouting.Commands.Tests.proj
@@ -0,0 +1,58 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk">
+
+  <ItemGroup>
+    <!-- Adjust to the desired souting queue. -->
+    <HelixTargetQueue Include="osx.amd64.iphone.scouting.open"/>
+  </ItemGroup>
+
+  <PropertyGroup>
+    <!-- 
+      Change to specify the exact Xcode version for testing.
+      In the CustomCommands below we can probe installed Xcode versions on Helix with `ls -al /Applications` and then choosing the write path.
+    -->
+    <XcodeVersionUnderTest>Xcode_16_beta_6</XcodeVersionUnderTest>
+    <iOSSimulatorVersionUnderTest>18.0</iOSSimulatorVersionUnderTest>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TestAppBundleName>System.Numerics.Vectors.Tests</TestAppBundleName>
+    <XHarnessTestAppBundleUrl>$(AssetsBaseUri)/ios/test-app/ios-simulator-64/$(TestAppBundleName).app.zip</XHarnessTestAppBundleUrl>
+    <TestAppDestinationDir>$(ArtifactsTmpDir)test-app\ios-simulator-64</TestAppDestinationDir>
+  </PropertyGroup>
+
+  <Target Name="TestApple" BeforeTargets="CoreTest">
+    <DownloadFile SourceUrl="$(XHarnessTestAppBundleUrl)" DestinationFolder="$(TestAppDestinationDir)" SkipUnchangedFiles="True" Retries="5">
+      <Output TaskParameter="DownloadedFile" ItemName="ZippedAppBundle" />
+    </DownloadFile>
+
+    <Message Text="Downloaded $(TestAppBundleName) from @(ZippedAppBundle). Extracting..." Importance="High" />
+    <Exec Command="tar -xzf @(ZippedAppBundle) -C $(TestAppDestinationDir)" />
+    <Message Text="Extracted to $(TestAppDestinationDir)" Importance="High" />
+
+    <ItemGroup>
+      <XHarnessAppBundleToTest Include="$(TestAppDestinationDir)\$(TestAppBundleName).app">
+        <TestTarget>ios-simulator-64_$(iOSSimulatorVersionUnderTest)</TestTarget>
+        <WorkItemTimeout>00:20:00</WorkItemTimeout>
+        <TestTimeout>00:07:00</TestTimeout>
+        <LaunchTimeout>00:03:30</LaunchTimeout>
+        <CustomCommands>
+        <![CDATA[
+          set -ex
+          xharness apple simulators install $target --verbosity=Debug --xcode /Applications/$(XcodeVersionUnderTest).app
+          xharness apple simulators reset-simulator --output-directory="$output_directory" --target=$target --verbosity=Debug --xcode /Applications/$(XcodeVersionUnderTest).app
+          deviceId=`xharness apple device $target`
+          xharness apple install -t=$target --device="$deviceId" -o="$output_directory" --app="$app" --timeout=$launch_timeout -v --xcode /Applications/$(XcodeVersionUnderTest).app
+          set +e
+          result=0
+          xharness apple just-test -t=$target --device="$deviceId" -o="$output_directory" --app="net.dot.$(TestAppBundleName)" --launch-timeout=$launch_timeout --timeout=$timeout -v --xcode /Applications/$(XcodeVersionUnderTest).app
+          ((result|=$?))
+          xharness apple uninstall -t=$target --device="$deviceId" -o="$output_directory" --app="net.dot.$(TestAppBundleName)" -v --xcode /Applications/$(XcodeVersionUnderTest).app
+          ((result|=$?))
+          exit $result
+        ]]>
+        </CustomCommands>
+      </XHarnessAppBundleToTest>
+    </ItemGroup>
+  </Target>
+
+</Project>
diff --git a/tests/integration-tests/README.md b/tests/integration-tests/README.md
@@ -0,0 +1,20 @@
+# Integration tests
+
+This folder includes integration tests projects for different support platforms (iOS, Android, WASM).
+They are used in end-to-end testing scenarios and are referenced from `azure-pipelines-public.yml` E2E templates.
+
+In the relevant `*.proj` files one can configure various setting for execution on Helix like:
+- configuring the Helix queue (e.g., `osx.13.amd64.iphone.open` via `HelixTargetQueue` item group)
+- app bundle to download, send to Helix and test (e.g., `System.Buffers.Tests.app`)
+- etc.
+
+## Testing on scouting queue
+
+NOTE: This is Apple-specific but can be applied to other platforms as well
+
+There are two test projects which can be used on scouting queues which are not used by default:
+
+- Apple/Simulator.Scouting.Tests.proj
+- Apple/Simulator.Scouting.Commands.Tests.proj
+
+When desired, these can be included in the `azure-pipelines-public.yml` so that the CI runs them on a desired scouting queue (check the `HelixTargetQueue` setting) with a particular version of Xcode.