Dedupe workload packs

src/Microsoft.DotNet.Build.Tasks.Workloads.Tests/GenerateVisualStudioWorkloadTests.cs

@@ -16,7 +16,7 @@ public class GenerateVisualStudioWorkloadTests
     {
         public string IntermediateBaseOutputPath = Path.Combine(AppContext.BaseDirectory, "obj");
 
-        public string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets");
+        public static readonly string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets");
 
         public string TestIntermediateBaseOutputPath => Path.Combine(IntermediateBaseOutputPath, Path.GetFileNameWithoutExtension(Path.GetTempFileName()));
 

src/Microsoft.DotNet.Build.Tasks.Workloads.Tests/GenerateWorkloadMsisTests.cs

@@ -0,0 +1,34 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.IO;
+using System.Linq;
+using Microsoft.Build.Utilities;
+using Xunit;
+
+namespace Microsoft.DotNet.Build.Tasks.Workloads.Tests
+{
+    public class GenerateWorkloadMsisTests
+    {
+        public string TestAssetsPath = Path.Combine(AppContext.BaseDirectory, "testassets");
+
+        [Fact]
+        public void ItRemovesDuplicateWorkloadPacks()
+        {
+            TaskItem[] manifests = new[]
+            {
+                new TaskItem(Path.Combine(TestAssetsPath, "emsdkWorkloadManifest.json")),
+                new TaskItem(Path.Combine(TestAssetsPath, "emsdkWorkloadManifest2.json")),
+            };
+
+            var packs = GenerateWorkloadMsis.GetWorkloadPacks(manifests).ToArray();
+
+            Assert.Equal(4, packs.Length);
+            Assert.Equal("Microsoft.NET.Runtime.Emscripten.Node", packs[0].Id.ToString());
+            Assert.Equal("7.0.0-alpha.2.22078.1", packs[0].Version);
+            Assert.Equal("Microsoft.NET.Runtime.Emscripten.Node", packs[3].Id.ToString());
+            Assert.Equal("7.0.0-alpha.2.22079.1", packs[3].Version);
+        }
+    }
+}

src/Microsoft.DotNet.Build.Tasks.Workloads.Tests/Microsoft.DotNet.Build.Tasks.Workloads.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>net472;netcoreapp3.1</TargetFrameworks>
@@ -21,6 +21,8 @@
   </ItemGroup>
 
   <ItemGroup>
+    <None Remove="testassets\emsdkWorkloadManifest.json" />
+    <None Remove="testassets\emsdkWorkloadManifest2.json" />
     <None Remove="testassets\mauiWorkloadManifest.json" />
   </ItemGroup>
 </Project>

src/Microsoft.DotNet.Build.Tasks.Workloads.Tests/testassets/emsdkWorkloadManifest.json

@@ -0,0 +1,46 @@
+{
+  "version": "7.0.0-alpha.2.22078.1",
+  "workloads": {
+    "microsoft-net-sdk-emscripten": {
+      "abstract": true,
+      "description": "Emscripten SDK compiler tooling",
+      "packs": [
+        "Microsoft.NET.Runtime.Emscripten.Node",
+        "Microsoft.NET.Runtime.Emscripten.Python",
+        "Microsoft.NET.Runtime.Emscripten.Sdk"
+      ],
+      "platforms": [ "win-x64", "linux-x64", "osx-x64", "osx-arm64" ]
+    }
+  },
+  "packs": {
+    "Microsoft.NET.Runtime.Emscripten.Node" : {
+      "kind": "Sdk",
+      "version": "7.0.0-alpha.2.22078.1",
+      "alias-to": {
+        "win-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.win-x64",
+        "linux-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.linux-x64",
+        "osx-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.osx-x64",
+        "osx-arm64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.osx-x64"
+      }
+    },
+    "Microsoft.NET.Runtime.Emscripten.Python" : {
+      "kind": "Sdk",
+      "version": "7.0.0-alpha.2.22078.1",
+      "alias-to": {
+        "win-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Python.win-x64",
+        "osx-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Python.osx-x64",
+        "osx-arm64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Python.osx-x64"
+      }
+    },
+    "Microsoft.NET.Runtime.Emscripten.Sdk" : {
+      "kind": "Sdk",
+      "version": "7.0.0-alpha.2.22078.1",
+      "alias-to": {
+        "win-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.win-x64",
+        "linux-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.linux-x64",
+        "osx-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.osx-x64",
+        "osx-arm64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.osx-x64"
+      }
+    }
+  }
+}

src/Microsoft.DotNet.Build.Tasks.Workloads.Tests/testassets/emsdkWorkloadManifest2.json

@@ -0,0 +1,46 @@
+{
+  "version": "7.0.0-alpha.2.22078.1",
+  "workloads": {
+    "microsoft-net-sdk-emscripten": {
+      "abstract": true,
+      "description": "Emscripten SDK compiler tooling",
+      "packs": [
+        "Microsoft.NET.Runtime.Emscripten.Node",
+        "Microsoft.NET.Runtime.Emscripten.Python",
+        "Microsoft.NET.Runtime.Emscripten.Sdk"
+      ],
+      "platforms": [ "win-x64", "linux-x64", "osx-x64", "osx-arm64" ]
+    }
+  },
+  "packs": {
+    "Microsoft.NET.Runtime.Emscripten.Node" : {
+      "kind": "Sdk",
+      "version": "7.0.0-alpha.2.22079.1",
+      "alias-to": {
+        "win-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.win-x64",
+        "linux-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.linux-x64",
+        "osx-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.osx-x64",
+        "osx-arm64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Node.osx-x64"
+      }
+    },
+    "Microsoft.NET.Runtime.Emscripten.Python" : {
+      "kind": "Sdk",
+      "version": "7.0.0-alpha.2.22078.1",
+      "alias-to": {
+        "win-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Python.win-x64",
+        "osx-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Python.osx-x64",
+        "osx-arm64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Python.osx-x64"
+      }
+    },
+    "Microsoft.NET.Runtime.Emscripten.Sdk" : {
+      "kind": "Sdk",
+      "version": "7.0.0-alpha.2.22078.1",
+      "alias-to": {
+        "win-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.win-x64",
+        "linux-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.linux-x64",
+        "osx-x64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.osx-x64",
+        "osx-arm64": "Microsoft.NET.Runtime.Emscripten.2.0.34.Sdk.osx-x64"
+      }
+    }
+  }
+}

src/Microsoft.DotNet.Build.Tasks.Workloads/src/GenerateWorkloadMsis.cs

@@ -69,10 +69,11 @@ public override bool Execute()
 
                 // Each pack maps to multiple packs and different MSI packages. We consider a pack
                 // to be missing when none of its dependent MSIs were found/generated.
-                IEnumerable<WorkloadPack> workloadPacks = GetWorkloadPacks();
+                IEnumerable<WorkloadPack> workloadPacks = GetWorkloadPacks(WorkloadManifests);
+
                 List<string> missingPackIds = new(workloadPacks.Select(p => $"{p.Id}"));
 
-                List<(string sourcePackage, string swixPackageId, string outputPath, WorkloadPackKind kind, string[] platforms)> packsToGenerate = new();
+                HashSet<(string sourcePackage, string swixPackageId, string outputPath, WorkloadPackKind kind, string[] platforms)> packsToGenerate = new();
 
                 foreach (WorkloadPack pack in workloadPacks)
                 {
@@ -99,14 +100,7 @@ public override bool Execute()
                         string swixPackageId = $"{pack.Id.ToString().Replace(ShortNames)}.{pack.Version}";
 
                         // Always select the pack ID for the VS MSI package, even when aliased.
-                        if (RunInParallel)
-                        {
-                            packsToGenerate.Add(new(sourcePackage, swixPackageId, OutputPath, pack.Kind, platforms));
-                        }
-                        else
-                        {
-                            msis.AddRange(Generate(sourcePackage, swixPackageId, OutputPath, pack.Kind, platforms));
-                        }
+                        packsToGenerate.Add(new(sourcePackage, swixPackageId, OutputPath, pack.Kind, platforms));
                     }
                 }
 
@@ -117,6 +111,13 @@ public override bool Execute()
                         msis.AddRange(Generate(p.sourcePackage, p.swixPackageId, p.outputPath, p.kind, p.platforms));
                     });
                 }
+                else
+                {
+                    foreach (var p in packsToGenerate)
+                    {
+                        msis.AddRange(Generate(p.sourcePackage, p.swixPackageId, p.outputPath, p.kind, p.platforms));
+                    }
+                }
 
                 Msis = msis.ToArray();
                 MissingPacks = missingPacks.ToArray();
@@ -130,25 +131,37 @@ public override bool Execute()
             return !Log.HasLoggedErrors;
         }
 
-        private IEnumerable<WorkloadPack> GetWorkloadPacks()
+        internal static IEnumerable<WorkloadPack> GetWorkloadPacks(ITaskItem[] workloadManifestItems)
         {
-            // We need to track duplicate packs so we only generate MSIs once. We'll key off the pack ID and version.
-            IEnumerable<WorkloadManifest> manifests = WorkloadManifests.Select(
-                w => WorkloadManifestReader.ReadWorkloadManifest(Path.GetFileNameWithoutExtension(w.ItemSpec), File.OpenRead(w.ItemSpec)));
-
-            // We want all workloads in all manifests iff the workload has no platform or at least one
-            // platform includes Windows
-            var workloads = manifests.SelectMany(m => m.Workloads).
-                Select(w => w.Value).
-                Where(wd => wd is WorkloadDefinition).
-                Where(wd => (((WorkloadDefinition)wd).Platforms == null) || ((WorkloadDefinition)wd).Platforms.Any(p => p.StartsWith("win")));
-
-            var packIds = workloads.Where(wd => wd is WorkloadDefinition).
-                                    Where(w => (((WorkloadDefinition)w).Packs != null)).SelectMany(w => ((WorkloadDefinition)w).Packs).Distinct();
-
-            return manifests.SelectMany(m => m.Packs.Values).
-                Where(p => packIds.Contains(p.Id)).
-                Distinct();
+            // We need to track duplicate packs (same ID and version) so we only build MSIs once when processing
+            // multiple manifests. We'll manually deduplicate the packs
+            // since WorkloadPack doesn't provide an override for GetHashCode/Equals.
+            Dictionary<string, WorkloadPack> packs = new();
+
+            foreach (ITaskItem item in workloadManifestItems)
+            {
+                var workloadManifest = WorkloadManifestReader.ReadWorkloadManifest(
+                    Path.GetFileNameWithoutExtension(item.ItemSpec), File.OpenRead(item.ItemSpec));
+
+                foreach (var workload in workloadManifest.Workloads.Values)
+                {
+                    if ((workload is WorkloadDefinition wd) && (wd.Platforms == null || wd.Platforms.Any(p => p.StartsWith("win"))) && (wd.Packs != null))
+                    {
+                        foreach (var packId in wd.Packs)
+                        {
+                            var pack = workloadManifest.Packs[packId];
+                            string key = $"{pack.Id},{pack.Version}";
+
+                            if (!packs.ContainsKey(key))
+                            {
+                                packs[key] = pack;
+                            }
+                        }
+                    }
+                }
+            }
+
+            return packs.Values;
         }
 
         /// <summary>