Change Object mode for package testing & remove older frameworks & rids

src/Microsoft.DotNet.PackageTesting.Tests/GetCompatibilePackageTargetFrameworksTests.cs

@@ -1,8 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-using NuGet.Frameworks;
 using System.Collections.Generic;
+using NuGet.Frameworks;
 using Xunit;
 
 namespace Microsoft.DotNet.PackageTesting.Tests
@@ -11,34 +11,132 @@ public class GetCompatibilePackageTargetFrameworksTests
     {
         public GetCompatibilePackageTargetFrameworksTests()
         {
-            GetCompatiblePackageTargetFrameworks.Initialize();
+            GetCompatiblePackageTargetFrameworks.Initialize("netcoreapp3.1;net5.0;net6.0;net461;net462;net471;net472;netstandard2.0;netstandard2.1");
         }
 
-        public static IEnumerable<object[]> PackageTFMData => new List<object[]>
+        public static IEnumerable<object[]> PackageTfmData => new List<object[]>
         {
             // single target framework in package
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard20}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard20, FrameworkConstants.CommonFrameworks.NetCoreApp20, FrameworkConstants.CommonFrameworks.Net463, FrameworkConstants.CommonFrameworks.Net461, FrameworkConstants.CommonFrameworks.Net462} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp20}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp20} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp21}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp21} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.Net461}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.Net461} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.Net45}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.Net45} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp30}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp30} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp31}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp31} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard21}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard21, FrameworkConstants.CommonFrameworks.NetCoreApp30 } },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard12}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard12, FrameworkConstants.CommonFrameworks.Net451 } },
-
-            // two target frameworks in package
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard20, FrameworkConstants.CommonFrameworks.Net461}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard20, FrameworkConstants.CommonFrameworks.NetCoreApp20, FrameworkConstants.CommonFrameworks.Net463, FrameworkConstants.CommonFrameworks.Net461, FrameworkConstants.CommonFrameworks.Net462} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard20, FrameworkConstants.CommonFrameworks.NetCoreApp30}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetStandard20, FrameworkConstants.CommonFrameworks.NetCoreApp30, FrameworkConstants.CommonFrameworks.NetCoreApp20, FrameworkConstants.CommonFrameworks.Net463, FrameworkConstants.CommonFrameworks.Net461, FrameworkConstants.CommonFrameworks.Net462} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp30, FrameworkConstants.CommonFrameworks.Net461}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp30, FrameworkConstants.CommonFrameworks.Net461} },
-            new object[] {  new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp30, FrameworkConstants.CommonFrameworks.Net50}, new List<NuGetFramework> { FrameworkConstants.CommonFrameworks.NetCoreApp30, FrameworkConstants.CommonFrameworks.Net50} },
+            new object[]
+            {  
+                new List<string> 
+                { 
+                    @"lib/netstandard2.0/TestPackage.dll"
+                },
+                new List<NuGetFramework> 
+                { 
+                    FrameworkConstants.CommonFrameworks.NetStandard20, 
+                    FrameworkConstants.CommonFrameworks.Net461, 
+                    FrameworkConstants.CommonFrameworks.Net462,
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31
+                } 
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"runtimes/win/lib/netstandard2.0/TestPackage.dll",
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.NetStandard20,
+                    FrameworkConstants.CommonFrameworks.Net461,
+                    FrameworkConstants.CommonFrameworks.Net462,
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31
+                }
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"lib/net5.0/TestPackage.dll",
+                    @"runtimes/win/lib/netstandard2.0/TestPackage.dll"
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.NetStandard20,
+                    FrameworkConstants.CommonFrameworks.Net461,
+                    FrameworkConstants.CommonFrameworks.Net462,
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31,
+                    FrameworkConstants.CommonFrameworks.Net50
+                }
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"lib/netcoreapp3.1/TestPackage.dll"
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31
+                }
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"lib/netcoreapp3.1/TestPackage.dll",
+                    @"lib/net461/TestPackage.dll"
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31,
+                    FrameworkConstants.CommonFrameworks.Net461
+                }
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"runtimes/unix/lib/netcoreapp3.1/TestPackage.dll",
+                    @"runtimes/win/lib/netstandard2.0/TestPackage.dll"
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31,
+                    FrameworkConstants.CommonFrameworks.NetStandard20,
+                    FrameworkConstants.CommonFrameworks.Net461,
+                    FrameworkConstants.CommonFrameworks.Net462
+                }
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"lib/net5.0/TestPackage.dll",
+                    @"lib/net472/TestPackage.dll",
+                    @"runtimes/win/lib/netstandard2.0/TestPackage.dll"
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.NetStandard20,
+                    FrameworkConstants.CommonFrameworks.Net461,
+                    FrameworkConstants.CommonFrameworks.Net462,
+                    NuGetFramework.Parse("net472"),
+                    FrameworkConstants.CommonFrameworks.NetCoreApp31,
+                    FrameworkConstants.CommonFrameworks.Net50
+                }
+            },
+            new object[]
+            {
+                new List<string>
+                {
+                    @"lib/net461/TestPackage.dll"
+                },
+                new List<NuGetFramework>
+                {
+                    FrameworkConstants.CommonFrameworks.Net461,
+                }
+            },
         };
 
         [Theory]
-        [MemberData(nameof(PackageTFMData))]
-        public void GetCompatibleFrameworks(List<NuGetFramework> packageFrameworks, List<NuGetFramework> expectedTestFrameworks)
+        [MemberData(nameof(PackageTfmData))]
+        public void GetCompatibleFrameworks(List<string> filePaths, List<NuGetFramework> expectedTestFrameworks)
         {
-            List<NuGetFramework> actualTestFrameworks = GetCompatiblePackageTargetFrameworks.GetTestFrameworks(packageFrameworks);
+            Package package = new("TestPackage", "1.0.0", filePaths);
+            IEnumerable<NuGetFramework> actualTestFrameworks = GetCompatiblePackageTargetFrameworks.GetTestFrameworks(package, "netcoreapp3.1");
             CollectionsEqual(expectedTestFrameworks, actualTestFrameworks);
         }
 

src/Microsoft.DotNet.PackageTesting.Tests/Microsoft.DotNet.PackageTesting.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>netcoreapp3.1;net472</TargetFrameworks>
@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildUtilitiesCoreVersion)" />
     <PackageReference Include="NuGet.Frameworks" Version="$(NuGetVersion)" />
+    <PackageReference Include="NuGet.Packaging" Version="$(NuGetVersion)" />
   </ItemGroup>
 
   <ItemGroup>

src/Microsoft.DotNet.PackageTesting/GetCompatiblePackageTargetFrameworks.cs

@@ -19,6 +19,9 @@ public class GetCompatiblePackageTargetFrameworks : BuildTask
         [Required]
         public string[] PackagePaths { get; set; }
 
+        [Required]
+        public string SupportedTestFrameworks { get; set; }
+
         [Output]
         public ITaskItem[] TestProjects { get; set; }
 
@@ -28,73 +31,55 @@ public override bool Execute()
             List<ITaskItem> testProjects = new List<ITaskItem>();
             try
             {
-                Initialize();
+                Initialize(SupportedTestFrameworks);
+                string minDotnetTargetFramework = allTargetFrameworks.Where(t => t.Framework == ".NETCoreApp").OrderBy(t => t.Version).FirstOrDefault()?.GetShortFolderName();
+
                 foreach (var packagePath in PackagePaths)
                 {
                     Package package = NupkgParser.CreatePackageObject(packagePath);
-                    List<NuGetFramework> packageTargetFrameworks = package.PackageAssets.Select(asset => asset.TargetFramework).Where(tfm => tfm != null).Distinct().ToList();
 
-                    List<NuGetFramework> frameworksToTest = GetTestFrameworks(packageTargetFrameworks);
-                    testProjects.AddRange(CreateItemFromTestFramework(package.PackageId, package.Version, frameworksToTest, GetRidsFromPackage(package)));
+                    IEnumerable<NuGetFramework> testFrameworks = GetTestFrameworks(package, minDotnetTargetFramework);
+                    testProjects.AddRange(CreateItemFromTestFramework(package.PackageId, package.Version, testFrameworks));
                 }
-                
+
                 // Removing empty items.
                 TestProjects = testProjects.Where(tfm => tfm.ItemSpec != "").ToArray();
             }
             catch (Exception e)
             {
                 Log.LogErrorFromException(e, showStackTrace: false);
             }
-            
+
             return result && !Log.HasLoggedErrors;
         }
 
-        public static List<NuGetFramework> GetTestFrameworks(List<NuGetFramework> packageTargetFrameworks)
+        public static IEnumerable<NuGetFramework> GetTestFrameworks(Package package, string minDotnetTargetFramework)
         {
-            List<NuGetFramework> frameworksToTest = new List<NuGetFramework>();
+            List<NuGetFramework> frameworksToTest= new List<NuGetFramework>();
+            IEnumerable<NuGetFramework> packageTargetFrameworks = package.FrameworksInPackage;
 
             // Testing the package installation on all tfms linked with package targetframeworks.
             foreach (var item in packageTargetFrameworks)
             {
                 if (packageTfmMapping.ContainsKey(item))
-                    frameworksToTest.AddRange(packageTfmMapping[item].ToList());
+                    frameworksToTest.AddRange(packageTfmMapping[item]);
+                // Adding the frameworks in the packages to the test matrix.
+                frameworksToTest.Add(item);
             }
 
-            // Pruning the test matrix by removing the frameworks we dont want to test.
-            frameworksToTest = frameworksToTest.Where(tfm => allTargetFrameworks.Contains(tfm)).ToList();
+            if (!string.IsNullOrEmpty(minDotnetTargetFramework) && frameworksToTest.Any(t => t.Framework == ".NETStandard"))
+                frameworksToTest.Add(NuGetFramework.Parse(minDotnetTargetFramework));
 
-            // Adding the frameworks in the packages to the test matrix;
-            frameworksToTest.AddRange(packageTargetFrameworks);
-            frameworksToTest = frameworksToTest.Distinct().ToList();
-            return frameworksToTest;
+            return frameworksToTest.Where(tfm => allTargetFrameworks.Contains(tfm)).Distinct();
         }
 
-        public static void Initialize()
+        public static void Initialize(string targetFrameworks)
         {
             // Defining the set of known frameworks that we care to test
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetCoreApp20);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetCoreApp21);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetCoreApp30);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetCoreApp31);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net50);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net45);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net451);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net452);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net46);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net461);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net462);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.Net463);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard10);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard11);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard12);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard13);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard14);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard15);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard16);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard17);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard20);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.NetStandard21);
-            allTargetFrameworks.Add(FrameworkConstants.CommonFrameworks.UAP10);
+            foreach (var tfm in targetFrameworks.Split(';'))
+            {
+                allTargetFrameworks.Add(NuGetFramework.Parse(tfm));
+            }
 
             // creating a map framework in package => frameworks to test based on default compatibilty mapping.
             foreach (var item in DefaultFrameworkMappings.Instance.CompatibilityMappings)
@@ -111,8 +96,8 @@ public static void Initialize()
                 }
             }
         }
-    
-        public List<ITaskItem> CreateItemFromTestFramework(string packageId, string version, List<NuGetFramework> testFrameworks, string rids)
+
+        public List<ITaskItem> CreateItemFromTestFramework(string packageId, string version, IEnumerable<NuGetFramework> testFrameworks)
         {
             List<ITaskItem> testprojects = new List<ITaskItem>();
             foreach (var framework in testFrameworks)
@@ -121,42 +106,10 @@ public List<ITaskItem> CreateItemFromTestFramework(string packageId, string vers
                 supportedPackage.SetMetadata("Version", version);
                 supportedPackage.SetMetadata("TargetFramework", framework.ToString());
                 supportedPackage.SetMetadata("TargetFrameworkShort", framework.GetShortFolderName());
-
-                if (!String.IsNullOrEmpty(rids))
-                {
-                    supportedPackage.SetMetadata("RuntimeIdentifiers", rids);
-                }
                 testprojects.Add(supportedPackage);
             }
 
             return testprojects;
         }
-
-        public string GetRidsFromPackage(Package package)
-        {
-            List<string> rids = new List<string>();
-            foreach (var item in package.PackageAssets)
-            {
-                if (item.AssetType == AssetType.RuntimeAsset)
-                {
-                    string testRid = item.Rid;
-                    string testArch = testRid == "browser" ? "-wasm" : "-x64";
-                    if (testRid == "unix")
-                    {
-                        if (!rids.Contains("linux" + testArch))
-                            rids.Add("linux" + testArch);
-
-                        if (!rids.Contains("osx" + testArch))
-                            rids.Add("osx" + testArch);
-                    }
-                    else
-                    {
-                        if (!rids.Contains(testRid + testArch))
-                            rids.Add(testRid + testArch);
-                    }             
-                }
-            }
-            return string.Join(";", rids);
-        }
-    }    
+    }
 }