Implement GetArtifactDownloadUris in the BaseProjectManager (#329)

* Implement GetArtifactDownloadUris in the BaseProjectManager to get any relevant URIs associated with a package version.
Implement it in NPM, NuGet and PyPI.
Create tests for it, and create a new PyPIProjectManagerTests class.
Fix scoped NPM name handling. Scoped npm purls should have the namespace prefixed with %40.

* Create an ArtifactUri struct that contains the Type, Uri, and Extension. This gets used instead of string for the IEnumerable returned in GetArtifactDownloadUris.

* Added remarks to PackageUrlExtension.HasNamespace to explain that it will just return the given namespace if it isn't an NPM package.

* Added a remark in BaseProjectManager.GetArtifactDownloadUris that clarify that this method doesn't check that the returning URI(s) actually exist.

* Added a generic to ArtifactUri<T> where T is the enum for the artifact type.
TypedManager takes in another generic `TArtifactUriType` that has to be an enum.
Create a NoOpPackageActions to use with NPM and PyPI as we don't have PackageActions implementations for them yet.
Remove GetArtifactDownloadUris from BaseProjectManager and moved it to the TypedManager.
NPM and PyPI now implement TypedManager.

* Remove extension from ArtifactUri constructor as it wasn't being used anyways. It now gets calculated from the Uri.AbsolutePath
Added the nuspecUri for NuGet packages as well.
Added UriExistsAsync logic to TypedManager. 

* Created a new Extensions helper "UriExtension" that has a method GetExtension in it, so that way we can get special extensions such as ".tar.gz" and ".tar.bz2" because the Path.GetExtension method would only return ".gz" or ".bz2" respectively.

* Add a TODO to track merging the ArtifactUriType into the PackageVersionMetadata.

src/Shared/Extensions/PackageUrlExtension.cs

@@ -7,6 +7,7 @@ namespace Microsoft.CST.OpenSource.Extensions;
 using System.Text.RegularExpressions;
 using PackageUrl;
 using System;
+using System.Net;
 
 public static class PackageUrlExtension
 {
@@ -40,6 +41,27 @@ public static bool HasNamespace(this PackageURL packageUrl)
         return packageUrl.Namespace.IsNotBlank();
     }
 
+    /// <summary>
+    /// We want npm package's namespace to be prefixed with "%40" the percent code of "@".
+    /// </summary>
+    /// <remarks>If the <paramref name="packageUrl"/> isn't npm, it returns the namespace with no changes.</remarks>
+    /// <param name="packageUrl">The <see cref="PackageURL"/> to get a formatted namespace for.</param>
+    /// <returns>The formatted namespace of <paramref name="packageUrl"/>.</returns>
+    public static string GetNamespaceFormatted(this PackageURL packageUrl)
+    {
+        if (packageUrl.Type != "npm" || packageUrl.Namespace.StartsWith("%40"))
+        {
+            return packageUrl.Namespace;
+        }
+
+        if (packageUrl.Namespace.StartsWith("@"))
+        {
+            return $"%40{packageUrl.Namespace.TrimStart('@')}";
+        }
+
+        return $"%40{packageUrl.Namespace}";
+    }
+
     /// <summary>
     /// Gets the package's full name including namespace if applicable.
     /// </summary>
@@ -53,19 +75,17 @@ public static bool HasNamespace(this PackageURL packageUrl)
     /// as it contains the namespace if there is one.
     /// </remarks>
     /// <param name="packageUrl">The <see cref="PackageURL"/> to get the full name for.</param>
+    /// <param name="encoded">If the name should be url encoded, defaults to false.</param>
     /// <returns>The full name.</returns>
-    public static string GetFullName(this PackageURL packageUrl)
+    public static string GetFullName(this PackageURL packageUrl, bool encoded = false)
     {
         if (!packageUrl.HasNamespace())
         {
             return packageUrl.Name;
         }
 
+        string name = $"{packageUrl.GetNamespaceFormatted()}/{packageUrl.Name}";
         // The full name for scoped npm packages should have an '@' at the beginning.
-        string? namespaceStr = packageUrl.Type.Equals("npm", StringComparison.OrdinalIgnoreCase)
-            ? $"@{packageUrl.Namespace}"
-            : packageUrl.Namespace;
-        return $"{namespaceStr}/{packageUrl.Name}";
-
+        return encoded ? name : WebUtility.UrlDecode(name);
     }
 }

src/Shared/Extensions/UriExtension.cs

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.Extensions;
+
+using System;
+using System.IO;
+
+public static class UriExtension
+{
+    private static readonly string[] SpecialExtensions = { ".tar.gz", ".tar.bz2" };
+
+    /// <summary>
+    /// Gets the extension from a <see cref="Uri.AbsolutePath"/>.
+    /// </summary>
+    /// <param name="uri">The <see cref="Uri"/> to get the extension from.</param>
+    /// <returns>The extension from the <paramref name="uri"/>, or an empty string if it doesn't have one.</returns>
+    public static string GetExtension(this Uri uri)
+    {
+        string absolutePath = uri.AbsolutePath;
+
+        foreach (string specialExtension in SpecialExtensions)
+        {
+            if (absolutePath.EndsWith(specialExtension))
+            {
+                return specialExtension;
+            }
+        }
+
+        return Path.GetExtension(absolutePath);
+    }
+}

src/Shared/Helpers/StringExtensions.cs

@@ -41,5 +41,25 @@ public static bool IsNotBlank([NotNullWhen(true)]this string? str)
         {
             return !str.IsBlank();
         }
+
+        /// <summary>
+        /// Returns the input string with a slash ('/') appended to it, unless there was already '/' at its end.
+        /// </summary>
+        /// <remarks>If the string is empty, no changes are made.</remarks>
+        public static string EnsureTrailingSlash(this string url)
+        {
+            if (url.IsBlank())
+            {
+                return url;
+            }
+
+            url = url.TrimEnd(' ');
+            if (!url.EndsWith('/'))
+            {
+                return url + '/';
+            }
+
+            return url;
+        }
     }
 }

src/Shared/Model/ArtifactUri.cs

@@ -0,0 +1,48 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.Model;
+
+using Extensions;
+using System;
+using System.IO;
+
+/// <summary>
+/// A record to represent the type, uri, and extension for an artifact associated with a package.
+/// </summary>
+/// <typeparam name="T">The enum to represent the artifact type.</typeparam>
+public record ArtifactUri<T> where T : Enum
+{
+    /// <summary>
+    /// Initializes a new instance of <see cref="ArtifactUri{T}"/>.
+    /// </summary>
+    /// <param name="type">The type of artifact for this <see cref="ArtifactUri{T}"/>.</param>
+    /// <param name="uri">The <see cref="Uri"/> this artifact can be found at.</param>
+    public ArtifactUri(T type, Uri uri)
+    {
+        Type = type;
+        Uri = uri;
+    }
+
+    /// <summary>
+    /// Initializes a new instance of <see cref="ArtifactUri{T}"/>.
+    /// </summary>
+    /// <param name="type">The type of artifact for this <see cref="ArtifactUri{T}"/>.</param>
+    /// <param name="uri">The string of the uri this artifact can be found at.</param>
+    public ArtifactUri(T type, string uri) : this(type, new Uri(uri)) { }
+
+    /// <summary>
+    /// The enum representing the artifact type for the project manager associated with this artifact.
+    /// </summary>
+    public T Type { get; }
+
+    /// <summary>
+    /// The <see cref="Uri"/> for where this artifact can be found online.
+    /// </summary>
+    public Uri Uri { get; }
+
+    /// <summary>
+    /// The file extension for this artifact file. Including the '.' at the beginning.
+    /// </summary>
+    /// <remarks>If the file has no extension, it will just be an empty string.</remarks>
+    public string Extension => Uri.GetExtension();
+}

src/Shared/PackageActions/NoOpPackageActions.cs

@@ -0,0 +1,43 @@
+// Copyright (c) Microsoft Corporation. Licensed under the MIT License.
+
+namespace Microsoft.CST.OpenSource.PackageActions;
+
+using Contracts;
+using PackageUrl;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+public class NoOpPackageActions : IManagerPackageActions<IManagerPackageVersionMetadata>
+{
+    public Task<string?> DownloadAsync(
+        PackageURL packageUrl,
+        string topLevelDirectory,
+        string targetPath,
+        bool doExtract,
+        bool cached = false,
+        CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);
+
+    public Task<bool> DoesPackageExistAsync(
+        PackageURL packageUrl,
+        bool useCache = true,
+        CancellationToken cancellationToken = default) => Task.FromResult(false);
+
+    public Task<IEnumerable<string>> GetAllVersionsAsync(
+        PackageURL packageUrl,
+        bool includePrerelease = true,
+        bool useCache = true,
+        CancellationToken cancellationToken = default) => Task.FromResult(Enumerable.Empty<string>());
+
+    public Task<string?> GetLatestVersionAsync(
+        PackageURL packageUrl,
+        bool includePrerelease = false,
+        bool useCache = true,
+        CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);
+
+    public Task<IManagerPackageVersionMetadata?> GetMetadataAsync(
+        PackageURL packageUrl,
+        bool useCache = true,
+        CancellationToken cancellationToken = default) => Task.FromResult<IManagerPackageVersionMetadata?>(null);
+}

src/Shared/PackageManagers/BaseProjectManager.cs

@@ -2,6 +2,7 @@
 
 namespace Microsoft.CST.OpenSource.PackageManagers
 {
+    using Contracts;
     using Microsoft.Extensions.Caching.Memory;
     using Microsoft.CST.OpenSource.Model;
     using System;
@@ -14,6 +15,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using Utilities;
     using Version = SemanticVersioning.Version;
     using PackageUrl;
+    using System.Net;
 
     public abstract class BaseProjectManager
     {
@@ -274,7 +276,7 @@ public static IEnumerable<string> SortVersions(IEnumerable<string> versionList)
         /// <returns>Paths (either files or directory names) pertaining to the downloaded files.</returns>
         public virtual Task<IEnumerable<string>> DownloadVersionAsync(PackageURL purl, bool doExtract, bool cached = false)
         {
-            throw new NotImplementedException("BaseProjectManager does not implement DownloadVersion.");
+            throw new NotImplementedException("BaseProjectManager does not implement DownloadVersionAsync.");
         }
 
         /// <summary>

src/Shared/PackageManagers/NPMProjectManager.cs

@@ -2,9 +2,11 @@
 
 namespace Microsoft.CST.OpenSource.PackageManagers
 {
+    using Contracts;
     using Extensions;
     using Helpers;
     using Microsoft.CST.OpenSource.Model;
+    using PackageActions;
     using PackageUrl;
     using System;
     using System.Collections.Generic;
@@ -16,7 +18,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using Utilities;
     using Version = SemanticVersioning.Version;
 
-    public class NPMProjectManager : BaseProjectManager
+    public class NPMProjectManager : TypedManager<IManagerPackageVersionMetadata, NPMProjectManager.NPMArtifactType>
     {
         /// <summary>
         /// The type of the project manager from the package-url type specifications.
@@ -29,12 +31,23 @@ public class NPMProjectManager : BaseProjectManager
         public static string ENV_NPM_API_ENDPOINT { get; set; } = "https://registry.npmjs.org";
         public static string ENV_NPM_ENDPOINT { get; set; } = "https://www.npmjs.com";
 
-        public NPMProjectManager(IHttpClientFactory httpClientFactory, string destinationDirectory) : base(httpClientFactory, destinationDirectory)
+        public NPMProjectManager(
+            string directory,
+            IManagerPackageActions<IManagerPackageVersionMetadata>? actions = null,
+            IHttpClientFactory? httpClientFactory = null)
+            : base(actions ?? new NoOpPackageActions(), httpClientFactory ?? new DefaultHttpClientFactory(), directory)
         {
         }
 
-        public NPMProjectManager(string destinationDirectory) : base(destinationDirectory)
+        /// <inheritdoc />
+        public override IEnumerable<ArtifactUri<NPMArtifactType>> GetArtifactDownloadUris(PackageURL purl)
         {
+            string feedUrl = (purl.Qualifiers?["repository_url"] ?? ENV_NPM_API_ENDPOINT).EnsureTrailingSlash();
+
+            string artifactUri = purl.HasNamespace() ? 
+                $"{feedUrl}{purl.GetNamespaceFormatted()}/{purl.Name}/-/{purl.Name}-{purl.Version}.tgz" : // If there's a namespace.
+                $"{feedUrl}{purl.Name}/-/{purl.Name}-{purl.Version}.tgz"; // If there isn't a namespace.
+            yield return new ArtifactUri<NPMArtifactType>(NPMArtifactType.Tarball, artifactUri);
         }
 
         /// <summary>
@@ -119,7 +132,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 string packageName = purl.GetFullName();
                 HttpClient httpClient = CreateHttpClient();
 
-                JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_NPM_API_ENDPOINT}/{packageName}", useCache);
+                JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_NPM_API_ENDPOINT}/{purl.GetFullName(encoded: true)}", useCache);
 
                 List<string> versionList = new();
 
@@ -175,7 +188,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
         {
             try
             {
-                string? packageName = purl.Namespace != null ? $"@{purl.Namespace}/{purl.Name}" : purl.Name;
+                string? packageName = purl.HasNamespace() ? $"{purl.GetNamespaceFormatted()}/{purl.Name}" : purl.Name;
                 HttpClient httpClient = CreateHttpClient();
 
                 string? content = await GetHttpStringCache(httpClient, $"{ENV_NPM_API_ENDPOINT}/{packageName}", useCache);
@@ -190,7 +203,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
 
         public override Uri GetPackageAbsoluteUri(PackageURL purl)
         {
-            return new Uri($"{ENV_NPM_API_ENDPOINT}/{purl?.Name}");
+            return new Uri(ENV_NPM_API_ENDPOINT.EnsureTrailingSlash() + (purl.HasNamespace() ? $"{purl.GetNamespaceFormatted()}/{purl.Name}" : purl.Name));
         }
 
         /// <inheritdoc />
@@ -547,5 +560,12 @@ protected async Task<Dictionary<PackageURL, double>> SearchRepoUrlsInPackageMeta
             "vm",
             "zlib"
         };
+
+        public enum NPMArtifactType
+        {
+            Unknown = 0,
+            Tarball,
+            PackageJson,
+        }
     }
 }

src/Shared/PackageManagers/NuGetProjectManager.cs

@@ -3,6 +3,7 @@
 namespace Microsoft.CST.OpenSource.PackageManagers
 {
     using Contracts;
+    using Helpers;
     using PackageUrl;
     using Model;
     using Model.Metadata;
@@ -18,7 +19,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Text.Json;
     using System.Threading.Tasks;
 
-    public class NuGetProjectManager : TypedManager<NuGetPackageVersionMetadata>
+    public class NuGetProjectManager : TypedManager<NuGetPackageVersionMetadata, NuGetProjectManager.NuGetArtifactType>
     {
         /// <summary>
         /// The type of the project manager from the package-url type specifications.
@@ -44,6 +45,17 @@ public NuGetProjectManager(
         {
             GetRegistrationEndpointAsync().Wait();
         }
+
+        /// <inheritdoc />
+        public override IEnumerable<ArtifactUri<NuGetArtifactType>> GetArtifactDownloadUris(PackageURL purl)
+        {
+            string feedUrl = (purl.Qualifiers?["repository_url"] ?? NUGET_DEFAULT_CONTENT_ENDPOINT).EnsureTrailingSlash();
+
+            string nupkgUri = $"{feedUrl}{purl.Name.ToLower()}/{purl.Version}/{purl.Name.ToLower()}.{purl.Version}.nupkg";
+            yield return new ArtifactUri<NuGetArtifactType>(NuGetArtifactType.Nupkg, nupkgUri);
+            string nuspecUri = $"{feedUrl}{purl.Name.ToLower()}/{purl.Version}/{purl.Name.ToLower()}.nuspec";
+            yield return new ArtifactUri<NuGetArtifactType>(NuGetArtifactType.Nuspec, nuspecUri);
+        }
 
         /// <summary>
         /// Dynamically identifies the registration endpoint.
@@ -342,5 +354,12 @@ protected override async Task<Dictionary<PackageURL, double>> SearchRepoUrlsInPa
 
         [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0044:Add readonly modifier", Justification = "Modified through reflection.")]
         private static string ENV_NUGET_HOMEPAGE = "https://www.nuget.org/packages";
+
+        public enum NuGetArtifactType
+        {
+            Unknown = 0,
+            Nupkg,
+            Nuspec,
+        }
     }
 }

src/Shared/PackageManagers/ProjectManagerFactory.cs

@@ -101,15 +101,15 @@ public static Dictionary<string, ConstructProjectManager> GetDefaultManagers(IHt
                 },
                 {
                     NPMProjectManager.Type, destinationDirectory =>
-                        new NPMProjectManager(httpClientFactory, destinationDirectory)
+                        new NPMProjectManager(destinationDirectory, new NoOpPackageActions(), httpClientFactory)
                 },
                 {
                     NuGetProjectManager.Type, destinationDirectory =>
                         new NuGetProjectManager(destinationDirectory, new NuGetPackageActions(), httpClientFactory) // Add the NuGetPackageActions to the NuGetProjectManager.
                 },
                 {
                     PyPIProjectManager.Type, destinationDirectory =>
-                        new PyPIProjectManager(httpClientFactory, destinationDirectory)
+                        new PyPIProjectManager(destinationDirectory, new NoOpPackageActions(), httpClientFactory)
                 },
                 {
                     UbuntuProjectManager.Type, destinationDirectory =>