Skip to content

Commit

Permalink
Implement GetArtifactDownloadUris in the BaseProjectManager (#329)
Browse files Browse the repository at this point in the history
* Implement GetArtifactDownloadUris in the BaseProjectManager to get any relevant URIs associated with a package version.
Implement it in NPM, NuGet and PyPI.
Create tests for it, and create a new PyPIProjectManagerTests class.
Fix scoped NPM name handling. Scoped npm purls should have the namespace prefixed with %40.

* Create an ArtifactUri struct that contains the Type, Uri, and Extension. This gets used instead of string for the IEnumerable returned in GetArtifactDownloadUris.

* Added remarks to PackageUrlExtension.HasNamespace to explain that it will just return the given namespace if it isn't an NPM package.

* Added a remark in BaseProjectManager.GetArtifactDownloadUris that clarify that this method doesn't check that the returning URI(s) actually exist.

* Added a generic to ArtifactUri<T> where T is the enum for the artifact type.
TypedManager takes in another generic `TArtifactUriType` that has to be an enum.
Create a NoOpPackageActions to use with NPM and PyPI as we don't have PackageActions implementations for them yet.
Remove GetArtifactDownloadUris from BaseProjectManager and moved it to the TypedManager.
NPM and PyPI now implement TypedManager.

* Remove extension from ArtifactUri constructor as it wasn't being used anyways. It now gets calculated from the Uri.AbsolutePath
Added the nuspecUri for NuGet packages as well.
Added UriExistsAsync logic to TypedManager. 

* Created a new Extensions helper "UriExtension" that has a method GetExtension in it, so that way we can get special extensions such as ".tar.gz" and ".tar.bz2" because the Path.GetExtension method would only return ".gz" or ".bz2" respectively.

* Add a TODO to track merging the ArtifactUriType into the PackageVersionMetadata.
  • Loading branch information
jpinz authored Apr 22, 2022
1 parent 06c4497 commit 05d7ad0
Show file tree
Hide file tree
Showing 21 changed files with 530 additions and 46 deletions.
32 changes: 26 additions & 6 deletions src/Shared/Extensions/PackageUrlExtension.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ namespace Microsoft.CST.OpenSource.Extensions;
using System.Text.RegularExpressions;
using PackageUrl;
using System;
using System.Net;

public static class PackageUrlExtension
{
Expand Down Expand Up @@ -40,6 +41,27 @@ public static bool HasNamespace(this PackageURL packageUrl)
return packageUrl.Namespace.IsNotBlank();
}

/// <summary>
/// We want npm package's namespace to be prefixed with "%40" the percent code of "@".
/// </summary>
/// <remarks>If the <paramref name="packageUrl"/> isn't npm, it returns the namespace with no changes.</remarks>
/// <param name="packageUrl">The <see cref="PackageURL"/> to get a formatted namespace for.</param>
/// <returns>The formatted namespace of <paramref name="packageUrl"/>.</returns>
public static string GetNamespaceFormatted(this PackageURL packageUrl)
{
if (packageUrl.Type != "npm" || packageUrl.Namespace.StartsWith("%40"))
{
return packageUrl.Namespace;
}

if (packageUrl.Namespace.StartsWith("@"))
{
return $"%40{packageUrl.Namespace.TrimStart('@')}";
}

return $"%40{packageUrl.Namespace}";
}

/// <summary>
/// Gets the package's full name including namespace if applicable.
/// </summary>
Expand All @@ -53,19 +75,17 @@ public static bool HasNamespace(this PackageURL packageUrl)
/// as it contains the namespace if there is one.
/// </remarks>
/// <param name="packageUrl">The <see cref="PackageURL"/> to get the full name for.</param>
/// <param name="encoded">If the name should be url encoded, defaults to false.</param>
/// <returns>The full name.</returns>
public static string GetFullName(this PackageURL packageUrl)
public static string GetFullName(this PackageURL packageUrl, bool encoded = false)
{
if (!packageUrl.HasNamespace())
{
return packageUrl.Name;
}

string name = $"{packageUrl.GetNamespaceFormatted()}/{packageUrl.Name}";
// The full name for scoped npm packages should have an '@' at the beginning.
string? namespaceStr = packageUrl.Type.Equals("npm", StringComparison.OrdinalIgnoreCase)
? $"@{packageUrl.Namespace}"
: packageUrl.Namespace;
return $"{namespaceStr}/{packageUrl.Name}";

return encoded ? name : WebUtility.UrlDecode(name);
}
}
31 changes: 31 additions & 0 deletions src/Shared/Extensions/UriExtension.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.

namespace Microsoft.CST.OpenSource.Extensions;

using System;
using System.IO;

public static class UriExtension
{
private static readonly string[] SpecialExtensions = { ".tar.gz", ".tar.bz2" };

/// <summary>
/// Gets the extension from a <see cref="Uri.AbsolutePath"/>.
/// </summary>
/// <param name="uri">The <see cref="Uri"/> to get the extension from.</param>
/// <returns>The extension from the <paramref name="uri"/>, or an empty string if it doesn't have one.</returns>
public static string GetExtension(this Uri uri)
{
string absolutePath = uri.AbsolutePath;

foreach (string specialExtension in SpecialExtensions)
{
if (absolutePath.EndsWith(specialExtension))
{
return specialExtension;
}
}

return Path.GetExtension(absolutePath);
}
}
20 changes: 20 additions & 0 deletions src/Shared/Helpers/StringExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,25 @@ public static bool IsNotBlank([NotNullWhen(true)]this string? str)
{
return !str.IsBlank();
}

/// <summary>
/// Returns the input string with a slash ('/') appended to it, unless there was already '/' at its end.
/// </summary>
/// <remarks>If the string is empty, no changes are made.</remarks>
public static string EnsureTrailingSlash(this string url)
{
if (url.IsBlank())
{
return url;
}

url = url.TrimEnd(' ');
if (!url.EndsWith('/'))
{
return url + '/';
}

return url;
}
}
}
48 changes: 48 additions & 0 deletions src/Shared/Model/ArtifactUri.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.

namespace Microsoft.CST.OpenSource.Model;

using Extensions;
using System;
using System.IO;

/// <summary>
/// A record to represent the type, uri, and extension for an artifact associated with a package.
/// </summary>
/// <typeparam name="T">The enum to represent the artifact type.</typeparam>
public record ArtifactUri<T> where T : Enum
{
/// <summary>
/// Initializes a new instance of <see cref="ArtifactUri{T}"/>.
/// </summary>
/// <param name="type">The type of artifact for this <see cref="ArtifactUri{T}"/>.</param>
/// <param name="uri">The <see cref="Uri"/> this artifact can be found at.</param>
public ArtifactUri(T type, Uri uri)
{
Type = type;
Uri = uri;
}

/// <summary>
/// Initializes a new instance of <see cref="ArtifactUri{T}"/>.
/// </summary>
/// <param name="type">The type of artifact for this <see cref="ArtifactUri{T}"/>.</param>
/// <param name="uri">The string of the uri this artifact can be found at.</param>
public ArtifactUri(T type, string uri) : this(type, new Uri(uri)) { }

/// <summary>
/// The enum representing the artifact type for the project manager associated with this artifact.
/// </summary>
public T Type { get; }

/// <summary>
/// The <see cref="Uri"/> for where this artifact can be found online.
/// </summary>
public Uri Uri { get; }

/// <summary>
/// The file extension for this artifact file. Including the '.' at the beginning.
/// </summary>
/// <remarks>If the file has no extension, it will just be an empty string.</remarks>
public string Extension => Uri.GetExtension();
}
43 changes: 43 additions & 0 deletions src/Shared/PackageActions/NoOpPackageActions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
// Copyright (c) Microsoft Corporation. Licensed under the MIT License.

namespace Microsoft.CST.OpenSource.PackageActions;

using Contracts;
using PackageUrl;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;

public class NoOpPackageActions : IManagerPackageActions<IManagerPackageVersionMetadata>
{
public Task<string?> DownloadAsync(
PackageURL packageUrl,
string topLevelDirectory,
string targetPath,
bool doExtract,
bool cached = false,
CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);

public Task<bool> DoesPackageExistAsync(
PackageURL packageUrl,
bool useCache = true,
CancellationToken cancellationToken = default) => Task.FromResult(false);

public Task<IEnumerable<string>> GetAllVersionsAsync(
PackageURL packageUrl,
bool includePrerelease = true,
bool useCache = true,
CancellationToken cancellationToken = default) => Task.FromResult(Enumerable.Empty<string>());

public Task<string?> GetLatestVersionAsync(
PackageURL packageUrl,
bool includePrerelease = false,
bool useCache = true,
CancellationToken cancellationToken = default) => Task.FromResult<string?>(null);

public Task<IManagerPackageVersionMetadata?> GetMetadataAsync(
PackageURL packageUrl,
bool useCache = true,
CancellationToken cancellationToken = default) => Task.FromResult<IManagerPackageVersionMetadata?>(null);
}
4 changes: 3 additions & 1 deletion src/Shared/PackageManagers/BaseProjectManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

namespace Microsoft.CST.OpenSource.PackageManagers
{
using Contracts;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.CST.OpenSource.Model;
using System;
Expand All @@ -14,6 +15,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
using Utilities;
using Version = SemanticVersioning.Version;
using PackageUrl;
using System.Net;

public abstract class BaseProjectManager
{
Expand Down Expand Up @@ -274,7 +276,7 @@ public static IEnumerable<string> SortVersions(IEnumerable<string> versionList)
/// <returns>Paths (either files or directory names) pertaining to the downloaded files.</returns>
public virtual Task<IEnumerable<string>> DownloadVersionAsync(PackageURL purl, bool doExtract, bool cached = false)
{
throw new NotImplementedException("BaseProjectManager does not implement DownloadVersion.");
throw new NotImplementedException("BaseProjectManager does not implement DownloadVersionAsync.");
}

/// <summary>
Expand Down
32 changes: 26 additions & 6 deletions src/Shared/PackageManagers/NPMProjectManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

namespace Microsoft.CST.OpenSource.PackageManagers
{
using Contracts;
using Extensions;
using Helpers;
using Microsoft.CST.OpenSource.Model;
using PackageActions;
using PackageUrl;
using System;
using System.Collections.Generic;
Expand All @@ -16,7 +18,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
using Utilities;
using Version = SemanticVersioning.Version;

public class NPMProjectManager : BaseProjectManager
public class NPMProjectManager : TypedManager<IManagerPackageVersionMetadata, NPMProjectManager.NPMArtifactType>
{
/// <summary>
/// The type of the project manager from the package-url type specifications.
Expand All @@ -29,12 +31,23 @@ public class NPMProjectManager : BaseProjectManager
public static string ENV_NPM_API_ENDPOINT { get; set; } = "https://registry.npmjs.org";
public static string ENV_NPM_ENDPOINT { get; set; } = "https://www.npmjs.com";

public NPMProjectManager(IHttpClientFactory httpClientFactory, string destinationDirectory) : base(httpClientFactory, destinationDirectory)
public NPMProjectManager(
string directory,
IManagerPackageActions<IManagerPackageVersionMetadata>? actions = null,
IHttpClientFactory? httpClientFactory = null)
: base(actions ?? new NoOpPackageActions(), httpClientFactory ?? new DefaultHttpClientFactory(), directory)
{
}

public NPMProjectManager(string destinationDirectory) : base(destinationDirectory)
/// <inheritdoc />
public override IEnumerable<ArtifactUri<NPMArtifactType>> GetArtifactDownloadUris(PackageURL purl)
{
string feedUrl = (purl.Qualifiers?["repository_url"] ?? ENV_NPM_API_ENDPOINT).EnsureTrailingSlash();

string artifactUri = purl.HasNamespace() ?
$"{feedUrl}{purl.GetNamespaceFormatted()}/{purl.Name}/-/{purl.Name}-{purl.Version}.tgz" : // If there's a namespace.
$"{feedUrl}{purl.Name}/-/{purl.Name}-{purl.Version}.tgz"; // If there isn't a namespace.
yield return new ArtifactUri<NPMArtifactType>(NPMArtifactType.Tarball, artifactUri);
}

/// <summary>
Expand Down Expand Up @@ -119,7 +132,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
string packageName = purl.GetFullName();
HttpClient httpClient = CreateHttpClient();

JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_NPM_API_ENDPOINT}/{packageName}", useCache);
JsonDocument doc = await GetJsonCache(httpClient, $"{ENV_NPM_API_ENDPOINT}/{purl.GetFullName(encoded: true)}", useCache);

List<string> versionList = new();

Expand Down Expand Up @@ -175,7 +188,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
{
try
{
string? packageName = purl.Namespace != null ? $"@{purl.Namespace}/{purl.Name}" : purl.Name;
string? packageName = purl.HasNamespace() ? $"{purl.GetNamespaceFormatted()}/{purl.Name}" : purl.Name;
HttpClient httpClient = CreateHttpClient();

string? content = await GetHttpStringCache(httpClient, $"{ENV_NPM_API_ENDPOINT}/{packageName}", useCache);
Expand All @@ -190,7 +203,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR

public override Uri GetPackageAbsoluteUri(PackageURL purl)
{
return new Uri($"{ENV_NPM_API_ENDPOINT}/{purl?.Name}");
return new Uri(ENV_NPM_API_ENDPOINT.EnsureTrailingSlash() + (purl.HasNamespace() ? $"{purl.GetNamespaceFormatted()}/{purl.Name}" : purl.Name));
}

/// <inheritdoc />
Expand Down Expand Up @@ -547,5 +560,12 @@ protected async Task<Dictionary<PackageURL, double>> SearchRepoUrlsInPackageMeta
"vm",
"zlib"
};

public enum NPMArtifactType
{
Unknown = 0,
Tarball,
PackageJson,
}
}
}
21 changes: 20 additions & 1 deletion src/Shared/PackageManagers/NuGetProjectManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace Microsoft.CST.OpenSource.PackageManagers
{
using Contracts;
using Helpers;
using PackageUrl;
using Model;
using Model.Metadata;
Expand All @@ -18,7 +19,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
using System.Text.Json;
using System.Threading.Tasks;

public class NuGetProjectManager : TypedManager<NuGetPackageVersionMetadata>
public class NuGetProjectManager : TypedManager<NuGetPackageVersionMetadata, NuGetProjectManager.NuGetArtifactType>
{
/// <summary>
/// The type of the project manager from the package-url type specifications.
Expand All @@ -44,6 +45,17 @@ public NuGetProjectManager(
{
GetRegistrationEndpointAsync().Wait();
}

/// <inheritdoc />
public override IEnumerable<ArtifactUri<NuGetArtifactType>> GetArtifactDownloadUris(PackageURL purl)
{
string feedUrl = (purl.Qualifiers?["repository_url"] ?? NUGET_DEFAULT_CONTENT_ENDPOINT).EnsureTrailingSlash();

string nupkgUri = $"{feedUrl}{purl.Name.ToLower()}/{purl.Version}/{purl.Name.ToLower()}.{purl.Version}.nupkg";
yield return new ArtifactUri<NuGetArtifactType>(NuGetArtifactType.Nupkg, nupkgUri);
string nuspecUri = $"{feedUrl}{purl.Name.ToLower()}/{purl.Version}/{purl.Name.ToLower()}.nuspec";
yield return new ArtifactUri<NuGetArtifactType>(NuGetArtifactType.Nuspec, nuspecUri);
}

/// <summary>
/// Dynamically identifies the registration endpoint.
Expand Down Expand Up @@ -342,5 +354,12 @@ protected override async Task<Dictionary<PackageURL, double>> SearchRepoUrlsInPa

[System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "IDE0044:Add readonly modifier", Justification = "Modified through reflection.")]
private static string ENV_NUGET_HOMEPAGE = "https://www.nuget.org/packages";

public enum NuGetArtifactType
{
Unknown = 0,
Nupkg,
Nuspec,
}
}
}
4 changes: 2 additions & 2 deletions src/Shared/PackageManagers/ProjectManagerFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -101,15 +101,15 @@ public static Dictionary<string, ConstructProjectManager> GetDefaultManagers(IHt
},
{
NPMProjectManager.Type, destinationDirectory =>
new NPMProjectManager(httpClientFactory, destinationDirectory)
new NPMProjectManager(destinationDirectory, new NoOpPackageActions(), httpClientFactory)
},
{
NuGetProjectManager.Type, destinationDirectory =>
new NuGetProjectManager(destinationDirectory, new NuGetPackageActions(), httpClientFactory) // Add the NuGetPackageActions to the NuGetProjectManager.
},
{
PyPIProjectManager.Type, destinationDirectory =>
new PyPIProjectManager(httpClientFactory, destinationDirectory)
new PyPIProjectManager(destinationDirectory, new NoOpPackageActions(), httpClientFactory)
},
{
UbuntuProjectManager.Type, destinationDirectory =>
Expand Down
Loading

0 comments on commit 05d7ad0

Please sign in to comment.