diff --git a/src/Shared/Model/PackageMetadata.cs b/src/Shared/Model/PackageMetadata.cs index 2925e538..671bd520 100644 --- a/src/Shared/Model/PackageMetadata.cs +++ b/src/Shared/Model/PackageMetadata.cs @@ -3,6 +3,7 @@ namespace Microsoft.CST.OpenSource.Model { using Newtonsoft.Json; + using System; using System.Collections.Generic; public class Downloads @@ -111,7 +112,7 @@ public class PackageMetadata public long? Size { get; set; } [JsonProperty(PropertyName = "upload_time", NullValueHandling = NullValueHandling.Ignore)] - public string? UploadTime { get; set; } + public DateTime? UploadTime { get; set; } [JsonProperty(PropertyName = "commit_id", NullValueHandling = NullValueHandling.Ignore)] public string? CommitId { get; set; } diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index a018b723..834b246a 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -170,6 +170,19 @@ public override async Task> EnumerateVersionsAsync(PackageUR } } + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + Check.NotNull(nameof(purl.Version), purl.Version); + DateTime? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime; + } + /// /// Gets the latest version of the package /// @@ -239,11 +252,21 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl) metadata.PackageVersion = latestVersion is null ? purl.Version : latestVersion?.ToString(); } - // if we found any version at all, get the deets + // if we found any version at all, get the information if (metadata.PackageVersion != null) { Version versionToGet = new(metadata.PackageVersion); JsonElement? versionElement = GetVersionElement(contentJSON, versionToGet); + + if (root.TryGetProperty("time", out JsonElement time)) + { + string? uploadTime = OssUtilities.GetJSONPropertyStringIfExists(time, metadata.PackageVersion); + if (uploadTime != null) + { + metadata.UploadTime = DateTime.Parse(uploadTime); + } + } + if (versionElement != null) { // redo the generic values to version specific values diff --git a/src/Shared/PackageManagers/NuGetProjectManager.cs b/src/Shared/PackageManagers/NuGetProjectManager.cs index be1fd03a..833a6e26 100644 --- a/src/Shared/PackageManagers/NuGetProjectManager.cs +++ b/src/Shared/PackageManagers/NuGetProjectManager.cs @@ -102,6 +102,19 @@ private async Task GetRegistrationEndpointAsync() return RegistrationEndpoint; } + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + Check.NotNull(nameof(purl.Version), purl.Version); + DateTime? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime; + } + /// public override async Task GetMetadataAsync(PackageURL purl, bool useCache = true) { @@ -233,7 +246,7 @@ private async Task UpdateVersionMetadata(PackageMetadata metadata, NuGetPackageV } // publishing info - metadata.UploadTime = packageVersionPackageVersionMetadata.Published?.ToString("MM/dd/yy HH:mm:ss zz"); + metadata.UploadTime = packageVersionPackageVersionMetadata.Published?.DateTime; } /// diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index 799809fc..e0335670 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -187,6 +187,19 @@ public override async Task> EnumerateVersionsAsync(PackageUR } } + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + Check.NotNull(nameof(purl.Version), purl.Version); + DateTime? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime; + } + public override async Task GetMetadataAsync(PackageURL purl, bool useCache = true) { try @@ -209,7 +222,6 @@ public override async Task> EnumerateVersionsAsync(PackageUR string? content = await GetMetadataAsync(purl, useCache); if (string.IsNullOrEmpty(content)) { return null; } - // convert NPM package data to normalized form JsonDocument contentJSON = JsonDocument.Parse(content); JsonElement root = contentJSON.RootElement; @@ -218,6 +230,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR metadata.Name = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "name"); metadata.Description = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "summary"); // Summary is the short description. Description is usually the readme. + metadata.LatestPackageVersion = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "version"); // Ran in the root, always points to latest version. + metadata.PackageManagerUri = ENV_PYPI_ENDPOINT; metadata.PackageUri = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "package_url"); metadata.Keywords = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(infoElement, "keywords")); @@ -266,19 +280,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR }); } - // get the version - List versions = GetVersions(contentJSON); - Version? latestVersion = GetLatestVersion(versions); - - if (purl.Version != null) - { - // find the version object from the collection - metadata.PackageVersion = purl.Version; - } - else - { - metadata.PackageVersion = latestVersion is null ? purl.Version : latestVersion?.ToString(); - } + // get the version, either use the provided one, or if null then use the LatestPackageVersion. + metadata.PackageVersion = purl.Version ?? metadata.LatestPackageVersion; // if we found any version at all, get the information. if (metadata.PackageVersion is not null) @@ -322,10 +325,15 @@ public override async Task> EnumerateVersionsAsync(PackageUR } metadata.Size = OssUtilities.GetJSONPropertyIfExists(releaseFile, "size")?.GetInt64(); - metadata.UploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time"); metadata.Active = !OssUtilities.GetJSONPropertyIfExists(releaseFile, "yanked")?.GetBoolean(); metadata.VersionUri = $"{ENV_PYPI_ENDPOINT}/project/{purl.Name}/{purl.Version}"; metadata.VersionDownloadUri = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "url"); + + string? uploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time"); + if (uploadTime != null) + { + metadata.UploadTime = DateTime.Parse(uploadTime); + } } } } diff --git a/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs index 24e605b1..2601a258 100644 --- a/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.CST.OpenSource.Tests.ProjectManagerTests using PackageManagers; using PackageUrl; using RichardSzalay.MockHttp; + using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -86,6 +87,28 @@ public async Task EnumerateVersionsSucceeds(string purlString, int count, string Assert.AreEqual(latestVersion, versions.First()); } + [DataTestMethod] + [DataRow("pkg:npm/lodash@4.17.15", "2019-07-19T02:28:46.584Z")] + [DataRow("pkg:npm/%40angular/core@13.2.5", "2022-03-02T18:25:31.169Z")] + [DataRow("pkg:npm/ds-modal@0.0.2", "2018-08-09T07:24:06.206Z")] + [DataRow("pkg:npm/monorepolint@0.4.0", "2019-08-07T16:20:53.525Z")] + [DataRow("pkg:npm/rly-cli@0.0.2", "2022-03-08T17:26:27.219Z")] + [DataRow("pkg:npm/example@0.0.0")] // No time property in the json. + public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) + { + PackageURL purl = new(purlString); + DateTime? time = await _projectManager.GetPublishedAtAsync(purl, useCache: false); + + if (expectedTime == null) + { + Assert.IsNull(time); + } + else + { + Assert.AreEqual(DateTime.Parse(expectedTime), time); + } + } + [DataTestMethod] [DataRow("pkg:npm/lodash@4.17.15", "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz")] [DataRow("pkg:npm/%40angular/core@13.2.5", "https://registry.npmjs.org/%40angular/core/-/core-13.2.5.tgz")] diff --git a/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs index 0749b5b9..03c3b5ae 100644 --- a/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.CST.OpenSource.Tests.ProjectManagerTests using PackageManagers; using PackageUrl; using RichardSzalay.MockHttp; + using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -122,6 +123,23 @@ public async Task EnumerateVersionsSucceeds( Assert.AreEqual(latestVersion, versions.First()); } + [DataTestMethod] + [DataRow("pkg:nuget/razorengine@4.2.3-beta1", "2015-10-06T17:53:46.37+00:00")] + [DataRow("pkg:nuget/razorengine@4.5.1-alpha001", "2017-09-02T05:17:55.973-04:00")] + public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) + { + PackageURL purl = new(purlString); + DateTime? time = await _projectManager.GetPublishedAtAsync(purl, useCache: false); + + if (expectedTime == null) + { + Assert.IsNull(time); + } + else + { + Assert.AreEqual(DateTime.Parse(expectedTime), time); + } + } [DataTestMethod] [DataRow("pkg:nuget/newtonsoft.json@13.0.1", "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/13.0.1/newtonsoft.json")] diff --git a/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs index cb4ab615..a5e5d8da 100644 --- a/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs @@ -49,7 +49,6 @@ public PyPIProjectManagerTests() _projectManager = new PyPIProjectManager(".", new NoOpPackageActions(), _httpFactory); } - [Ignore(message: "Ignored until https://github.com/microsoft/OSSGadget/issues/328 is addressed.")] [DataTestMethod] [DataRow("pkg:pypi/pandas@1.4.2", "Powerful data structures for data analysis, time series, and statistics")] [DataRow("pkg:pypi/plotly@5.7.0", "An open-source, interactive data visualization library for Python")] @@ -64,6 +63,25 @@ public async Task MetadataSucceeds(string purlString, string? description = null Assert.AreEqual(description, metadata.Description); } + [DataTestMethod] + [DataRow("pkg:pypi/pandas@1.4.2", "2022-04-02T10:37:04")] + [DataRow("pkg:pypi/plotly@5.7.0", "2022-04-05T16:26:12")] + [DataRow("pkg:pypi/requests@2.27.1", "2022-01-05T15:40:51")] + public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) + { + PackageURL purl = new(purlString); + DateTime? time = await _projectManager.GetPublishedAtAsync(purl, useCache: false); + + if (expectedTime == null) + { + Assert.IsNull(time); + } + else + { + Assert.AreEqual(DateTime.Parse(expectedTime), time); + } + } + [DataTestMethod] [DataRow("pkg:pypi/pandas@1.4.2", 86, "1.4.2")] [DataRow("pkg:pypi/plotly@3.7.1", 276, "5.7.0")]