From 7aa83fba1c2c66ab184c461c6db2278d493c9178 Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Mon, 25 Apr 2022 20:18:50 -0400 Subject: [PATCH 1/7] Add a new method in BaseProjectManager, GetPublishedAtAsync that returns the DateTime of a given purl (with a specified version) for when it was published. Or returns null if it couldn't be found. --- .../PackageManagers/BaseProjectManager.cs | 11 +++++++++ .../PackageManagers/NPMProjectManager.cs | 24 +++++++++++++++++++ .../PackageManagers/NuGetProjectManager.cs | 9 +++++++ .../PackageManagers/PyPIProjectManager.cs | 14 +++++++++++ .../NPMProjectManagerTests.cs | 23 ++++++++++++++++++ .../NuGetProjectManagerTests.cs | 18 ++++++++++++++ .../PyPIProjectManagerTests.cs | 20 ++++++++++++++++ 7 files changed, 119 insertions(+) diff --git a/src/Shared/PackageManagers/BaseProjectManager.cs b/src/Shared/PackageManagers/BaseProjectManager.cs index 18eeb72c..35fc6b0a 100644 --- a/src/Shared/PackageManagers/BaseProjectManager.cs +++ b/src/Shared/PackageManagers/BaseProjectManager.cs @@ -291,6 +291,17 @@ public virtual Task> EnumerateVersionsAsync(PackageURL purl, { throw new NotImplementedException("BaseProjectManager does not implement EnumerateVersions."); } + + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public virtual Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + throw new NotImplementedException("BaseProjectManager does not implement GetPublishedAtAsync."); + } /// /// Gets the latest version from the package metadata. diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index a018b723..6579a2bc 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -170,6 +170,30 @@ public override async Task> EnumerateVersionsAsync(PackageUR } } + public override async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + string? content = await GetMetadataAsync(purl, useCache); + if (string.IsNullOrEmpty(content)) { return null; } + + // convert NPM package data to normalized form + JsonDocument contentJSON = JsonDocument.Parse(content); + JsonElement root = contentJSON.RootElement; + + if (!root.TryGetProperty("time", out JsonElement time)) + { + return null; + } + + string? publishedTime = OssUtilities.GetJSONPropertyStringIfExists(time, purl.Version); + if (publishedTime == null) + { + return null; + } + + return DateTime.Parse(publishedTime); + + } + /// /// Gets the latest version of the package /// diff --git a/src/Shared/PackageManagers/NuGetProjectManager.cs b/src/Shared/PackageManagers/NuGetProjectManager.cs index be1fd03a..50888fad 100644 --- a/src/Shared/PackageManagers/NuGetProjectManager.cs +++ b/src/Shared/PackageManagers/NuGetProjectManager.cs @@ -102,6 +102,15 @@ private async Task GetRegistrationEndpointAsync() return RegistrationEndpoint; } + public override async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + Check.NotNull(nameof(purl.Version), purl.Version); + NuGetPackageVersionMetadata? packageVersionMetadata = + await Actions.GetMetadataAsync(purl, useCache: useCache); + + return packageVersionMetadata?.Published?.DateTime; + } + /// public override async Task GetMetadataAsync(PackageURL purl, bool useCache = true) { diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index 799809fc..a273cbe7 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -187,6 +187,20 @@ public override async Task> EnumerateVersionsAsync(PackageUR } } + public override async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + { + Check.NotNull(nameof(purl.Version), purl.Version); + PackageMetadata? metadata = await this.GetPackageMetadataAsync(purl, useCache); + + string? uploadTime = metadata?.UploadTime; + if (uploadTime == null) + { + return null; + } + + return DateTime.Parse(uploadTime); + } + public override async Task GetMetadataAsync(PackageURL purl, bool useCache = true) { try diff --git a/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs index 24e605b1..2601a258 100644 --- a/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs @@ -11,6 +11,7 @@ namespace Microsoft.CST.OpenSource.Tests.ProjectManagerTests using PackageManagers; using PackageUrl; using RichardSzalay.MockHttp; + using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -86,6 +87,28 @@ public async Task EnumerateVersionsSucceeds(string purlString, int count, string Assert.AreEqual(latestVersion, versions.First()); } + [DataTestMethod] + [DataRow("pkg:npm/lodash@4.17.15", "2019-07-19T02:28:46.584Z")] + [DataRow("pkg:npm/%40angular/core@13.2.5", "2022-03-02T18:25:31.169Z")] + [DataRow("pkg:npm/ds-modal@0.0.2", "2018-08-09T07:24:06.206Z")] + [DataRow("pkg:npm/monorepolint@0.4.0", "2019-08-07T16:20:53.525Z")] + [DataRow("pkg:npm/rly-cli@0.0.2", "2022-03-08T17:26:27.219Z")] + [DataRow("pkg:npm/example@0.0.0")] // No time property in the json. + public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) + { + PackageURL purl = new(purlString); + DateTime? time = await _projectManager.GetPublishedAtAsync(purl, useCache: false); + + if (expectedTime == null) + { + Assert.IsNull(time); + } + else + { + Assert.AreEqual(DateTime.Parse(expectedTime), time); + } + } + [DataTestMethod] [DataRow("pkg:npm/lodash@4.17.15", "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz")] [DataRow("pkg:npm/%40angular/core@13.2.5", "https://registry.npmjs.org/%40angular/core/-/core-13.2.5.tgz")] diff --git a/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs index 0749b5b9..03c3b5ae 100644 --- a/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs @@ -13,6 +13,7 @@ namespace Microsoft.CST.OpenSource.Tests.ProjectManagerTests using PackageManagers; using PackageUrl; using RichardSzalay.MockHttp; + using System; using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; @@ -122,6 +123,23 @@ public async Task EnumerateVersionsSucceeds( Assert.AreEqual(latestVersion, versions.First()); } + [DataTestMethod] + [DataRow("pkg:nuget/razorengine@4.2.3-beta1", "2015-10-06T17:53:46.37+00:00")] + [DataRow("pkg:nuget/razorengine@4.5.1-alpha001", "2017-09-02T05:17:55.973-04:00")] + public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) + { + PackageURL purl = new(purlString); + DateTime? time = await _projectManager.GetPublishedAtAsync(purl, useCache: false); + + if (expectedTime == null) + { + Assert.IsNull(time); + } + else + { + Assert.AreEqual(DateTime.Parse(expectedTime), time); + } + } [DataTestMethod] [DataRow("pkg:nuget/newtonsoft.json@13.0.1", "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/13.0.1/newtonsoft.json")] diff --git a/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs index cb4ab615..a9621ae7 100644 --- a/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs @@ -64,6 +64,26 @@ public async Task MetadataSucceeds(string purlString, string? description = null Assert.AreEqual(description, metadata.Description); } + [Ignore(message: "Ignored until https://github.com/microsoft/OSSGadget/issues/328 is addressed.")] + [DataTestMethod] + [DataRow("pkg:pypi/pandas@1.4.2", "")] + [DataRow("pkg:pypi/plotly@5.7.0", "")] + [DataRow("pkg:pypi/requests@2.27.1", "")] + public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) + { + PackageURL purl = new(purlString); + DateTime? time = await _projectManager.GetPublishedAtAsync(purl, useCache: false); + + if (expectedTime == null) + { + Assert.IsNull(time); + } + else + { + Assert.AreEqual(DateTime.Parse(expectedTime), time); + } + } + [DataTestMethod] [DataRow("pkg:pypi/pandas@1.4.2", 86, "1.4.2")] [DataRow("pkg:pypi/plotly@3.7.1", 276, "5.7.0")] From a7018768a59f92e10642676bd360b291eed3c79f Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Tue, 26 Apr 2022 12:06:35 -0400 Subject: [PATCH 2/7] I realized that GetPackageMetadataAsync for NPM didn't ever populate the UploadTime property, so I added that functionality as well. --- src/Shared/PackageManagers/NPMProjectManager.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index 6579a2bc..90d38c24 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -263,11 +263,18 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl) metadata.PackageVersion = latestVersion is null ? purl.Version : latestVersion?.ToString(); } - // if we found any version at all, get the deets + // if we found any version at all, get the information if (metadata.PackageVersion != null) { Version versionToGet = new(metadata.PackageVersion); JsonElement? versionElement = GetVersionElement(contentJSON, versionToGet); + + if (root.TryGetProperty("time", out JsonElement time)) + { + string? uploadTime = OssUtilities.GetJSONPropertyStringIfExists(time, metadata.PackageVersion); + metadata.UploadTime = uploadTime; + } + if (versionElement != null) { // redo the generic values to version specific values From 36b32a16264381cf5cb3ce8ff2b3afbadf39228d Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Tue, 26 Apr 2022 18:16:48 -0400 Subject: [PATCH 3/7] Move GetPublishedAtAsync from BaseProjectManager to just the 3 ones that implement it. --- src/Shared/PackageManagers/BaseProjectManager.cs | 11 ----------- src/Shared/PackageManagers/NPMProjectManager.cs | 8 +++++++- src/Shared/PackageManagers/NuGetProjectManager.cs | 8 +++++++- src/Shared/PackageManagers/PyPIProjectManager.cs | 8 +++++++- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/Shared/PackageManagers/BaseProjectManager.cs b/src/Shared/PackageManagers/BaseProjectManager.cs index 35fc6b0a..18eeb72c 100644 --- a/src/Shared/PackageManagers/BaseProjectManager.cs +++ b/src/Shared/PackageManagers/BaseProjectManager.cs @@ -291,17 +291,6 @@ public virtual Task> EnumerateVersionsAsync(PackageURL purl, { throw new NotImplementedException("BaseProjectManager does not implement EnumerateVersions."); } - - /// - /// Gets the a package version was published at. - /// - /// Package URL specifying the package. Version is mandatory. - /// If the cache should be used when looking for the published time. - /// The when this version was published, or null if not found. - public virtual Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) - { - throw new NotImplementedException("BaseProjectManager does not implement GetPublishedAtAsync."); - } /// /// Gets the latest version from the package metadata. diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index 90d38c24..d58e3192 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -170,7 +170,13 @@ public override async Task> EnumerateVersionsAsync(PackageUR } } - public override async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { string? content = await GetMetadataAsync(purl, useCache); if (string.IsNullOrEmpty(content)) { return null; } diff --git a/src/Shared/PackageManagers/NuGetProjectManager.cs b/src/Shared/PackageManagers/NuGetProjectManager.cs index 50888fad..3a9ee93b 100644 --- a/src/Shared/PackageManagers/NuGetProjectManager.cs +++ b/src/Shared/PackageManagers/NuGetProjectManager.cs @@ -102,7 +102,13 @@ private async Task GetRegistrationEndpointAsync() return RegistrationEndpoint; } - public override async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); NuGetPackageVersionMetadata? packageVersionMetadata = diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index a273cbe7..58f33512 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -187,7 +187,13 @@ public override async Task> EnumerateVersionsAsync(PackageUR } } - public override async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) + /// + /// Gets the a package version was published at. + /// + /// Package URL specifying the package. Version is mandatory. + /// If the cache should be used when looking for the published time. + /// The when this version was published, or null if not found. + public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); PackageMetadata? metadata = await this.GetPackageMetadataAsync(purl, useCache); From 20be29ca94b05cb8af6ef773ed6eaa11755b053d Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Tue, 26 Apr 2022 18:18:32 -0400 Subject: [PATCH 4/7] Found a fix for the PyPI failures with metadata parsing. The latest version is always the version on the root object we get when calling GetMetadataAsync because it isn't called on a version (similar to NPM) This way we can get the latest version as a fallback for GetPackageMetadata without having to parse all the versions and sort through them. This also allowed me to re-enable the ignored tests for the PyPIProjectManager. --- .../PackageManagers/PyPIProjectManager.cs | 17 ++++------------- .../PyPIProjectManagerTests.cs | 8 +++----- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index 58f33512..c30f8939 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -238,6 +238,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR metadata.Name = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "name"); metadata.Description = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "summary"); // Summary is the short description. Description is usually the readme. + metadata.LatestPackageVersion = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "version"); // Ran in the root, always points to latest version. + metadata.PackageManagerUri = ENV_PYPI_ENDPOINT; metadata.PackageUri = OssUtilities.GetJSONPropertyStringIfExists(infoElement, "package_url"); metadata.Keywords = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(infoElement, "keywords")); @@ -286,19 +288,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR }); } - // get the version - List versions = GetVersions(contentJSON); - Version? latestVersion = GetLatestVersion(versions); - - if (purl.Version != null) - { - // find the version object from the collection - metadata.PackageVersion = purl.Version; - } - else - { - metadata.PackageVersion = latestVersion is null ? purl.Version : latestVersion?.ToString(); - } + // get the version, either use the provided one, or if null then use the LatestPackageVersion. + metadata.PackageVersion = purl.Version ?? metadata.LatestPackageVersion; // if we found any version at all, get the information. if (metadata.PackageVersion is not null) diff --git a/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs b/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs index a9621ae7..a5e5d8da 100644 --- a/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs +++ b/src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs @@ -49,7 +49,6 @@ public PyPIProjectManagerTests() _projectManager = new PyPIProjectManager(".", new NoOpPackageActions(), _httpFactory); } - [Ignore(message: "Ignored until https://github.com/microsoft/OSSGadget/issues/328 is addressed.")] [DataTestMethod] [DataRow("pkg:pypi/pandas@1.4.2", "Powerful data structures for data analysis, time series, and statistics")] [DataRow("pkg:pypi/plotly@5.7.0", "An open-source, interactive data visualization library for Python")] @@ -64,11 +63,10 @@ public async Task MetadataSucceeds(string purlString, string? description = null Assert.AreEqual(description, metadata.Description); } - [Ignore(message: "Ignored until https://github.com/microsoft/OSSGadget/issues/328 is addressed.")] [DataTestMethod] - [DataRow("pkg:pypi/pandas@1.4.2", "")] - [DataRow("pkg:pypi/plotly@5.7.0", "")] - [DataRow("pkg:pypi/requests@2.27.1", "")] + [DataRow("pkg:pypi/pandas@1.4.2", "2022-04-02T10:37:04")] + [DataRow("pkg:pypi/plotly@5.7.0", "2022-04-05T16:26:12")] + [DataRow("pkg:pypi/requests@2.27.1", "2022-01-05T15:40:51")] public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime = null) { PackageURL purl = new(purlString); From 5aa01311b5b54d83b7bcc9fef519660666846672 Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Tue, 26 Apr 2022 18:27:52 -0400 Subject: [PATCH 5/7] Change the PyPIProjectManager.GetPublishedAtAsync to only do the necessary work to get the uploadTime, and no longer uses the entire GetPackageMetadataAsync method to just take the one value. --- .../PackageManagers/PyPIProjectManager.cs | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index c30f8939..17e8893c 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -196,9 +196,28 @@ public override async Task> EnumerateVersionsAsync(PackageUR public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); - PackageMetadata? metadata = await this.GetPackageMetadataAsync(purl, useCache); + string? content = await GetMetadataAsync(purl, useCache); + if (string.IsNullOrEmpty(content)) { return null; } + + JsonDocument contentJSON = JsonDocument.Parse(content); + string? uploadTime = null; + try + { + JsonElement versionElement = contentJSON.RootElement.GetProperty("releases").GetProperty(purl.Version); + + // Only get the upload time for the tarball. + foreach (JsonElement releaseFile in versionElement.EnumerateArray() + .Where(releaseFile => + OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "packagetype") == "sdist")) + { + uploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time"); + } + } + catch (KeyNotFoundException) + { + return null; + } - string? uploadTime = metadata?.UploadTime; if (uploadTime == null) { return null; @@ -229,7 +248,6 @@ public override async Task> EnumerateVersionsAsync(PackageUR string? content = await GetMetadataAsync(purl, useCache); if (string.IsNullOrEmpty(content)) { return null; } - // convert NPM package data to normalized form JsonDocument contentJSON = JsonDocument.Parse(content); JsonElement root = contentJSON.RootElement; From e1ae1901b992bf792294084ae77f85e7287b1916 Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Tue, 26 Apr 2022 18:59:11 -0400 Subject: [PATCH 6/7] Change the GetPublishedAtAsync calls to just parse the time from the GetPackageMetadata call. --- .../PackageManagers/NPMProjectManager.cs | 23 ++------------ .../PackageManagers/NuGetProjectManager.cs | 6 ++-- .../PackageManagers/PyPIProjectManager.cs | 30 ++----------------- 3 files changed, 7 insertions(+), 52 deletions(-) diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index d58e3192..d086ec8c 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -178,26 +178,9 @@ public override async Task> EnumerateVersionsAsync(PackageUR /// The when this version was published, or null if not found. public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { - string? content = await GetMetadataAsync(purl, useCache); - if (string.IsNullOrEmpty(content)) { return null; } - - // convert NPM package data to normalized form - JsonDocument contentJSON = JsonDocument.Parse(content); - JsonElement root = contentJSON.RootElement; - - if (!root.TryGetProperty("time", out JsonElement time)) - { - return null; - } - - string? publishedTime = OssUtilities.GetJSONPropertyStringIfExists(time, purl.Version); - if (publishedTime == null) - { - return null; - } - - return DateTime.Parse(publishedTime); - + Check.NotNull(nameof(purl.Version), purl.Version); + string? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime == null ? null : DateTime.Parse(uploadTime); } /// diff --git a/src/Shared/PackageManagers/NuGetProjectManager.cs b/src/Shared/PackageManagers/NuGetProjectManager.cs index 3a9ee93b..fa61d760 100644 --- a/src/Shared/PackageManagers/NuGetProjectManager.cs +++ b/src/Shared/PackageManagers/NuGetProjectManager.cs @@ -111,10 +111,8 @@ private async Task GetRegistrationEndpointAsync() public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); - NuGetPackageVersionMetadata? packageVersionMetadata = - await Actions.GetMetadataAsync(purl, useCache: useCache); - - return packageVersionMetadata?.Published?.DateTime; + string? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime == null ? null : DateTime.Parse(uploadTime); } /// diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index 17e8893c..46b2ba05 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -196,34 +196,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); - string? content = await GetMetadataAsync(purl, useCache); - if (string.IsNullOrEmpty(content)) { return null; } - - JsonDocument contentJSON = JsonDocument.Parse(content); - string? uploadTime = null; - try - { - JsonElement versionElement = contentJSON.RootElement.GetProperty("releases").GetProperty(purl.Version); - - // Only get the upload time for the tarball. - foreach (JsonElement releaseFile in versionElement.EnumerateArray() - .Where(releaseFile => - OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "packagetype") == "sdist")) - { - uploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time"); - } - } - catch (KeyNotFoundException) - { - return null; - } - - if (uploadTime == null) - { - return null; - } - - return DateTime.Parse(uploadTime); + string? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime == null ? null : DateTime.Parse(uploadTime); } public override async Task GetMetadataAsync(PackageURL purl, bool useCache = true) From 3084916eb189ba32a00ae249534ce554b759c406 Mon Sep 17 00:00:00 2001 From: Julian Pinzer Date: Tue, 26 Apr 2022 19:06:42 -0400 Subject: [PATCH 7/7] Changed UploadTime to a DateTime object in PackageMetadata. This allows us to not have to do the parsing in GetPublishedAtAsync. --- src/Shared/Model/PackageMetadata.cs | 3 ++- src/Shared/PackageManagers/NPMProjectManager.cs | 9 ++++++--- src/Shared/PackageManagers/NuGetProjectManager.cs | 6 +++--- src/Shared/PackageManagers/PyPIProjectManager.cs | 11 ++++++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/Shared/Model/PackageMetadata.cs b/src/Shared/Model/PackageMetadata.cs index 2925e538..671bd520 100644 --- a/src/Shared/Model/PackageMetadata.cs +++ b/src/Shared/Model/PackageMetadata.cs @@ -3,6 +3,7 @@ namespace Microsoft.CST.OpenSource.Model { using Newtonsoft.Json; + using System; using System.Collections.Generic; public class Downloads @@ -111,7 +112,7 @@ public class PackageMetadata public long? Size { get; set; } [JsonProperty(PropertyName = "upload_time", NullValueHandling = NullValueHandling.Ignore)] - public string? UploadTime { get; set; } + public DateTime? UploadTime { get; set; } [JsonProperty(PropertyName = "commit_id", NullValueHandling = NullValueHandling.Ignore)] public string? CommitId { get; set; } diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs index d086ec8c..834b246a 100644 --- a/src/Shared/PackageManagers/NPMProjectManager.cs +++ b/src/Shared/PackageManagers/NPMProjectManager.cs @@ -179,8 +179,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); - string? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; - return uploadTime == null ? null : DateTime.Parse(uploadTime); + DateTime? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime; } /// @@ -261,7 +261,10 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl) if (root.TryGetProperty("time", out JsonElement time)) { string? uploadTime = OssUtilities.GetJSONPropertyStringIfExists(time, metadata.PackageVersion); - metadata.UploadTime = uploadTime; + if (uploadTime != null) + { + metadata.UploadTime = DateTime.Parse(uploadTime); + } } if (versionElement != null) diff --git a/src/Shared/PackageManagers/NuGetProjectManager.cs b/src/Shared/PackageManagers/NuGetProjectManager.cs index fa61d760..833a6e26 100644 --- a/src/Shared/PackageManagers/NuGetProjectManager.cs +++ b/src/Shared/PackageManagers/NuGetProjectManager.cs @@ -111,8 +111,8 @@ private async Task GetRegistrationEndpointAsync() public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); - string? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; - return uploadTime == null ? null : DateTime.Parse(uploadTime); + DateTime? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime; } /// @@ -246,7 +246,7 @@ private async Task UpdateVersionMetadata(PackageMetadata metadata, NuGetPackageV } // publishing info - metadata.UploadTime = packageVersionPackageVersionMetadata.Published?.ToString("MM/dd/yy HH:mm:ss zz"); + metadata.UploadTime = packageVersionPackageVersionMetadata.Published?.DateTime; } /// diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs index 46b2ba05..e0335670 100644 --- a/src/Shared/PackageManagers/PyPIProjectManager.cs +++ b/src/Shared/PackageManagers/PyPIProjectManager.cs @@ -196,8 +196,8 @@ public override async Task> EnumerateVersionsAsync(PackageUR public async Task GetPublishedAtAsync(PackageURL purl, bool useCache = true) { Check.NotNull(nameof(purl.Version), purl.Version); - string? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; - return uploadTime == null ? null : DateTime.Parse(uploadTime); + DateTime? uploadTime = (await this.GetPackageMetadataAsync(purl, useCache))?.UploadTime; + return uploadTime; } public override async Task GetMetadataAsync(PackageURL purl, bool useCache = true) @@ -325,10 +325,15 @@ public override async Task> EnumerateVersionsAsync(PackageUR } metadata.Size = OssUtilities.GetJSONPropertyIfExists(releaseFile, "size")?.GetInt64(); - metadata.UploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time"); metadata.Active = !OssUtilities.GetJSONPropertyIfExists(releaseFile, "yanked")?.GetBoolean(); metadata.VersionUri = $"{ENV_PYPI_ENDPOINT}/project/{purl.Name}/{purl.Version}"; metadata.VersionDownloadUri = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "url"); + + string? uploadTime = OssUtilities.GetJSONPropertyStringIfExists(releaseFile, "upload_time"); + if (uploadTime != null) + { + metadata.UploadTime = DateTime.Parse(uploadTime); + } } } }