diff --git a/src/NuGet.Core/NuGet.Commands/RestoreCommand/ProjectRestoreCommand.cs b/src/NuGet.Core/NuGet.Commands/RestoreCommand/ProjectRestoreCommand.cs index 99b845f8e20..f7009df21b3 100644 --- a/src/NuGet.Core/NuGet.Commands/RestoreCommand/ProjectRestoreCommand.cs +++ b/src/NuGet.Core/NuGet.Commands/RestoreCommand/ProjectRestoreCommand.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -358,6 +358,12 @@ private RuntimeGraph GetRuntimeGraph(RestoreTargetGraph graph, IReadOnlyList> _dependencyInfoCache + = new ConcurrentDictionary>(); + + private readonly ConcurrentDictionary> _libraryMatchCache + = new ConcurrentDictionary>(); + // Limiting concurrent requests to limit the amount of files open at a time on Mac OSX // the default is 256 which is easy to hit if we don't limit concurrency private readonly static SemaphoreSlim _throttle = @@ -52,6 +61,12 @@ public SourceRepositoryDependencyProvider( public bool IsHttp => _sourceRepository.PackageSource.IsHttp; + public PackageSource Source => _sourceRepository.PackageSource; + + /// + /// Discovers all versions of a package from a source and selects the best match. + /// This does not download the package. + /// public async Task FindLibraryAsync( LibraryRange libraryRange, NuGetFramework targetFramework, @@ -59,50 +74,44 @@ public async Task FindLibraryAsync( ILogger logger, CancellationToken cancellationToken) { - await EnsureResource(); + AsyncLazy result = null; - IEnumerable packageVersions = null; - try - { - if (_throttle != null) - { - await _throttle.WaitAsync(); - } - packageVersions = await _findPackagesByIdResource.GetAllVersionsAsync( - libraryRange.Name, - cacheContext, - logger, - cancellationToken); - } - catch (FatalProtocolException e) when (_ignoreFailedSources) + var action = new AsyncLazy(async () => + await FindLibraryCoreAsync(libraryRange, targetFramework, cacheContext, logger, cancellationToken)); + + if (cacheContext.RefreshMemoryCache) { - if (!_ignoreWarning) - { - _logger.LogWarning(e.Message); - } - return null; + result = _libraryMatchCache.AddOrUpdate(libraryRange, action, (k, v) => action); } - finally + else { - _throttle?.Release(); + result = _libraryMatchCache.GetOrAdd(libraryRange, action); } + return await result; + } + + public async Task FindLibraryCoreAsync( + LibraryRange libraryRange, + NuGetFramework targetFramework, + SourceCacheContext cacheContext, + ILogger logger, + CancellationToken cancellationToken) + { + await EnsureResource(); + + // Discover all versions from the feed + var packageVersions = await GetPackageVersionsAsync(libraryRange.Name, cacheContext, logger, cancellationToken); + + // Select the best match var packageVersion = packageVersions?.FindBestMatch(libraryRange.VersionRange, version => version); if (packageVersion != null) { - // Use the original package identity for the library identity - var packageIdentity = await _findPackagesByIdResource.GetOriginalIdentityAsync( - libraryRange.Name, - packageVersion, - cacheContext, - logger, - cancellationToken); - return new LibraryIdentity { - Name = packageIdentity.Id, - Version = packageIdentity.Version, + Name = libraryRange.Name, + Version = packageVersion, Type = LibraryType.Package }; } @@ -110,7 +119,33 @@ public async Task FindLibraryAsync( return null; } - public async Task> GetDependenciesAsync( + public async Task GetDependenciesAsync( + LibraryIdentity match, + NuGetFramework targetFramework, + SourceCacheContext cacheContext, + ILogger logger, + CancellationToken cancellationToken) + { + AsyncLazy result = null; + + var action = new AsyncLazy(async () => + await GetDependenciesCoreAsync(match, targetFramework, cacheContext, logger, cancellationToken)); + + var key = new LibraryRangeCacheKey(match, targetFramework); + + if (cacheContext.RefreshMemoryCache) + { + result = _dependencyInfoCache.AddOrUpdate(key, action, (k,v) => action); + } + else + { + result = _dependencyInfoCache.GetOrAdd(key, action); + } + + return await result; + } + + private async Task GetDependenciesCoreAsync( LibraryIdentity match, NuGetFramework targetFramework, SourceCacheContext cacheContext, @@ -126,6 +161,8 @@ public async Task> GetDependenciesAsync( { await _throttle.WaitAsync(); } + + // Read package info, this will download the package if needed. packageInfo = await _findPackagesByIdResource.GetDependencyInfoAsync( match.Name, match.Version, @@ -133,20 +170,35 @@ public async Task> GetDependenciesAsync( logger, cancellationToken); } - catch (FatalProtocolException e) when (_ignoreFailedSources) + catch (FatalProtocolException e) when (_ignoreFailedSources && !(e is InvalidCacheProtocolException)) { if (!_ignoreWarning) { _logger.LogWarning(e.Message); } - return new List(); } finally { _throttle?.Release(); } - return GetDependencies(packageInfo, targetFramework); + if (packageInfo == null) + { + // Package was not found + return LibraryDependencyInfo.CreateUnresolved(match, targetFramework); + } + else + { + // Package found + var originalIdentity = new LibraryIdentity( + packageInfo.PackageIdentity.Id, + packageInfo.PackageIdentity.Version, + match.Type); + + var dependencies = GetDependencies(packageInfo, targetFramework); + + return LibraryDependencyInfo.Create(originalIdentity, targetFramework, dependencies); + } } public async Task CopyToAsync( @@ -194,8 +246,9 @@ private IEnumerable GetDependencies(FindPackageByIdDependency { if (packageInfo == null) { - return new List(); + return Enumerable.Empty(); } + var dependencies = NuGetFrameworkUtility.GetNearest(packageInfo.DependencyGroups, targetFramework, item => item.TargetFramework); @@ -203,18 +256,15 @@ private IEnumerable GetDependencies(FindPackageByIdDependency return GetDependencies(targetFramework, dependencies); } - private static IList GetDependencies(NuGetFramework targetFramework, + private static IEnumerable GetDependencies(NuGetFramework targetFramework, PackageDependencyGroup dependencies) { - var libraryDependencies = new List(); - if (dependencies != null) { - libraryDependencies.AddRange( - dependencies.Packages.Select(PackagingUtility.GetLibraryDependencyFromNuspec)); + return dependencies.Packages.Select(PackagingUtility.GetLibraryDependencyFromNuspec).ToArray(); } - return libraryDependencies; + return Enumerable.Empty(); } private async Task EnsureResource() @@ -232,5 +282,42 @@ private async Task EnsureResource() } } } + + /// + /// Discover all package versions from a feed. + /// + private async Task> GetPackageVersionsAsync(string id, + SourceCacheContext cacheContext, + ILogger logger, + CancellationToken cancellationToken) + { + IEnumerable packageVersions = null; + try + { + if (_throttle != null) + { + await _throttle.WaitAsync(); + } + packageVersions = await _findPackagesByIdResource.GetAllVersionsAsync( + id, + cacheContext, + logger, + cancellationToken); + } + catch (FatalProtocolException e) when (_ignoreFailedSources) + { + if (!_ignoreWarning) + { + _logger.LogWarning(e.Message); + } + return null; + } + finally + { + _throttle?.Release(); + } + + return packageVersions; + } } } diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/LibraryRangeCacheKey.cs b/src/NuGet.Core/NuGet.DependencyResolver.Core/LibraryRangeCacheKey.cs index 8d7a0278eb1..c449555f034 100644 --- a/src/NuGet.Core/NuGet.DependencyResolver.Core/LibraryRangeCacheKey.cs +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/LibraryRangeCacheKey.cs @@ -37,12 +37,7 @@ public override bool Equals(object obj) public override int GetHashCode() { - var combiner = new HashCodeCombiner(); - - combiner.AddObject(LibraryRange); - combiner.AddObject(Framework); - - return combiner.CombinedHash; + return HashCodeCombiner.GetHashCode(LibraryRange, Framework); } public bool Equals(LibraryRangeCacheKey other) diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/NuGet.DependencyResolver.Core.csproj b/src/NuGet.Core/NuGet.DependencyResolver.Core/NuGet.DependencyResolver.Core.csproj index 518f1c82a99..9967208fe4d 100644 --- a/src/NuGet.Core/NuGet.DependencyResolver.Core/NuGet.DependencyResolver.Core.csproj +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/NuGet.DependencyResolver.Core.csproj @@ -7,6 +7,7 @@ $(NetStandardPackageVersion) true true + NuGet.DependencyResolver @@ -15,5 +16,20 @@ + + + True + True + Strings.resx + + + + + + ResXFileCodeGenerator + Strings.Designer.cs + + + diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/IRemoteDependencyProvider.cs b/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/IRemoteDependencyProvider.cs index 0fc48aa215a..bb8f0a4dbb5 100644 --- a/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/IRemoteDependencyProvider.cs +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/IRemoteDependencyProvider.cs @@ -6,8 +6,10 @@ using System.Threading; using System.Threading.Tasks; using NuGet.Common; +using NuGet.Configuration; using NuGet.Frameworks; using NuGet.LibraryModel; +using NuGet.Packaging.Core; using NuGet.Protocol.Core.Types; namespace NuGet.DependencyResolver @@ -16,6 +18,12 @@ public interface IRemoteDependencyProvider { bool IsHttp { get; } + /// + /// Feed package source. + /// + /// Optional. This will be null for project providers. + PackageSource Source { get; } + Task FindLibraryAsync( LibraryRange libraryRange, NuGetFramework targetFramework, @@ -23,7 +31,7 @@ Task FindLibraryAsync( ILogger logger, CancellationToken cancellationToken); - Task> GetDependenciesAsync( + Task GetDependenciesAsync( LibraryIdentity match, NuGetFramework targetFramework, SourceCacheContext cacheContext, diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/LocalDependencyProvider.cs b/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/LocalDependencyProvider.cs index fa2cafcab1e..d34ac78e831 100644 --- a/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/LocalDependencyProvider.cs +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/Providers/LocalDependencyProvider.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using NuGet.Common; +using NuGet.Configuration; using NuGet.Frameworks; using NuGet.LibraryModel; using NuGet.Protocol.Core.Types; @@ -24,6 +25,8 @@ public LocalDependencyProvider(IDependencyProvider dependencyProvider) public bool IsHttp { get; private set; } + public PackageSource Source { get; private set; } + public Task FindLibraryAsync( LibraryRange libraryRange, NuGetFramework targetFramework, @@ -41,7 +44,7 @@ public Task FindLibraryAsync( return Task.FromResult(library.Identity); } - public Task> GetDependenciesAsync( + public Task GetDependenciesAsync( LibraryIdentity library, NuGetFramework targetFramework, SourceCacheContext cacheContext, @@ -50,7 +53,12 @@ public Task> GetDependenciesAsync( { var description = _dependencyProvider.GetLibrary(library, targetFramework); - return Task.FromResult(description.Dependencies); + var dependencyInfo = LibraryDependencyInfo.Create( + description.Identity, + targetFramework, + description.Dependencies); + + return Task.FromResult(dependencyInfo); } public Task CopyToAsync( diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/Remote/RemoteDependencyWalker.cs b/src/NuGet.Core/NuGet.DependencyResolver.Core/Remote/RemoteDependencyWalker.cs index f23d3031d94..24152b2b417 100644 --- a/src/NuGet.Core/NuGet.DependencyResolver.Core/Remote/RemoteDependencyWalker.cs +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/Remote/RemoteDependencyWalker.cs @@ -1,11 +1,9 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -39,9 +37,9 @@ private async Task> CreateGraphNode( GraphEdge outerEdge) { var dependencies = new List(); - var runtimeDependencies = new HashSet(); + var runtimeDependencies = new HashSet(StringComparer.OrdinalIgnoreCase); - if (!string.IsNullOrEmpty(runtimeName) && runtimeGraph != null) + if (runtimeGraph != null && !string.IsNullOrEmpty(runtimeName)) { // HACK(davidfowl): This is making runtime.json support package redirects @@ -79,11 +77,12 @@ private async Task> CreateGraphNode( var node = new GraphNode(libraryRange) { // Resolve the dependency from the cache or sources - Item = await FindLibraryCached( + Item = await ResolverUtility.FindLibraryCachedAsync( _context.FindLibraryEntryCache, libraryRange, framework, outerEdge, + _context, CancellationToken.None) }; @@ -153,7 +152,7 @@ private async Task> CreateGraphNode( } } - while (tasks.Any()) + while (tasks.Count > 0) { // Wait for any node to finish resolving var task = await Task.WhenAny(tasks); @@ -316,325 +315,6 @@ private static NuGetVersion GetReleaseLabelFreeVersion(VersionRange versionRange } } - private Task> FindLibraryCached( - ConcurrentDictionary>> cache, - LibraryRange libraryRange, - NuGetFramework framework, - GraphEdge outerEdge, - CancellationToken cancellationToken) - { - var key = new LibraryRangeCacheKey(libraryRange, framework); - - return cache.GetOrAdd(key, (cacheKey) => - FindLibraryEntry(cacheKey.LibraryRange, framework, outerEdge, cancellationToken)); - } - - private async Task> FindLibraryEntry( - LibraryRange libraryRange, - NuGetFramework framework, - GraphEdge outerEdge, - CancellationToken cancellationToken) - { - var match = await FindLibraryMatch(libraryRange, framework, outerEdge, cancellationToken); - - if (match == null) - { - return CreateUnresolvedMatch(libraryRange); - } - - IEnumerable dependencies; - - // For local matches such as projects get the dependencies from the LocalLibrary property. - var localMatch = match as LocalMatch; - - if (localMatch != null) - { - dependencies = localMatch.LocalLibrary.Dependencies; - } - else - { - // Look up the dependencies from the source - dependencies = await match.Provider.GetDependenciesAsync( - match.Library, - framework, - _context.CacheContext, - _context.Logger, - cancellationToken); - } - - return new GraphItem(match.Library) - { - Data = new RemoteResolveResult - { - Match = match, - Dependencies = dependencies - }, - }; - } - - private static GraphItem CreateUnresolvedMatch(LibraryRange libraryRange) - { - var identity = new LibraryIdentity() - { - Name = libraryRange.Name, - Type = LibraryType.Unresolved, - Version = libraryRange.VersionRange?.MinVersion - }; - return new GraphItem(identity) - { - Data = new RemoteResolveResult() - { - Match = new RemoteMatch() - { - Library = identity, - Path = null, - Provider = null - }, - Dependencies = Enumerable.Empty() - } - }; - } - - private async Task FindLibraryMatch( - LibraryRange libraryRange, - NuGetFramework framework, - GraphEdge outerEdge, - CancellationToken cancellationToken) - { - var projectMatch = await FindProjectMatch(libraryRange, framework, outerEdge, cancellationToken); - - if (projectMatch != null) - { - return projectMatch; - } - - if (libraryRange.VersionRange == null) - { - return null; - } - - // The resolution below is only for package types - if (!libraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)) - { - return null; - } - - if (libraryRange.VersionRange.IsFloating) - { - // For snapshot dependencies, get the version remotely first. - var remoteMatch = await FindLibraryByVersion(libraryRange, framework, _context.RemoteLibraryProviders, cancellationToken); - if (remoteMatch != null) - { - // Try to see if the specific version found on the remote exists locally. This avoids any unnecessary - // remote access incase we already have it in the cache/local packages folder. - var localMatch = await FindLibraryByVersion(remoteMatch.Library, framework, _context.LocalLibraryProviders, cancellationToken); - - if (localMatch != null - && localMatch.Library.Version.Equals(remoteMatch.Library.Version)) - { - // If we have a local match, and it matches the version *exactly* then use it. - return localMatch; - } - - // We found something locally, but it wasn't an exact match - // for the resolved remote match. - } - - return remoteMatch; - } - else - { - // Check for the specific version locally. - var localMatch = await FindLibraryByVersion(libraryRange, framework, _context.LocalLibraryProviders, cancellationToken); - - if (localMatch != null - && localMatch.Library.Version.Equals(libraryRange.VersionRange.MinVersion)) - { - // We have an exact match so use it. - return localMatch; - } - - // Either we found a local match but it wasn't the exact version, or - // we didn't find a local match. - var remoteMatch = await FindLibraryByVersion(libraryRange, framework, _context.RemoteLibraryProviders, cancellationToken); - - if (remoteMatch != null - && localMatch == null) - { - // There wasn't any local match for the specified version but there was a remote match. - // See if that version exists locally. - localMatch = await FindLibraryByVersion(remoteMatch.Library, framework, _context.LocalLibraryProviders, cancellationToken); - } - - if (localMatch != null - && remoteMatch != null) - { - // We found a match locally and remotely, so pick the better version - // in relation to the specified version. - if (libraryRange.VersionRange.IsBetter( - current: localMatch.Library.Version, - considering: remoteMatch.Library.Version)) - { - return remoteMatch; - } - else - { - return localMatch; - } - } - - // Prefer local over remote generally. - return localMatch ?? remoteMatch; - } - } - - private Task FindProjectMatch( - LibraryRange libraryRange, - NuGetFramework framework, - GraphEdge outerEdge, - CancellationToken cancellationToken) - { - RemoteMatch result = null; - - // Check if projects are allowed for this dependency - if (libraryRange.TypeConstraintAllowsAnyOf( - (LibraryDependencyTarget.Project | LibraryDependencyTarget.ExternalProject))) - { - foreach (var provider in _context.ProjectLibraryProviders) - { - if (provider.SupportsType(libraryRange.TypeConstraint)) - { - var match = provider.GetLibrary(libraryRange, framework); - - if (match != null) - { - result = new LocalMatch - { - LocalLibrary = match, - Library = match.Identity, - LocalProvider = provider, - Provider = new LocalDependencyProvider(provider), - Path = match.Path, - }; - } - } - } - } - - return Task.FromResult(result); - } - - private async Task FindLibraryByVersion(LibraryRange libraryRange, NuGetFramework framework, IEnumerable providers, CancellationToken token) - { - if (libraryRange.VersionRange.IsFloating) - { - // Don't optimize the non http path for floating versions or we'll miss things - return await FindLibrary( - libraryRange, - providers, - provider => provider.FindLibraryAsync( - libraryRange, - framework, - _context.CacheContext, - _context.Logger, - token)); - } - - // Try the non http sources first - var nonHttpMatch = await FindLibrary( - libraryRange, - providers.Where(p => !p.IsHttp), - provider => provider.FindLibraryAsync( - libraryRange, - framework, - _context.CacheContext, - _context.Logger, - token)); - - // If we found an exact match then use it - if (nonHttpMatch != null && nonHttpMatch.Library.Version.Equals(libraryRange.VersionRange.MinVersion)) - { - return nonHttpMatch; - } - - // Otherwise try the http sources - var httpMatch = await FindLibrary( - libraryRange, - providers.Where(p => p.IsHttp), - provider => provider.FindLibraryAsync( - libraryRange, - framework, - _context.CacheContext, - _context.Logger, - token)); - - // Pick the best match of the 2 - if (libraryRange.VersionRange.IsBetter( - nonHttpMatch?.Library?.Version, - httpMatch?.Library.Version)) - { - return httpMatch; - } - - return nonHttpMatch; - } - - private static async Task FindLibrary( - LibraryRange libraryRange, - IEnumerable providers, - Func> action) - { - var tasks = new List>(); - foreach (var provider in providers) - { - Func> taskWrapper = async () => - { - var library = await action(provider); - if (library != null) - { - return new RemoteMatch - { - Provider = provider, - Library = library - }; - } - - return null; - }; - - tasks.Add(taskWrapper()); - } - - RemoteMatch bestMatch = null; - - while (tasks.Count > 0) - { - var task = await Task.WhenAny(tasks); - tasks.Remove(task); - var match = await task; - - // If we found an exact match then use it. - // This allows us to shortcircuit slow feeds even if there's an exact match - if (!libraryRange.VersionRange.IsFloating && - match?.Library?.Version != null && - libraryRange.VersionRange.IsMinInclusive && - match.Library.Version.Equals(libraryRange.VersionRange.MinVersion)) - { - return match; - } - - // Otherwise just find the best out of the matches - if (libraryRange.VersionRange.IsBetter( - current: bestMatch?.Library?.Version, - considering: match?.Library?.Version)) - { - bestMatch = match; - } - } - - return bestMatch; - } - private enum DependencyResult { Acceptable, diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/ResolverUtility.cs b/src/NuGet.Core/NuGet.DependencyResolver.Core/ResolverUtility.cs new file mode 100644 index 00000000000..f88c44fa48d --- /dev/null +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/ResolverUtility.cs @@ -0,0 +1,412 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using NuGet.Common; +using NuGet.Frameworks; +using NuGet.LibraryModel; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; + +namespace NuGet.DependencyResolver +{ + public static class ResolverUtility + { + public static Task> FindLibraryCachedAsync( + ConcurrentDictionary>> cache, + LibraryRange libraryRange, + NuGetFramework framework, + GraphEdge outerEdge, + RemoteWalkContext context, + CancellationToken cancellationToken) + { + var key = new LibraryRangeCacheKey(libraryRange, framework); + + return cache.GetOrAdd(key, (cacheKey) => + FindLibraryEntryAsync(cacheKey.LibraryRange, framework, outerEdge, context, cancellationToken)); + } + + public static async Task> FindLibraryEntryAsync( + LibraryRange libraryRange, + NuGetFramework framework, + GraphEdge outerEdge, + RemoteWalkContext context, + CancellationToken cancellationToken) + { + GraphItem graphItem = null; + var currentCacheContext = context.CacheContext; + + // Try up to two times to get the package. The second + // retry will refresh the cache if a package is listed + // but fails to download. This can happen if the feed prunes + // the package. + for (var i = 0; i < 2 && graphItem == null; i++) + { + var match = await FindLibraryMatchAsync( + libraryRange, + framework, + outerEdge, + context.RemoteLibraryProviders, + context.LocalLibraryProviders, + context.ProjectLibraryProviders, + currentCacheContext, + context.Logger, + cancellationToken); + + if (match == null) + { + return CreateUnresolvedMatch(libraryRange); + } + + try + { + graphItem = await CreateGraphItemAsync(match, framework, currentCacheContext, context.Logger, cancellationToken); + } + catch (InvalidCacheProtocolException) when (i == 0) + { + // 1st failure, invalidate the cache and try again. + // Clear the on disk and memory caches during the next request. + currentCacheContext = currentCacheContext.WithRefreshCacheTrue(); + } + catch (PackageNotFoundProtocolException ex) when (match.Provider.IsHttp && match.Provider.Source != null) + { + // 2nd failure, the feed is likely corrupt or removing packages too fast to keep up with. + var message = string.Format(CultureInfo.CurrentCulture, + Strings.Error_PackageNotFoundWhenExpected, + match.Provider.Source, + ex.PackageIdentity.ToString()); + + throw new FatalProtocolException(message, ex); + } + } + + return graphItem; + } + + public static async Task> CreateGraphItemAsync( + RemoteMatch match, + NuGetFramework framework, + SourceCacheContext cacheContext, + ILogger logger, + CancellationToken cancellationToken) + { + LibraryDependencyInfo dependencies; + + // For local matches such as projects get the dependencies from the LocalLibrary property. + var localMatch = match as LocalMatch; + + if (localMatch != null) + { + dependencies = LibraryDependencyInfo.Create( + localMatch.LocalLibrary.Identity, + framework, + localMatch.LocalLibrary.Dependencies); + } + else + { + // Look up the dependencies from the source + dependencies = await match.Provider.GetDependenciesAsync( + match.Library, + framework, + cacheContext, + logger, + cancellationToken); + } + + // Copy the original identity to the remote match. + // This ensures that the correct casing is used for + // the id/version. + match.Library = dependencies.Library; + + return new GraphItem(match.Library) + { + Data = new RemoteResolveResult + { + Match = match, + Dependencies = dependencies.Dependencies + }, + }; + } + + public static GraphItem CreateUnresolvedMatch(LibraryRange libraryRange) + { + var identity = new LibraryIdentity() + { + Name = libraryRange.Name, + Type = LibraryType.Unresolved, + Version = libraryRange.VersionRange?.MinVersion + }; + return new GraphItem(identity) + { + Data = new RemoteResolveResult() + { + Match = new RemoteMatch() + { + Library = identity, + Path = null, + Provider = null + }, + Dependencies = Enumerable.Empty() + } + }; + } + + public static async Task FindLibraryMatchAsync( + LibraryRange libraryRange, + NuGetFramework framework, + GraphEdge outerEdge, + IEnumerable remoteProviders, + IEnumerable localProviders, + IEnumerable projectProviders, + SourceCacheContext cacheContext, + ILogger logger, + CancellationToken cancellationToken) + { + var projectMatch = await FindProjectMatchAsync(libraryRange, framework, outerEdge, projectProviders, cancellationToken); + + if (projectMatch != null) + { + return projectMatch; + } + + if (libraryRange.VersionRange == null) + { + return null; + } + + // The resolution below is only for package types + if (!libraryRange.TypeConstraintAllows(LibraryDependencyTarget.Package)) + { + return null; + } + + if (libraryRange.VersionRange.IsFloating) + { + // For snapshot dependencies, get the version remotely first. + var remoteMatch = await FindLibraryByVersionAsync(libraryRange, framework, remoteProviders, cacheContext, logger, cancellationToken); + if (remoteMatch != null) + { + // Try to see if the specific version found on the remote exists locally. This avoids any unnecessary + // remote access incase we already have it in the cache/local packages folder. + var localMatch = await FindLibraryByVersionAsync(remoteMatch.Library, framework, localProviders, cacheContext, logger, cancellationToken); + + if (localMatch != null + && localMatch.Library.Version.Equals(remoteMatch.Library.Version)) + { + // If we have a local match, and it matches the version *exactly* then use it. + return localMatch; + } + + // We found something locally, but it wasn't an exact match + // for the resolved remote match. + } + + return remoteMatch; + } + else + { + // Check for the specific version locally. + var localMatch = await FindLibraryByVersionAsync(libraryRange, framework, localProviders, cacheContext, logger, cancellationToken); + + if (localMatch != null + && localMatch.Library.Version.Equals(libraryRange.VersionRange.MinVersion)) + { + // We have an exact match so use it. + return localMatch; + } + + // Either we found a local match but it wasn't the exact version, or + // we didn't find a local match. + var remoteMatch = await FindLibraryByVersionAsync(libraryRange, framework, remoteProviders, cacheContext, logger, cancellationToken); + + if (remoteMatch != null + && localMatch == null) + { + // There wasn't any local match for the specified version but there was a remote match. + // See if that version exists locally. + localMatch = await FindLibraryByVersionAsync(remoteMatch.Library, framework, remoteProviders, cacheContext, logger, cancellationToken); + } + + if (localMatch != null + && remoteMatch != null) + { + // We found a match locally and remotely, so pick the better version + // in relation to the specified version. + if (libraryRange.VersionRange.IsBetter( + current: localMatch.Library.Version, + considering: remoteMatch.Library.Version)) + { + return remoteMatch; + } + else + { + return localMatch; + } + } + + // Prefer local over remote generally. + return localMatch ?? remoteMatch; + } + } + + public static Task FindProjectMatchAsync( + LibraryRange libraryRange, + NuGetFramework framework, + GraphEdge outerEdge, + IEnumerable projectProviders, + CancellationToken cancellationToken) + { + RemoteMatch result = null; + + // Check if projects are allowed for this dependency + if (libraryRange.TypeConstraintAllowsAnyOf( + (LibraryDependencyTarget.Project | LibraryDependencyTarget.ExternalProject))) + { + foreach (var provider in projectProviders) + { + if (provider.SupportsType(libraryRange.TypeConstraint)) + { + var match = provider.GetLibrary(libraryRange, framework); + + if (match != null) + { + result = new LocalMatch + { + LocalLibrary = match, + Library = match.Identity, + LocalProvider = provider, + Provider = new LocalDependencyProvider(provider), + Path = match.Path, + }; + } + } + } + } + + return Task.FromResult(result); + } + + public static async Task FindLibraryByVersionAsync( + LibraryRange libraryRange, + NuGetFramework framework, + IEnumerable providers, + SourceCacheContext cacheContext, + ILogger logger, + CancellationToken token) + { + if (libraryRange.VersionRange.IsFloating) + { + // Don't optimize the non http path for floating versions or we'll miss things + return await FindLibraryFromSourcesAsync( + libraryRange, + providers, + provider => provider.FindLibraryAsync( + libraryRange, + framework, + cacheContext, + logger, + token)); + } + + // Try the non http sources first + var nonHttpMatch = await FindLibraryFromSourcesAsync( + libraryRange, + providers.Where(p => !p.IsHttp), + provider => provider.FindLibraryAsync( + libraryRange, + framework, + cacheContext, + logger, + token)); + + // If we found an exact match then use it + if (nonHttpMatch != null && nonHttpMatch.Library.Version.Equals(libraryRange.VersionRange.MinVersion)) + { + return nonHttpMatch; + } + + // Otherwise try the http sources + var httpMatch = await FindLibraryFromSourcesAsync( + libraryRange, + providers.Where(p => p.IsHttp), + provider => provider.FindLibraryAsync( + libraryRange, + framework, + cacheContext, + logger, + token)); + + // Pick the best match of the 2 + if (libraryRange.VersionRange.IsBetter( + nonHttpMatch?.Library?.Version, + httpMatch?.Library.Version)) + { + return httpMatch; + } + + return nonHttpMatch; + } + + public static async Task FindLibraryFromSourcesAsync( + LibraryRange libraryRange, + IEnumerable providers, + Func> action) + { + var tasks = new List>(); + foreach (var provider in providers) + { + Func> taskWrapper = async () => + { + var library = await action(provider); + if (library != null) + { + return new RemoteMatch + { + Provider = provider, + Library = library + }; + } + + return null; + }; + + tasks.Add(taskWrapper()); + } + + RemoteMatch bestMatch = null; + + while (tasks.Count > 0) + { + var task = await Task.WhenAny(tasks); + tasks.Remove(task); + var match = await task; + + // If we found an exact match then use it. + // This allows us to shortcircuit slow feeds even if there's an exact match + if (!libraryRange.VersionRange.IsFloating && + match?.Library?.Version != null && + libraryRange.VersionRange.IsMinInclusive && + match.Library.Version.Equals(libraryRange.VersionRange.MinVersion)) + { + return match; + } + + // Otherwise just find the best out of the matches + if (libraryRange.VersionRange.IsBetter( + current: bestMatch?.Library?.Version, + considering: match?.Library?.Version)) + { + bestMatch = match; + } + } + + return bestMatch; + } + } +} diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.Designer.cs b/src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.Designer.cs new file mode 100644 index 00000000000..e8bdb9a3d7b --- /dev/null +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.Designer.cs @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace NuGet.DependencyResolver { + using System; + using System.Reflection; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "4.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Strings { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Strings() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("NuGet.DependencyResolver.Strings", typeof(Strings).GetTypeInfo().Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to The feed '{0}' lists package '{1}' but multiple attempts to download the nupkg have failed. The feed is either invalid or required packages were removed while the current operation was in progress. Verify the package exists on the feed and try again.. + /// + internal static string Error_PackageNotFoundWhenExpected { + get { + return ResourceManager.GetString("Error_PackageNotFoundWhenExpected", resourceCulture); + } + } + } +} diff --git a/src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.resx b/src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.resx new file mode 100644 index 00000000000..9646fb0dabc --- /dev/null +++ b/src/NuGet.Core/NuGet.DependencyResolver.Core/Strings.resx @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + The feed '{0}' lists package '{1}' but multiple attempts to download the nupkg have failed. The feed is either invalid or required packages were removed while the current operation was in progress. Verify the package exists on the feed and try again. + + \ No newline at end of file diff --git a/src/NuGet.Core/NuGet.LibraryModel/LibraryDependencyInfo.cs b/src/NuGet.Core/NuGet.LibraryModel/LibraryDependencyInfo.cs new file mode 100644 index 00000000000..00f472f4173 --- /dev/null +++ b/src/NuGet.Core/NuGet.LibraryModel/LibraryDependencyInfo.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using NuGet.Frameworks; + +namespace NuGet.LibraryModel +{ + public class LibraryDependencyInfo + { + /// + /// False if the package could not be found. + /// + public bool Resolved { get; } + + /// + /// Original library identity from the nuspec. + /// This contains the original casing for the id/version. + /// + public LibraryIdentity Library { get; } + + /// + /// Dependencies + /// + public IEnumerable Dependencies { get; } + + /// + /// Target framework used to select the dependencies. + /// + public NuGetFramework Framework { get; } + + public LibraryDependencyInfo( + LibraryIdentity library, + bool resolved, + NuGetFramework framework, + IEnumerable dependencies) + { + Resolved = resolved; + Library = library ?? throw new ArgumentNullException(nameof(library)); + Framework = framework ?? throw new ArgumentNullException(nameof(framework)); + Dependencies = dependencies ?? throw new ArgumentNullException(nameof(dependencies)); + } + + /// + /// Unresolved + /// + public static LibraryDependencyInfo CreateUnresolved(LibraryIdentity library, NuGetFramework framework) + { + return new LibraryDependencyInfo(library, resolved: false, framework: framework, dependencies: Enumerable.Empty()); + } + + /// + /// Resolved + /// + public static LibraryDependencyInfo Create(LibraryIdentity library, NuGetFramework framework, IEnumerable dependencies) + { + return new LibraryDependencyInfo(library, resolved: false, framework: framework, dependencies: dependencies); + } + } +} diff --git a/src/NuGet.Core/NuGet.LibraryModel/LibraryIdentity.cs b/src/NuGet.Core/NuGet.LibraryModel/LibraryIdentity.cs index de86be04a1d..06d06d64171 100644 --- a/src/NuGet.Core/NuGet.LibraryModel/LibraryIdentity.cs +++ b/src/NuGet.Core/NuGet.LibraryModel/LibraryIdentity.cs @@ -1,7 +1,8 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using NuGet.Shared; using NuGet.Versioning; namespace NuGet.LibraryModel @@ -40,9 +41,10 @@ public bool Equals(LibraryIdentity other) { return true; } - return string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase) && - Equals(Version, other.Version) && - Equals(Type, other.Type); + + return Equals(Type, other.Type) + && Equals(Version, other.Version) + && string.Equals(Name, other.Name, StringComparison.OrdinalIgnoreCase); } public override bool Equals(object obj) @@ -52,12 +54,13 @@ public override bool Equals(object obj) public override int GetHashCode() { - unchecked - { - return ((Name != null ? StringComparer.OrdinalIgnoreCase.GetHashCode(Name) : 0) * 397) ^ - (Version != null ? Version.GetHashCode() : 0) ^ - (Type != null ? Type.GetHashCode() : 0); - } + var combiner = new HashCodeCombiner(); + + combiner.AddStringIgnoreCase(Name); + combiner.AddObject(Version); + combiner.AddObject(Type); + + return combiner.CombinedHash; } public static bool operator ==(LibraryIdentity left, LibraryIdentity right) diff --git a/src/NuGet.Core/NuGet.LibraryModel/LibraryRange.cs b/src/NuGet.Core/NuGet.LibraryModel/LibraryRange.cs index ad8c2fb6fd0..5bf1ff4e332 100644 --- a/src/NuGet.Core/NuGet.LibraryModel/LibraryRange.cs +++ b/src/NuGet.Core/NuGet.LibraryModel/LibraryRange.cs @@ -3,6 +3,7 @@ using System; using System.Text; +using NuGet.Shared; using NuGet.Versioning; namespace NuGet.LibraryModel @@ -146,12 +147,13 @@ public override bool Equals(object obj) public override int GetHashCode() { - unchecked - { - return ((Name != null ? StringComparer.OrdinalIgnoreCase.GetHashCode(Name) : 0) * 397) ^ - (VersionRange != null ? VersionRange.GetHashCode() : 0) ^ - TypeConstraint.GetHashCode(); - } + var combiner = new HashCodeCombiner(); + + combiner.AddStringIgnoreCase(Name); + combiner.AddObject(VersionRange); + combiner.AddObject(TypeConstraint); + + return combiner.CombinedHash; } public static bool operator ==(LibraryRange left, LibraryRange right) diff --git a/src/NuGet.Core/NuGet.Protocol/Exceptions/InvalidCacheProtocolException.cs b/src/NuGet.Core/NuGet.Protocol/Exceptions/InvalidCacheProtocolException.cs new file mode 100644 index 00000000000..8ccfd79d6b9 --- /dev/null +++ b/src/NuGet.Core/NuGet.Protocol/Exceptions/InvalidCacheProtocolException.cs @@ -0,0 +1,24 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Protocol +{ + /// + /// Failure due to an invalid cache. + /// + public abstract class InvalidCacheProtocolException : FatalProtocolException + { + public InvalidCacheProtocolException(string message) + : base(message) + { + } + + public InvalidCacheProtocolException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/NuGet.Core/NuGet.Protocol/Exceptions/PackageNotFoundProtocolException.cs b/src/NuGet.Core/NuGet.Protocol/Exceptions/PackageNotFoundProtocolException.cs new file mode 100644 index 00000000000..b695fbf261a --- /dev/null +++ b/src/NuGet.Core/NuGet.Protocol/Exceptions/PackageNotFoundProtocolException.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Globalization; +using NuGet.Packaging.Core; +using NuGet.Protocol.Core.Types; + +namespace NuGet.Protocol +{ + /// + /// Thrown when a package cannot be found on a feed. + /// + public class PackageNotFoundProtocolException : InvalidCacheProtocolException + { + /// + /// Package that was not found. + /// + public PackageIdentity PackageIdentity { get; } + + public PackageNotFoundProtocolException(PackageIdentity package) + : base(GetMessage(package)) + { + PackageIdentity = package ?? throw new ArgumentNullException(nameof(package)); + } + + public PackageNotFoundProtocolException(PackageIdentity package, Exception innerException) + : base(GetMessage(package), innerException) + { + PackageIdentity = package ?? throw new ArgumentNullException(nameof(package)); + } + + private static string GetMessage(PackageIdentity package) + { + return string.Format( + CultureInfo.CurrentCulture, + Strings.PackageNotFound, + package.ToString()); + } + } +} diff --git a/src/NuGet.Core/NuGet.Protocol/FindPackageByIdDependencyInfo.cs b/src/NuGet.Core/NuGet.Protocol/FindPackageByIdDependencyInfo.cs index d9fd4d4d06b..393cb9572ba 100644 --- a/src/NuGet.Core/NuGet.Protocol/FindPackageByIdDependencyInfo.cs +++ b/src/NuGet.Core/NuGet.Protocol/FindPackageByIdDependencyInfo.cs @@ -5,17 +5,36 @@ using System.Collections.Generic; using System.Linq; using NuGet.Packaging; +using NuGet.Packaging.Core; namespace NuGet.Protocol.Core.Types { public class FindPackageByIdDependencyInfo { + /// + /// Original package identity from the package. + /// This contains the exact casing for the id and version. + /// + public PackageIdentity PackageIdentity { get; } + + /// + /// Gets the package dependecy groups. + /// + public IReadOnlyList DependencyGroups { get; } + + /// + /// Gets the framework reference groups. + /// + public IReadOnlyList FrameworkReferenceGroups { get; } + /// /// DependencyInfo /// + /// original package identity /// package dependency groups /// Sequence of s. public FindPackageByIdDependencyInfo( + PackageIdentity packageIdentity, IEnumerable dependencyGroups, IEnumerable frameworkReferenceGroups) { @@ -29,18 +48,9 @@ public FindPackageByIdDependencyInfo( throw new ArgumentNullException(nameof(frameworkReferenceGroups)); } + PackageIdentity = packageIdentity ?? throw new ArgumentNullException(nameof(packageIdentity)); DependencyGroups = dependencyGroups.ToList(); FrameworkReferenceGroups = frameworkReferenceGroups.ToList(); } - - /// - /// Gets the package dependecy groups. - /// - public IReadOnlyList DependencyGroups { get; } - - /// - /// Gets the framework reference groups. - /// - public IReadOnlyList FrameworkReferenceGroups { get; } } } diff --git a/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV2FindPackageByIdResource.cs b/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV2FindPackageByIdResource.cs index bccdd5b5104..1ab05ac26d7 100644 --- a/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV2FindPackageByIdResource.cs +++ b/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV2FindPackageByIdResource.cs @@ -39,26 +39,10 @@ public override Task> GetAllVersionsAsync( ILogger logger, CancellationToken token) { - var infos = GetPackageInfos(id, logger); + var infos = GetPackageInfos(id, cacheContext, logger); return Task.FromResult(infos.Select(p => p.Identity.Version)); } - public override Task GetOriginalIdentityAsync( - string id, - NuGetVersion version, - SourceCacheContext cacheContext, - ILogger logger, - CancellationToken token) - { - var info = GetPackageInfo(id, version, logger); - if (info != null) - { - return Task.FromResult(info.Identity); - } - - return Task.FromResult(null); - } - public override async Task CopyNupkgToStreamAsync( string id, NuGetVersion version, @@ -67,7 +51,7 @@ public override async Task CopyNupkgToStreamAsync( ILogger logger, CancellationToken token) { - var info = GetPackageInfo(id, version, logger); + var info = GetPackageInfo(id, version, cacheContext, logger); if (info != null) { @@ -89,7 +73,7 @@ public override Task GetDependencyInfoAsync( CancellationToken token) { FindPackageByIdDependencyInfo dependencyInfo = null; - var info = GetPackageInfo(id, version, logger); + var info = GetPackageInfo(id, version, cacheContext, logger); if (info != null) { dependencyInfo = GetDependencyInfo(info.Nuspec); @@ -98,14 +82,27 @@ public override Task GetDependencyInfoAsync( return Task.FromResult(dependencyInfo); } - private LocalPackageInfo GetPackageInfo(string id, NuGetVersion version, ILogger logger) + private LocalPackageInfo GetPackageInfo(string id, NuGetVersion version, SourceCacheContext cacheContext, ILogger logger) { - return GetPackageInfos(id, logger).FirstOrDefault(package => package.Identity.Version == version); + return GetPackageInfos(id, cacheContext, logger).FirstOrDefault(package => package.Identity.Version == version); } - private IReadOnlyList GetPackageInfos(string id, ILogger logger) + private IReadOnlyList GetPackageInfos(string id, SourceCacheContext cacheContext, ILogger logger) { - return _packageInfoCache.GetOrAdd(id, (packageId) => GetPackageInfosCore(packageId, logger)); + IReadOnlyList results = null; + + Func> findPackages = (packageId) => GetPackageInfosCore(packageId, logger); + + if (cacheContext.RefreshMemoryCache) + { + results = _packageInfoCache.AddOrUpdate(id, findPackages, (k, v) => findPackages(k)); + } + else + { + results = _packageInfoCache.GetOrAdd(id, findPackages); + } + + return results; } private IReadOnlyList GetPackageInfosCore(string id, ILogger logger) diff --git a/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV3FindPackageByIdResource.cs b/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV3FindPackageByIdResource.cs index 5a49de4ab66..6c0b1627f03 100644 --- a/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV3FindPackageByIdResource.cs +++ b/src/NuGet.Core/NuGet.Protocol/LocalRepositories/LocalV3FindPackageByIdResource.cs @@ -24,8 +24,6 @@ public class LocalV3FindPackageByIdResource : FindPackageByIdResource // Use cache insensitive compare for windows private readonly ConcurrentDictionary> _cache = new ConcurrentDictionary>(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary _packageIdentityCache - = new ConcurrentDictionary(); private readonly string _source; private readonly VersionFolderPathResolver _resolver; @@ -49,7 +47,7 @@ public override Task> GetAllVersionsAsync( ILogger logger, CancellationToken token) { - return Task.FromResult(GetVersions(id, logger).AsEnumerable()); + return Task.FromResult(GetVersions(id, cacheContext, logger).AsEnumerable()); } public override async Task CopyNupkgToStreamAsync( @@ -60,7 +58,7 @@ public override async Task CopyNupkgToStreamAsync( ILogger logger, CancellationToken token) { - var matchedVersion = GetVersion(id, version, logger); + var matchedVersion = GetVersion(id, version, cacheContext, logger); if (matchedVersion != null) { @@ -76,31 +74,6 @@ public override async Task CopyNupkgToStreamAsync( return false; } - public override Task GetOriginalIdentityAsync( - string id, - NuGetVersion version, - SourceCacheContext cacheContext, - ILogger logger, - CancellationToken token) - { - var matchedVersion = GetVersion(id, version, logger); - PackageIdentity outputIdentity = null; - if (matchedVersion != null) - { - outputIdentity = _packageIdentityCache.GetOrAdd( - new PackageIdentity(id, matchedVersion), - inputIdentity => - { - return ProcessNuspecReader( - inputIdentity.Id, - inputIdentity.Version, - nuspecReader => nuspecReader.GetIdentity()); - }); - } - - return Task.FromResult(outputIdentity); - } - public override Task GetDependencyInfoAsync( string id, NuGetVersion version, @@ -108,21 +81,19 @@ public override Task GetDependencyInfoAsync( ILogger logger, CancellationToken token) { - var matchedVersion = GetVersion(id, version, logger); + var matchedVersion = GetVersion(id, version, cacheContext, logger); FindPackageByIdDependencyInfo dependencyInfo = null; if (matchedVersion != null) { - dependencyInfo = ProcessNuspecReader( - id, - matchedVersion, - nuspecReader => - { - // Populate the package identity cache while we have the .nuspec open. - var identity = nuspecReader.GetIdentity(); - _packageIdentityCache.TryAdd(identity, identity); + var identity = new PackageIdentity(id, matchedVersion); - return GetDependencyInfo(nuspecReader); - }); + dependencyInfo = ProcessNuspecReader( + id, + matchedVersion, + nuspecReader => + { + return GetDependencyInfo(nuspecReader); + }); } return Task.FromResult(dependencyInfo); @@ -156,14 +127,27 @@ private T ProcessNuspecReader(string id, NuGetVersion version, Func v == version); + return GetVersions(id, cacheContext, logger).FirstOrDefault(v => v == version); } - private List GetVersions(string id, ILogger logger) + private List GetVersions(string id, SourceCacheContext cacheContext, ILogger logger) { - return _cache.GetOrAdd(id, keyId => GetVersionsCore(keyId, logger)); + List results = null; + + Func> findPackages = (keyId) => GetVersionsCore(keyId, logger); + + if (cacheContext.RefreshMemoryCache) + { + results = _cache.AddOrUpdate(id, findPackages, (k, v) => findPackages(k)); + } + else + { + results = _cache.GetOrAdd(id, findPackages); + } + + return results; } private List GetVersionsCore(string id, ILogger logger) diff --git a/src/NuGet.Core/NuGet.Protocol/PackagesFolder/LocalPackageInfo.cs b/src/NuGet.Core/NuGet.Protocol/PackagesFolder/LocalPackageInfo.cs index 07c694596db..9e668106513 100644 --- a/src/NuGet.Core/NuGet.Protocol/PackagesFolder/LocalPackageInfo.cs +++ b/src/NuGet.Core/NuGet.Protocol/PackagesFolder/LocalPackageInfo.cs @@ -1,7 +1,7 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System.IO; +using System; using NuGet.Packaging; using NuGet.Versioning; @@ -9,20 +9,22 @@ namespace NuGet.Repositories { public class LocalPackageInfo { - private NuspecReader _nuspec; + private readonly Lazy _nuspec; public LocalPackageInfo( string packageId, NuGetVersion version, string path, string manifestPath, - string zipPath) + string zipPath, + Lazy nuspec) { Id = packageId; Version = version; ExpandedPath = path; ManifestPath = manifestPath; ZipPath = zipPath; + _nuspec = nuspec; } public string Id { get; } @@ -39,30 +41,7 @@ public LocalPackageInfo( /// Caches the nuspec reader. /// If the nuspec does not exist this will throw a friendly exception. /// - public NuspecReader Nuspec - { - get - { - if (_nuspec == null) - { - // Verify that the nuspec has the correct name before opening it - if (File.Exists(ManifestPath)) - { - _nuspec = new NuspecReader(File.OpenRead(ManifestPath)); - } - else - { - // Scan the folder for the nuspec - var folderReader = new PackageFolderReader(ExpandedPath); - - // This will throw if the nuspec is not found - _nuspec = new NuspecReader(folderReader.GetNuspec()); - } - } - - return _nuspec; - } - } + public NuspecReader Nuspec => _nuspec.Value; public override string ToString() { diff --git a/src/NuGet.Core/NuGet.Protocol/PackagesFolder/NuGetv3LocalRepository.cs b/src/NuGet.Core/NuGet.Protocol/PackagesFolder/NuGetv3LocalRepository.cs index 64d22569839..8719b75b29b 100644 --- a/src/NuGet.Core/NuGet.Protocol/PackagesFolder/NuGetv3LocalRepository.cs +++ b/src/NuGet.Core/NuGet.Protocol/PackagesFolder/NuGetv3LocalRepository.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using NuGet.Common; using NuGet.Packaging; using NuGet.Versioning; @@ -19,7 +20,7 @@ public class NuGetv3LocalRepository { // Folder path -> Package private readonly ConcurrentDictionary _packageCache - = new ConcurrentDictionary(StringComparer.Ordinal); + = new ConcurrentDictionary(PathUtility.GetStringComparerBasedOnOS()); // Id -> Packages private readonly ConcurrentDictionary> _cache @@ -29,6 +30,10 @@ private readonly ConcurrentDictionary> _ca private readonly ConcurrentDictionary _idLocks = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + // Cache nuspecs lazily + private readonly ConcurrentDictionary> _nuspecCache + = new ConcurrentDictionary>(PathUtility.GetStringComparerBasedOnOS()); + public VersionFolderPathResolver PathResolver { get; } public NuGetv3LocalRepository(string path) @@ -49,12 +54,24 @@ public LocalPackageInfo FindPackage(string packageId, NuGetVersion version) return null; } + // Check for an exact match on casing + if (StringComparer.Ordinal.Equals(packageId, package.Id) + && StringComparer.Ordinal.Equals(version.ToNormalizedString(), package.Version.ToNormalizedString())) + { + return package; + } + + // nuspec + var nuspec = _nuspecCache.GetOrAdd(package.ExpandedPath, new Lazy(() => GetNuspec(package.ManifestPath, package.ExpandedPath))); + + // Create a new info to match the given id/version return new LocalPackageInfo( packageId, version, package.ExpandedPath, package.ManifestPath, - package.ZipPath); + package.ZipPath, + nuspec); } public IEnumerable FindPackagesById(string packageId) @@ -108,7 +125,9 @@ private List GetPackages(string id) var manifestPath = PathResolver.GetManifestFilePath(id, version); var zipPath = PathResolver.GetPackageFilePath(id, version); - package = new LocalPackageInfo(id, version, fullVersionDir, manifestPath, zipPath); + var nuspec = _nuspecCache.GetOrAdd(fullVersionDir, new Lazy(() => GetNuspec(manifestPath, fullVersionDir))); + + package = new LocalPackageInfo(id, version, fullVersionDir, manifestPath, zipPath, nuspec); // Cache the package, if it is valid it will not change // for the life of this restore. @@ -148,5 +167,26 @@ private object GetLockObj(string privateId) { return _idLocks.GetOrAdd(privateId, new object()); } + + private static NuspecReader GetNuspec(string manifest, string expanded) + { + NuspecReader nuspec = null; + + // Verify that the nuspec has the correct name before opening it + if (File.Exists(manifest)) + { + nuspec = new NuspecReader(File.OpenRead(manifest)); + } + else + { + // Scan the folder for the nuspec + var folderReader = new PackageFolderReader(expanded); + + // This will throw if the nuspec is not found + nuspec = new NuspecReader(folderReader.GetNuspec()); + } + + return nuspec; + } } } \ No newline at end of file diff --git a/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/HttpFileSystemBasedFindPackageByIdResource.cs b/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/HttpFileSystemBasedFindPackageByIdResource.cs index 791ddbdbe00..1d8f37a8a04 100644 --- a/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/HttpFileSystemBasedFindPackageByIdResource.cs +++ b/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/HttpFileSystemBasedFindPackageByIdResource.cs @@ -9,7 +9,6 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Newtonsoft.Json.Linq; using NuGet.Common; using NuGet.Packaging.Core; using NuGet.Protocol.Core.Types; @@ -31,10 +30,8 @@ public class HttpFileSystemBasedFindPackageByIdResource : FindPackageByIdResourc { private const int MaxRetries = 3; private readonly HttpSource _httpSource; - private readonly ConcurrentDictionary>> _packageInfoCache = - new ConcurrentDictionary>>(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary> _packageIdentityCache - = new ConcurrentDictionary>(); + private readonly ConcurrentDictionary>> _packageInfoCache = + new ConcurrentDictionary>>(StringComparer.OrdinalIgnoreCase); private readonly IReadOnlyList _baseUris; private readonly FindPackagesByIdNupkgDownloader _nupkgDownloader; @@ -71,36 +68,6 @@ public override async Task> GetAllVersionsAsync( return packageInfos.Keys; } - public override async Task GetOriginalIdentityAsync( - string id, - NuGetVersion version, - SourceCacheContext cacheContext, - ILogger logger, - CancellationToken cancellationToken) - { - var packageInfos = await EnsurePackagesAsync(id, cacheContext, logger, cancellationToken); - - PackageInfo packageInfo; - if (!packageInfos.TryGetValue(version, out packageInfo)) - { - return null; - } - - return await _packageIdentityCache.GetOrAdd( - packageInfo.Identity, - async identity => - { - var reader = await _nupkgDownloader.GetNuspecReaderFromNupkgAsync( - packageInfo.Identity, - packageInfo.ContentUri, - cacheContext, - logger, - cancellationToken); - - return reader.GetIdentity(); - }); - } - public override async Task GetDependencyInfoAsync( string id, NuGetVersion version, @@ -120,10 +87,6 @@ public override async Task GetDependencyInfoAsync logger, cancellationToken); - // Populate the package identity cache while we have the .nuspec open. - var identity = reader.GetIdentity(); - _packageIdentityCache.TryAdd(identity, Task.FromResult(identity)); - return GetDependencyInfo(reader); } @@ -155,17 +118,33 @@ public override async Task CopyNupkgToStreamAsync( return false; } - private Task> EnsurePackagesAsync( + private async Task> EnsurePackagesAsync( string id, SourceCacheContext cacheContext, ILogger logger, CancellationToken cancellationToken) { - return _packageInfoCache.GetOrAdd(id, (keyId) => FindPackagesByIdAsync( - keyId, - cacheContext, - logger, - cancellationToken)); + AsyncLazy> result = null; + + Func>> findPackages = + (keyId) => new AsyncLazy>(() => FindPackagesByIdAsync( + keyId, + cacheContext, + logger, + cancellationToken)); + + if (cacheContext.RefreshMemoryCache) + { + // Update the cache + result = _packageInfoCache.AddOrUpdate(id, findPackages, (k, v) => findPackages(id)); + } + else + { + // Read the cache if it exists + result = _packageInfoCache.GetOrAdd(id, findPackages); + } + + return await result; } private async Task> FindPackagesByIdAsync( @@ -246,7 +225,7 @@ private async Task> FindPackagesById private async Task> ConsumeFlatContainerIndexAsync(Stream stream, string id, string baseUri) { - JObject doc = await stream.AsJObjectAsync(); + var doc = await stream.AsJObjectAsync(); var streamResults = new SortedDictionary(); diff --git a/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV2FindPackageByIdResource.cs b/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV2FindPackageByIdResource.cs index 225e64e35ea..c16e68fb8f3 100644 --- a/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV2FindPackageByIdResource.cs +++ b/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV2FindPackageByIdResource.cs @@ -35,8 +35,6 @@ public class RemoteV2FindPackageByIdResource : FindPackageByIdResource private readonly string _baseUri; private readonly HttpSource _httpSource; private readonly Dictionary>> _packageVersionsCache = new Dictionary>>(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary> _packageIdentityCache - = new ConcurrentDictionary>(); private readonly FindPackagesByIdNupkgDownloader _nupkgDownloader; public RemoteV2FindPackageByIdResource(PackageSource packageSource, HttpSource httpSource) @@ -60,40 +58,6 @@ public override async Task> GetAllVersionsAsync( return result.Select(item => item.Identity.Version); } - public override async Task GetOriginalIdentityAsync( - string id, - NuGetVersion version, - SourceCacheContext cacheContext, - ILogger logger, - CancellationToken cancellationToken) - { - return await _packageIdentityCache.GetOrAdd( - new PackageIdentity(id, version), - async original => - { - var packageInfo = await GetPackageInfoAsync( - original.Id, - original.Version, - cacheContext, - logger, - cancellationToken); - - if (packageInfo == null) - { - return null; - } - - var reader = await _nupkgDownloader.GetNuspecReaderFromNupkgAsync( - packageInfo.Identity, - packageInfo.ContentUri, - cacheContext, - logger, - cancellationToken); - - return reader.GetIdentity(); - }); - } - public override async Task GetDependencyInfoAsync( string id, NuGetVersion version, @@ -115,11 +79,6 @@ public override async Task GetDependencyInfoAsync logger, cancellationToken); - // Populate the package identity cache while we have the .nuspec open. - _packageIdentityCache.TryAdd( - new PackageIdentity(id, version), - Task.FromResult(reader.GetIdentity())); - return GetDependencyInfo(reader); } @@ -167,7 +126,7 @@ private Task> EnsurePackagesAsync( lock (_packageVersionsCache) { - if (!_packageVersionsCache.TryGetValue(id, out task)) + if (cacheContext.RefreshMemoryCache || !_packageVersionsCache.TryGetValue(id, out task)) { task = FindPackagesByIdAsyncCore(id, cacheContext, logger, cancellationToken); _packageVersionsCache[id] = task; @@ -267,7 +226,7 @@ private async Task> FindPackagesByIdAsyncCore( } catch (Exception ex) when (retry < 2) { - string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsyncCore), uri) + var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_RetryingFindPackagesById, nameof(FindPackagesByIdAsyncCore), uri) + Environment.NewLine + ExceptionUtilities.DisplayMessage(ex); logger.LogMinimal(message); diff --git a/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV3FindPackageByIdResource.cs b/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV3FindPackageByIdResource.cs index b42fe2cd0ad..602be466579 100644 --- a/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV3FindPackageByIdResource.cs +++ b/src/NuGet.Core/NuGet.Protocol/RemoteRepositories/RemoteV3FindPackageByIdResource.cs @@ -41,26 +41,10 @@ public override async Task> GetAllVersionsAsync( ILogger logger, CancellationToken cancellationToken) { - var result = await EnsurePackagesAsync(id, logger, cancellationToken); + var result = await EnsurePackagesAsync(id, cacheContext, logger, cancellationToken); return result.Select(item => item.Identity.Version); } - public override async Task GetOriginalIdentityAsync( - string id, - NuGetVersion version, - SourceCacheContext cacheContext, - ILogger logger, - CancellationToken cancellationToken) - { - var packageInfo = await GetPackageInfoAsync(id, version, logger, cancellationToken); - if (packageInfo == null) - { - return null; - } - - return packageInfo.Identity; - } - public override async Task GetDependencyInfoAsync( string id, NuGetVersion version, @@ -68,7 +52,7 @@ public override async Task GetDependencyInfoAsync ILogger logger, CancellationToken cancellationToken) { - var packageInfo = await GetPackageInfoAsync(id, version, logger, cancellationToken); + var packageInfo = await GetPackageInfoAsync(id, version, cacheContext, logger, cancellationToken); if (packageInfo == null) { return null; @@ -92,7 +76,7 @@ public override async Task CopyNupkgToStreamAsync( ILogger logger, CancellationToken cancellationToken) { - var packageInfo = await GetPackageInfoAsync(id, version, logger, cancellationToken); + var packageInfo = await GetPackageInfoAsync(id, version, cacheContext, logger, cancellationToken); if (packageInfo == null) { return false; @@ -110,15 +94,17 @@ public override async Task CopyNupkgToStreamAsync( private async Task GetPackageInfoAsync( string id, NuGetVersion version, + SourceCacheContext cacheContext, ILogger logger, CancellationToken cancellationToken) { - var packageInfos = await EnsurePackagesAsync(id, logger, cancellationToken); + var packageInfos = await EnsurePackagesAsync(id, cacheContext, logger, cancellationToken); return packageInfos.FirstOrDefault(p => p.Identity.Version == version); } private Task> EnsurePackagesAsync( string id, + SourceCacheContext cacheContext, ILogger logger, CancellationToken cancellationToken) { @@ -126,7 +112,7 @@ private Task> EnsurePackagesAsync( lock (_packageVersionsCache) { - if (!_packageVersionsCache.TryGetValue(id, out task)) + if (cacheContext.RefreshMemoryCache || !_packageVersionsCache.TryGetValue(id, out task)) { task = FindPackagesByIdAsyncCore(id, logger, cancellationToken); _packageVersionsCache[id] = task; diff --git a/src/NuGet.Core/NuGet.Protocol/Resources/FindPackageByIdResource.cs b/src/NuGet.Core/NuGet.Protocol/Resources/FindPackageByIdResource.cs index 9d1a42978f0..0cd89402a82 100644 --- a/src/NuGet.Core/NuGet.Protocol/Resources/FindPackageByIdResource.cs +++ b/src/NuGet.Core/NuGet.Protocol/Resources/FindPackageByIdResource.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using NuGet.Common; using NuGet.Packaging; -using NuGet.Packaging.Core; using NuGet.Versioning; namespace NuGet.Protocol.Core.Types @@ -48,24 +47,6 @@ public abstract Task CopyNupkgToStreamAsync( ILogger logger, CancellationToken token); - /// - /// Gets the original ID and version for a package. This is useful when finding the - /// canonical casing for a package ID. Note that the casing of a package ID can vary from - /// version to version. - /// - /// The package ID. This value is case insensitive. - /// The version. - /// The source cache context. - /// The logger. - /// The cancellation token. - /// The package identity, with the ID having the case provided by the package author. - public abstract Task GetOriginalIdentityAsync( - string id, - NuGetVersion version, - SourceCacheContext cacheContext, - ILogger logger, - CancellationToken token); - /// /// Read dependency info from a nuspec. /// @@ -78,6 +59,7 @@ protected static FindPackageByIdDependencyInfo GetDependencyInfo(NuspecReader re // Create dependency info return new FindPackageByIdDependencyInfo( + reader.GetIdentity(), reader.GetDependencyGroups(), reader.GetFrameworkReferenceGroups()); } diff --git a/src/NuGet.Core/NuGet.Protocol/SourceCacheContext.cs b/src/NuGet.Core/NuGet.Protocol/SourceCacheContext.cs index 0c895f34cc1..8b36afdc91a 100644 --- a/src/NuGet.Core/NuGet.Protocol/SourceCacheContext.cs +++ b/src/NuGet.Core/NuGet.Protocol/SourceCacheContext.cs @@ -41,6 +41,13 @@ public class SourceCacheContext : IDisposable /// If the value is null the default expiration will be used. public DateTimeOffset? MaxAge { get; set; } + /// + /// Force the in-memory cache to reload. This avoids allowing other calls to populate + /// the memory cache again from cached files on disk using a different source context. + /// This should only be used for retries. + /// + public bool RefreshMemoryCache { get; set; } + /// /// Package version lists from the server older than this time span /// will be fetched from the server. @@ -91,18 +98,50 @@ public virtual string GeneratedTempFolder "TempCache", Guid.NewGuid().ToString()); - Interlocked.CompareExchange(ref _generatedTempFolder, newTempFolder, null); + Interlocked.CompareExchange(ref _generatedTempFolder, newTempFolder, comparand: null); } return _generatedTempFolder; } + + set => Interlocked.CompareExchange(ref _generatedTempFolder, value, comparand: null); } public bool IgnoreFailedSources { get; set; } + /// + /// Clones the current SourceCacheContext. + /// + public virtual SourceCacheContext Clone() + { + return new SourceCacheContext() + { + DirectDownload = DirectDownload, + IgnoreFailedSources = IgnoreFailedSources, + MaxAge = MaxAge, + NoCache = NoCache, + GeneratedTempFolder = _generatedTempFolder, + RefreshMemoryCache = RefreshMemoryCache + }; + } + + /// + /// Clones the current cache context and does the following: + /// 1. Sets MaxAge to Now + /// 2. RefreshMemoryCache to true + /// + public virtual SourceCacheContext WithRefreshCacheTrue() + { + var updatedContext = Clone(); + updatedContext.MaxAge = DateTimeOffset.UtcNow; + updatedContext.RefreshMemoryCache = true; + + return updatedContext; + } + public void Dispose() { - var currentTempFolder = Interlocked.CompareExchange(ref _generatedTempFolder, null, null); + var currentTempFolder = Interlocked.CompareExchange(ref _generatedTempFolder, value: null, comparand: null); if (currentTempFolder != null) { diff --git a/src/NuGet.Core/NuGet.Protocol/Strings.Designer.cs b/src/NuGet.Core/NuGet.Protocol/Strings.Designer.cs index a463a289fa5..3e7aa2f4fd3 100644 --- a/src/NuGet.Core/NuGet.Protocol/Strings.Designer.cs +++ b/src/NuGet.Core/NuGet.Protocol/Strings.Designer.cs @@ -322,15 +322,6 @@ internal static string Log_FailedToFetchV2Feed { } } - /// - /// Looks up a localized string similar to Unable to load package '{0}'.. - /// - internal static string Log_FailedToGetNupkgStream { - get { - return ResourceManager.GetString("Log_FailedToGetNupkgStream", resourceCulture); - } - } - /// /// Looks up a localized string similar to Unable to load nuspec from package '{0}'.. /// @@ -493,6 +484,15 @@ internal static string PackageActionDescriptionWrapper_UnrecognizedAction { } } + /// + /// Looks up a localized string similar to Unable to find package '{0}'.. + /// + internal static string PackageNotFound { + get { + return ResourceManager.GetString("PackageNotFound", resourceCulture); + } + } + /// /// Looks up a localized string similar to ERROR: This version of nuget.exe does not support updating packages to package source '{0}'.. /// diff --git a/src/NuGet.Core/NuGet.Protocol/Strings.resx b/src/NuGet.Core/NuGet.Protocol/Strings.resx index 99348f3118e..ebc6a7027f2 100644 --- a/src/NuGet.Core/NuGet.Protocol/Strings.resx +++ b/src/NuGet.Core/NuGet.Protocol/Strings.resx @@ -110,8 +110,8 @@ The "ms" should be localized to the abbreviation for milliseconds. {0} is the identity of the package. {1} is the URI. - - Unable to load package '{0}'. + + Unable to find package '{0}'. Unable to load nuspec from package '{0}'. diff --git a/src/NuGet.Core/NuGet.Protocol/Utility/FindPackagesByIdNupkgDownloader.cs b/src/NuGet.Core/NuGet.Protocol/Utility/FindPackagesByIdNupkgDownloader.cs index 3ba54364984..67bb846fcef 100644 --- a/src/NuGet.Core/NuGet.Protocol/Utility/FindPackagesByIdNupkgDownloader.cs +++ b/src/NuGet.Core/NuGet.Protocol/Utility/FindPackagesByIdNupkgDownloader.cs @@ -79,10 +79,10 @@ await ProcessNupkgStreamAsync( if (reader == null) { - throw new FatalProtocolException(string.Format( - CultureInfo.CurrentCulture, - Strings.Log_FailedToGetNupkgStream, - identity.Id)); + // The package was not found on the feed. This typically means + // that the feed listed the package, but then returned 404 for the nupkg. + // The cache needs to be invaldiated and the download call made again. + throw new PackageNotFoundProtocolException(identity); } lock (_nuspecReadersLock) @@ -277,7 +277,7 @@ private async Task ProcessHttpSourceResultAsync( catch (TaskCanceledException) when (retry < 2) { // Requests can get cancelled if we got the data from elsewhere, no reason to warn. - string message = string.Format(CultureInfo.CurrentCulture, Strings.Log_CanceledNupkgDownload, url); + var message = string.Format(CultureInfo.CurrentCulture, Strings.Log_CanceledNupkgDownload, url); logger.LogMinimal(message); } diff --git a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/FeedPackagePruningTests.cs b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/FeedPackagePruningTests.cs new file mode 100644 index 00000000000..6fd58b4621d --- /dev/null +++ b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/FeedPackagePruningTests.cs @@ -0,0 +1,165 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using System.Net; +using Newtonsoft.Json.Linq; +using NuGet.Frameworks; +using NuGet.Protocol; +using NuGet.Test.Utility; +using Xunit; + +namespace NuGet.CommandLine.Test +{ + public class FeedPackagePruningTests + { + [Fact] + public void FeedPackagePruning_GivenThatAV3FeedPrunesAPackageDuringRestoreVerifyRestoreRecovers() + { + // Arrange + using (var server = new MockServer()) + using (var pathContext = new SimpleTestPathContext()) + { + // Set up solution, project, and packages + var testLogger = new TestLogger(); + var solution = new SimpleTestSolutionContext(pathContext.SolutionRoot); + var serverRepoPath = Path.Combine(pathContext.WorkingDirectory, "serverPackages"); + + var packageX100 = new SimpleTestPackageContext("x", "1.0.0"); + var packageX200 = new SimpleTestPackageContext("x", "2.0.0"); + + SimpleTestPackageUtility.CreatePackages( + serverRepoPath, + packageX100, + packageX200); + + var projectA = SimpleTestProjectContext.CreateNETCore( + "a", + pathContext.SolutionRoot, + NuGetFramework.Parse("net45")); + projectA.AddPackageToAllFrameworks(packageX200); + solution.Projects.Add(projectA); + + var projectB = SimpleTestProjectContext.CreateNETCore( + "b", + pathContext.SolutionRoot, + NuGetFramework.Parse("net45")); + projectB.AddPackageToAllFrameworks(packageX100); + solution.Projects.Add(projectB); + + solution.Create(pathContext.SolutionRoot); + + // Server setup + var indexJson = Util.CreateIndexJson(); + Util.AddFlatContainerResource(indexJson, server); + Util.AddRegistrationResource(indexJson, server); + + server.Get.Add("/", request => + { + return ServerHandlerV3(request, server, indexJson, serverRepoPath); + }); + + server.Start(); + + var feedUrl = server.Uri + "index.json"; + + // Restore x 2.0.0 and populate the http cache + var r = Util.Restore(pathContext, projectA.ProjectPath, 0, "-Source", feedUrl); + + // Delete x 1.0.0 + File.Delete(LocalFolderUtility.GetPackageV2(serverRepoPath, packageX100.Identity, testLogger).Path); + + // Act + // Restore x 1.0.0 + r = Util.Restore(pathContext, projectB.ProjectPath, 0, "-Source", feedUrl); + + var xLib = projectB.AssetsFile.Libraries.SingleOrDefault(e => e.Name == "x"); + + // Assert + Assert.Equal("2.0.0", xLib.Version.ToNormalizedString()); + } + } + + private Action ServerHandlerV3( + HttpListenerRequest request, + MockServer server, + JObject indexJson, + string repositoryPath) + { + try + { + var path = server.GetRequestUrlAbsolutePath(request); + var parts = request.Url.AbsolutePath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + + if (path == "/index.json") + { + return new Action(response => + { + response.StatusCode = 200; + response.ContentType = "text/javascript"; + MockServer.SetResponseContent(response, indexJson.ToString()); + }); + } + else if (path.StartsWith("/flat/") && path.EndsWith("/index.json")) + { + return new Action(response => + { + response.ContentType = "text/javascript"; + + var versionsJson = JObject.Parse(@"{ ""versions"": [] }"); + var array = versionsJson["versions"] as JArray; + + var id = parts[parts.Length - 2]; + + foreach (var pkg in LocalFolderUtility.GetPackagesV2(repositoryPath, id, new TestLogger())) + { + array.Add(pkg.Identity.Version.ToNormalizedString()); + } + + MockServer.SetResponseContent(response, versionsJson.ToString()); + }); + } + else if (path.StartsWith("/flat/") && path.EndsWith(".nupkg")) + { + var file = new FileInfo(Path.Combine(repositoryPath, parts.Last())); + + if (file.Exists) + { + return new Action(response => + { + response.ContentType = "application/zip"; + using (var stream = file.OpenRead()) + { + var content = stream.ReadAllBytes(); + MockServer.SetResponseContent(response, content); + } + }); + } + else + { + return new Action(response => + { + response.StatusCode = 404; + }); + } + } + else if (path == "/nuget") + { + return new Action(response => + { + response.StatusCode = 200; + }); + } + + throw new Exception("This test needs to be updated to support: " + path); + } + catch (Exception) + { + // Debug here + throw; + } + } + } +} diff --git a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/RestoreNETCoreTest.cs b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/RestoreNETCoreTest.cs index 83ed6dd2cc5..be8d003460a 100644 --- a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/RestoreNETCoreTest.cs +++ b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/RestoreNETCoreTest.cs @@ -100,7 +100,7 @@ public void RestoreNetCore_VerifyPerProjectConfigSourcesAreUsedForChildProjectsW solution.Create(pathContext.SolutionRoot); // Act - var r = Restore(pathContext, projectRoot.ProjectPath, exitCode: 0, additionalArgs: "-Recursive"); + var r = Util.Restore(pathContext, projectRoot.ProjectPath, expectedExitCode: 0, additionalArgs: "-Recursive"); // Assert Assert.True(projects.Count > 0); @@ -190,7 +190,7 @@ public void RestoreNetCore_VerifyProjectConfigCanOverrideSolutionConfig() solution.Create(pathContext.SolutionRoot); // Act - var r = Restore(pathContext, project.ProjectPath); + var r = Util.Restore(pathContext, project.ProjectPath); // Assert Assert.True(project.AssetsFile.Libraries.Select(e => e.Name).Contains("packageB")); @@ -229,7 +229,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var dgPath = Path.Combine(pathContext.WorkingDirectory, "out.dg"); var dgSpec = DependencyGraphSpec.Load(dgPath); @@ -289,7 +289,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var dgPath = Path.Combine(pathContext.WorkingDirectory, "out.dg"); var dgSpec = DependencyGraphSpec.Load(dgPath); @@ -354,7 +354,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( projectXML.Save(projectA.ProjectPath); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var dgPath = Path.Combine(pathContext.WorkingDirectory, "out.dg"); var dgSpec = DependencyGraphSpec.Load(dgPath); @@ -542,7 +542,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -594,7 +594,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext, exitCode: 1); + var r = Util.RestoreSolution(pathContext, expectedExitCode: 1); // Assert Assert.True(r.Item1 == 1); @@ -638,7 +638,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(r.Item1 == 0); @@ -692,7 +692,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext, exitCode: 1); + var r = Util.RestoreSolution(pathContext, expectedExitCode: 1); // Assert Assert.True(r.Item1 == 1); @@ -733,7 +733,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -782,7 +782,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -831,7 +831,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -872,7 +872,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -950,7 +950,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var zPath = Path.Combine(pathContext.UserPackagesFolder, ".tools", "z"); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert // Version should not be used @@ -1012,7 +1012,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var zPath = Path.Combine(pathContext.UserPackagesFolder, ".tools", "z"); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(path), r.Item2); @@ -1070,7 +1070,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var path = Path.Combine(pathContext.UserPackagesFolder, ".tools", "z", "1.0.0", "netcoreapp1.0", "project.assets.json"); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(path), r.Item2); @@ -1132,7 +1132,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var path = Path.Combine(pathContext.UserPackagesFolder, ".tools", "z", "1.0.0", "netcoreapp1.0", "project.assets.json"); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(path), r.Item2); @@ -1194,11 +1194,11 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var path = Path.Combine(pathContext.UserPackagesFolder, ".tools", "z", "1.0.0", "netcoreapp1.0", "project.assets.json"); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); File.AppendAllText(path, "\n\n\n\n\n"); - r = RestoreSolution(pathContext); + r = Util.RestoreSolution(pathContext); var text = File.ReadAllText(path); @@ -1249,7 +1249,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var path = Path.Combine(pathContext.UserPackagesFolder, ".tools", "x", "1.0.0", "netcoreapp1.0", "project.assets.json"); // Act - var r = RestoreSolution(pathContext, exitCode: 0, additionalArgs: "-Recursive"); + var r = Util.RestoreSolution(pathContext, expectedExitCode: 0, additionalArgs: "-Recursive"); // Assert Assert.True(File.Exists(path), r.Item2); @@ -1297,7 +1297,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( var path = Path.Combine(pathContext.UserPackagesFolder, ".tools", "x", "1.0.0", "netcoreapp1.0", "project.assets.json"); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.False(File.Exists(path), r.Item2); @@ -1374,7 +1374,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.TargetsOutput), r.Item2); @@ -1450,7 +1450,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.TargetsOutput), r.Item2); @@ -1500,7 +1500,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.TargetsOutput), r.Item2); @@ -1560,7 +1560,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.False(File.Exists(projectA.TargetsOutput), r.Item2); @@ -1603,7 +1603,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var msbuildTargetsItems = TargetsUtility.GetMSBuildPackageImports(projectA.TargetsOutput); var msbuildPropsItems = TargetsUtility.GetMSBuildPackageImports(projectA.PropsOutput); @@ -1663,7 +1663,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageY); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var msbuildTargetsItems = TargetsUtility.GetMSBuildPackageImports(projectA.TargetsOutput); var msbuildPropsItems = TargetsUtility.GetMSBuildPackageImports(projectA.PropsOutput); @@ -1709,7 +1709,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Restore one - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var msbuildTargetsItems = TargetsUtility.GetMSBuildPackageImports(projectA.TargetsOutput); var msbuildPropsItems = TargetsUtility.GetMSBuildPackageImports(projectA.PropsOutput); @@ -1721,7 +1721,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( // Act - r = RestoreSolution(pathContext); + r = Util.RestoreSolution(pathContext); Assert.True(File.Exists(projectA.TargetsOutput), r.Item2); msbuildTargetsItems = TargetsUtility.GetMSBuildPackageImports(projectA.TargetsOutput); @@ -1778,11 +1778,11 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Restore one - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); Assert.True(File.Exists(projectA.TargetsOutput), r.Item2); // Act - r = RestoreSolution(pathContext); + r = Util.RestoreSolution(pathContext); Assert.True(r.Item1 == 0); Assert.True(File.Exists(projectA.TargetsOutput), r.Item2); } @@ -1877,7 +1877,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targets = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.ToDictionary(e => e.Name); @@ -1927,7 +1927,7 @@ public void RestoreNetCore_ProjectToProject_NETCoreToUnknown() solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetB = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.SingleOrDefault(e => e.Name == "b"); @@ -1982,7 +1982,7 @@ public void RestoreNetCore_ProjectToProject_NETCoreToUAP() solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetB = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.SingleOrDefault(e => e.Name == "b"); @@ -2057,7 +2057,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var tfm = NuGetFramework.Parse("UAP10.0"); @@ -2118,7 +2118,7 @@ public void RestoreNetCore_ProjectToProject_UAPToUnknown() solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetB = projectA.AssetsFile.Targets.Single(e => e.TargetFramework.Equals(NuGetFramework.Parse("UAP10.0"))).Libraries.SingleOrDefault(e => e.Name == "b"); @@ -2736,7 +2736,7 @@ public void RestoreNetCore_VerifyPropsAndTargetsAreWrittenWhenRestoreFails() // Act // Verify failure - var r = RestoreSolution(pathContext, exitCode: 1); + var r = Util.RestoreSolution(pathContext, expectedExitCode: 1); var targets = TargetsUtility.GetMSBuildPackageImports(projectA.TargetsOutput); @@ -2780,7 +2780,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -2825,7 +2825,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var xTarget = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.Single(); // Assert @@ -2869,7 +2869,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var xTarget = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.Single(); // Assert @@ -2913,7 +2913,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert Assert.True(File.Exists(projectA.AssetsFileOutputPath), r.Item2); @@ -2943,7 +2943,7 @@ public void RestoreNetCore_SingleProject_NonNuGet() // Act && Assert // Verify this is a noop and not a failure - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); } } @@ -2974,7 +2974,7 @@ public void RestoreNetCore_NETCore_ProjectToProject_VerifyProjectInTarget() solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetB = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.SingleOrDefault(e => e.Name == "b"); @@ -3022,7 +3022,7 @@ public void RestoreNetCore_NETCore_ProjectToProject_VerifyPackageIdUsed() solution.Create(pathContext.SolutionRoot); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetB = projectA.AssetsFile @@ -3078,7 +3078,7 @@ public void RestoreNetCore_NETCore_ProjectToProject_MissingProjectReference() File.Delete(projectB.ProjectPath); // Act && Assert - var r = RestoreSolution(pathContext, exitCode: 1); + var r = Util.RestoreSolution(pathContext, expectedExitCode: 1); } } @@ -3123,7 +3123,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetX = projectA.AssetsFile.Targets.Single(target => string.IsNullOrEmpty(target.RuntimeIdentifier)).Libraries.SingleOrDefault(e => e.Name == "x"); @@ -3185,7 +3185,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageY); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetNet = projectA.AssetsFile.Targets.Single(e => e.TargetFramework.Equals(NuGetFramework.Parse("net46")) && string.IsNullOrEmpty(e.RuntimeIdentifier)); @@ -3248,7 +3248,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var targetNet = projectA.AssetsFile.Targets.Single(e => e.TargetFramework.Equals(NuGetFramework.Parse("net46")) && string.IsNullOrEmpty(e.RuntimeIdentifier)); @@ -3313,7 +3313,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); // Assert var xLibraryInA = projectA.AssetsFile.Libraries.Single(x => x.Name == packageX.Id); @@ -3360,7 +3360,7 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var xLibrary = projectA.AssetsFile.Libraries.Single(); // Assert @@ -3403,52 +3403,12 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( packageX); // Act - var r = RestoreSolution(pathContext); + var r = Util.RestoreSolution(pathContext); var xLibrary = projectA.AssetsFile.Libraries.Single(); // Assert Assert.Equal("packagex/1.0.0-beta", xLibrary.Path); } } - - private static CommandRunnerResult RestoreSolution(SimpleTestPathContext pathContext, int exitCode = 0, params string[] additionalArgs) - { - return Restore(pathContext, pathContext.SolutionRoot, exitCode, additionalArgs); - } - - private static CommandRunnerResult Restore(SimpleTestPathContext pathContext, string inputPath, int exitCode = 0, params string[] additionalArgs) - { - var nugetexe = Util.GetNuGetExePath(); - - // Store the dg file for debugging - var dgPath = Path.Combine(pathContext.WorkingDirectory, "out.dg"); - var envVars = new Dictionary() - { - { "NUGET_PERSIST_DG", "true" }, - { "NUGET_PERSIST_DG_PATH", dgPath } - }; - - var args = new string[] { - "restore", - inputPath, - "-Verbosity", - "detailed" - }; - - args = args.Concat(additionalArgs).ToArray(); - - // Act - var r = CommandRunner.Run( - nugetexe, - pathContext.WorkingDirectory.Path, - string.Join(" ", args), - waitForExit: true, - environmentVariables: envVars); - - // Assert - Assert.True(exitCode == r.Item1, r.Item3 + "\n\n" + r.Item2); - - return r; - } } } \ No newline at end of file diff --git a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/Util.cs b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/Util.cs index c8ef93f8ec2..4936d48a031 100644 --- a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/Util.cs +++ b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/Util.cs @@ -40,6 +40,53 @@ public static string GetResource(string name) } } + /// + /// Restore a solution. + /// + public static CommandRunnerResult RestoreSolution(SimpleTestPathContext pathContext, int expectedExitCode = 0, params string[] additionalArgs) + { + return Restore(pathContext, pathContext.SolutionRoot, expectedExitCode, additionalArgs); + } + + /// + /// Run nuget.exe restore {inputPath} + /// + public static CommandRunnerResult Restore(SimpleTestPathContext pathContext, string inputPath, int expectedExitCode = 0, params string[] additionalArgs) + { + var nugetexe = Util.GetNuGetExePath(); + + // Store the dg file for debugging + var dgPath = Path.Combine(pathContext.WorkingDirectory, "out.dg"); + var envVars = new Dictionary() + { + { "NUGET_PERSIST_DG", "true" }, + { "NUGET_PERSIST_DG_PATH", dgPath }, + { "NUGET_HTTP_CACHE_PATH", pathContext.HttpCacheFolder } + }; + + var args = new string[] { + "restore", + inputPath, + "-Verbosity", + "detailed" + }; + + args = args.Concat(additionalArgs).ToArray(); + + // Act + var r = CommandRunner.Run( + nugetexe, + pathContext.WorkingDirectory.Path, + string.Join(" ", args), + waitForExit: true, + environmentVariables: envVars); + + // Assert + Assert.True(expectedExitCode == r.Item1, r.Item3 + "\n\n" + r.Item2); + + return r; + } + public static string CreateTestPackage( string packageId, string version, @@ -412,7 +459,17 @@ public static MockServer CreateMockServer(IList packages) return server; } + /// + /// Path to nuget.exe for tests. + /// public static string GetNuGetExePath() + { + return _nuGetExePath.Value; + } + + private static readonly Lazy _nuGetExePath = new Lazy(GetNuGetExePathCore); + + private static string GetNuGetExePathCore() { var targetDir = ConfigurationManager.AppSettings["TestTargetDir"] ?? Directory.GetCurrentDirectory(); var nugetexe = Path.Combine(targetDir, "NuGet.exe"); diff --git a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/xunit.runner.json b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/xunit.runner.json index 4c62fd7c26c..ba24e656967 100644 --- a/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/xunit.runner.json +++ b/test/NuGet.Clients.Tests/NuGet.CommandLine.Test/xunit.runner.json @@ -1,4 +1,5 @@ { "maxParallelThreads": 1, - "parallelizeTestCollections": false + "parallelizeTestCollections": false, + "longRunningTestSeconds": 1200 } \ No newline at end of file diff --git a/test/NuGet.Core.Tests/NuGet.Commands.Test/SourceRepositoryDependencyProviderTests.cs b/test/NuGet.Core.Tests/NuGet.Commands.Test/SourceRepositoryDependencyProviderTests.cs new file mode 100644 index 00000000000..bc7aac9a257 --- /dev/null +++ b/test/NuGet.Core.Tests/NuGet.Commands.Test/SourceRepositoryDependencyProviderTests.cs @@ -0,0 +1,163 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NuGet.Common; +using NuGet.Configuration; +using NuGet.Frameworks; +using NuGet.LibraryModel; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Test.Utility; +using NuGet.Versioning; +using Xunit; + +namespace NuGet.Commands.Test +{ + public class SourceRepositoryDependencyProviderTests + { + [Fact] + public async Task SourceRepositoryDependencyProvider_VerifyGetDependencyInfoAsyncThrowsWhenListedPackageIsMissing() + { + // Arrange + var testLogger = new TestLogger(); + var cacheContext = new SourceCacheContext(); + + var findResource = new Mock(); + findResource.Setup(s => s.GetAllVersionsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new[] { NuGetVersion.Parse("1.0.0"), NuGetVersion.Parse("2.0.0") }); + + findResource.Setup(s => s.GetDependencyInfoAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ThrowsAsync(new PackageNotFoundProtocolException(new PackageIdentity("x", NuGetVersion.Parse("1.0.0")))); + + var source = new Mock(); + source.Setup(s => s.GetResourceAsync()) + .ReturnsAsync(findResource.Object); + source.SetupGet(s => s.PackageSource) + .Returns(new PackageSource("http://test/index.json")); + + var libraryRange = new LibraryRange("x", new VersionRange(new NuGetVersion(1, 0, 0)), LibraryDependencyTarget.Package); + var provider = new SourceRepositoryDependencyProvider(source.Object, testLogger, cacheContext, ignoreFailedSources: true, ignoreWarning: true); + + // Act && Assert + // Verify the exception it thrown even with ignoreFailedSources: true + await Assert.ThrowsAsync(async () => + await provider.GetDependenciesAsync(new LibraryIdentity("x", NuGetVersion.Parse("1.0.0"), + LibraryType.Package), + NuGetFramework.Parse("net45"), + cacheContext, + testLogger, + CancellationToken.None)); + } + + [Fact] + public async Task SourceRepositoryDependencyProvider_VerifyGetDependenciesAsyncReturnsOriginalIdentity() + { + // Arrange + var testLogger = new TestLogger(); + var cacheContext = new SourceCacheContext(); + + var findResource = new Mock(); + findResource.Setup(s => s.GetAllVersionsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new[] { NuGetVersion.Parse("1.0.0-beta"), NuGetVersion.Parse("2.0.0") }); + + findResource.Setup(s => s.GetDependencyInfoAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new FindPackageByIdDependencyInfo( + new PackageIdentity("X", NuGetVersion.Parse("1.0.0-bEta")), + Enumerable.Empty(), + Enumerable.Empty())); + + var source = new Mock(); + source.Setup(s => s.GetResourceAsync()) + .ReturnsAsync(findResource.Object); + source.SetupGet(s => s.PackageSource) + .Returns(new PackageSource("http://test/index.json")); + + var libraryRange = new LibraryRange("x", new VersionRange(new NuGetVersion(1, 0, 0, "beta")), LibraryDependencyTarget.Package); + var provider = new SourceRepositoryDependencyProvider(source.Object, testLogger, cacheContext, ignoreFailedSources: true, ignoreWarning: true); + + // Act + var library = await provider.GetDependenciesAsync( + new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package), + NuGetFramework.Parse("net45"), + cacheContext, + testLogger, + CancellationToken.None); + + // Assert + Assert.Equal("X", library.Library.Name); + Assert.Equal("1.0.0-bEta", library.Library.Version.ToString()); + } + + [Fact] + public async Task SourceRepositoryDependencyProvider_VerifyValuesAreCachedAndFindResourceIsHitOnce() + { + // Arrange + var testLogger = new TestLogger(); + var cacheContext = new SourceCacheContext(); + + var versionsHitCount = 0; + + var findResource = new Mock(); + findResource.Setup(s => s.GetAllVersionsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new[] { NuGetVersion.Parse("1.0.0-beta"), NuGetVersion.Parse("2.0.0") }) + .Callback(() => versionsHitCount++); + + var dependencyHitCount = 0; + + findResource.Setup(s => s.GetDependencyInfoAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(new FindPackageByIdDependencyInfo( + new PackageIdentity("x", NuGetVersion.Parse("1.0.0-beta")), + Enumerable.Empty(), + Enumerable.Empty())) + .Callback(() => dependencyHitCount++); + + var source = new Mock(); + source.Setup(s => s.GetResourceAsync()) + .ReturnsAsync(findResource.Object); + source.SetupGet(s => s.PackageSource) + .Returns(new PackageSource("http://test/index.json")); + + var libraryRange = new LibraryRange("x", new VersionRange(new NuGetVersion(1, 0, 0, "beta")), LibraryDependencyTarget.Package); + var provider = new SourceRepositoryDependencyProvider(source.Object, testLogger, cacheContext, ignoreFailedSources: true, ignoreWarning: true); + + // Act + var library = await provider.GetDependenciesAsync( + new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package), + NuGetFramework.Parse("net45"), + cacheContext, + testLogger, + CancellationToken.None); + + library = await provider.GetDependenciesAsync( + new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package), + NuGetFramework.Parse("net45"), + cacheContext, + testLogger, + CancellationToken.None); + + var versions = await provider.FindLibraryAsync( + new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package), + NuGetFramework.Parse("net45"), + cacheContext, + testLogger, + CancellationToken.None); + + versions = await provider.FindLibraryAsync( + new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package), + NuGetFramework.Parse("net45"), + cacheContext, + testLogger, + CancellationToken.None); + + // Assert + Assert.Equal(1, versionsHitCount); + Assert.Equal(1, dependencyHitCount); + } + } +} diff --git a/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/FindPackageTests.cs b/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/FindPackageTests.cs new file mode 100644 index 00000000000..a19a5879fe6 --- /dev/null +++ b/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/FindPackageTests.cs @@ -0,0 +1,209 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using NuGet.Configuration; +using NuGet.Frameworks; +using NuGet.LibraryModel; +using NuGet.Packaging.Core; +using NuGet.Protocol; +using NuGet.Protocol.Core.Types; +using NuGet.Test.Utility; +using NuGet.Versioning; +using Xunit; + +namespace NuGet.DependencyResolver.Core.Tests +{ + public class FindPackageTests + { + [Fact] + public async Task FindPackage_VerifyFloatingPackageIsRequiredOnlyFromASingleSource() + { + // Arrange + var range = new LibraryRange("x", VersionRange.Parse("1.0.0-*"), LibraryDependencyTarget.Package); + var cacheContext = new SourceCacheContext(); + var testLogger = new TestLogger(); + var framework = NuGetFramework.Parse("net45"); + var context = new RemoteWalkContext(cacheContext, testLogger); + var token = CancellationToken.None; + var edge = new GraphEdge(null, null, null); + var actualIdentity = new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta.1"), LibraryType.Package); + var higherIdentity = new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta.2"), LibraryType.Package); + var dependencies = new[] { new LibraryDependency() { LibraryRange = new LibraryRange("y", VersionRange.All, LibraryDependencyTarget.Package) } }; + var dependencyInfo = LibraryDependencyInfo.Create(actualIdentity, framework, dependencies); + var dependencyInfo2 = LibraryDependencyInfo.Create(higherIdentity, framework, dependencies); + + var downloadCount = 0; + + // Source1 returns 1.0.0-beta.1 + var remoteProvider = new Mock(); + remoteProvider.Setup(e => e.FindLibraryAsync(range, It.IsAny(), It.IsAny(), testLogger, token)) + .ReturnsAsync(actualIdentity); + remoteProvider.SetupGet(e => e.IsHttp).Returns(true); + remoteProvider.SetupGet(e => e.Source).Returns(new PackageSource("test")); + remoteProvider.Setup(e => e.GetDependenciesAsync(It.IsAny(), It.IsAny(), It.IsAny(), testLogger, token)) + .ReturnsAsync(dependencyInfo) + .Callback(() => ++downloadCount); + context.RemoteLibraryProviders.Add(remoteProvider.Object); + + // Source2 returns 1.0.0-beta.2 + var remoteProvider2 = new Mock(); + remoteProvider2.Setup(e => e.FindLibraryAsync(range, It.IsAny(), It.IsAny(), testLogger, token)) + .ReturnsAsync(higherIdentity); + remoteProvider2.SetupGet(e => e.IsHttp).Returns(true); + remoteProvider2.SetupGet(e => e.Source).Returns(new PackageSource("test")); + remoteProvider2.Setup(e => e.GetDependenciesAsync(It.IsAny(), It.IsAny(), It.IsAny(), testLogger, token)) + .ReturnsAsync(dependencyInfo2) + .Callback(() => ++downloadCount); + context.RemoteLibraryProviders.Add(remoteProvider2.Object); + + // Act + var result = await ResolverUtility.FindLibraryEntryAsync(range, framework, edge, context, token); + + // Assert + // Verify only one download happened + Assert.Equal(1, downloadCount); + Assert.Equal("1.0.0-beta.2", result.Key.Version.ToString()); + } + + [Fact] + public async Task FindPackage_VerifyMissingListedPackageSucceedsOnRetry() + { + // Arrange + var range = new LibraryRange("x", VersionRange.Parse("1.0.0-beta"), LibraryDependencyTarget.Package); + var cacheContext = new SourceCacheContext(); + var testLogger = new TestLogger(); + var framework = NuGetFramework.Parse("net45"); + var context = new RemoteWalkContext(cacheContext, testLogger); + var token = CancellationToken.None; + var edge = new GraphEdge(null, null, null); + var actualIdentity = new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package); + var dependencies = new[] { new LibraryDependency() { LibraryRange = new LibraryRange("y", VersionRange.All, LibraryDependencyTarget.Package) } }; + var dependencyInfo = LibraryDependencyInfo.Create(actualIdentity, framework, dependencies); + + var remoteProvider = new Mock(); + remoteProvider.Setup(e => e.FindLibraryAsync(range, framework, It.IsAny(), testLogger, token)) + .ReturnsAsync(actualIdentity); + remoteProvider.SetupGet(e => e.IsHttp).Returns(true); + remoteProvider.SetupGet(e => e.Source).Returns(new PackageSource("test")); + + var hitCount = 0; + + remoteProvider.Setup(e => e.GetDependenciesAsync(actualIdentity, framework, cacheContext, testLogger, token)) + .ThrowsAsync(new PackageNotFoundProtocolException(new PackageIdentity(actualIdentity.Name, actualIdentity.Version))) + .Callback(() => ++hitCount); + + remoteProvider.Setup(e => e.GetDependenciesAsync(actualIdentity, framework, It.IsAny(), testLogger, token)) + .ReturnsAsync(dependencyInfo) + .Callback(() => ++hitCount); + + context.RemoteLibraryProviders.Add(remoteProvider.Object); + + // Act + var result = await ResolverUtility.FindLibraryEntryAsync(range, framework, edge, context, token); + + // Assert + Assert.Equal(1, hitCount); + Assert.Equal("x", result.Key.Name); + } + + [Fact] + public async Task FindPackage_VerifyMissingListedPackageThrowsNotFound() + { + // Arrange + var range = new LibraryRange("x", VersionRange.Parse("1.0.0-beta"), LibraryDependencyTarget.Package); + var cacheContext = new SourceCacheContext(); + var testLogger = new TestLogger(); + var framework = NuGetFramework.Parse("net45"); + var context = new RemoteWalkContext(cacheContext, testLogger); + var token = CancellationToken.None; + var edge = new GraphEdge(null, null, null); + var actualIdentity = new LibraryIdentity("x", NuGetVersion.Parse("1.0.0-beta"), LibraryType.Package); + var dependencies = new[] { new LibraryDependency() { LibraryRange = new LibraryRange("y", VersionRange.All, LibraryDependencyTarget.Package) } }; + var dependencyInfo = LibraryDependencyInfo.Create(actualIdentity, framework, dependencies); + + var remoteProvider = new Mock(); + remoteProvider.Setup(e => e.FindLibraryAsync(range, framework, It.IsAny(), testLogger, token)) + .ReturnsAsync(actualIdentity); + remoteProvider.SetupGet(e => e.IsHttp).Returns(true); + remoteProvider.SetupGet(e => e.Source).Returns(new PackageSource("test")); + + var hitCount = 0; + + remoteProvider.Setup(e => e.GetDependenciesAsync(actualIdentity, framework, It.IsAny(), testLogger, token)) + .ThrowsAsync(new PackageNotFoundProtocolException(new PackageIdentity(actualIdentity.Name, actualIdentity.Version))) + .Callback(() => ++hitCount); + + context.RemoteLibraryProviders.Add(remoteProvider.Object); + + // Act + await Assert.ThrowsAsync(async () => await ResolverUtility.FindLibraryEntryAsync(range, framework, edge, context, token)); + + // Assert + Assert.Equal(2, hitCount); + } + + [Fact] + public async Task FindPackage_VerifyFindLibraryEntryReturnsOriginalCase() + { + // Arrange + var range = new LibraryRange("x", VersionRange.Parse("1.0.0-beta"), LibraryDependencyTarget.Package); + var cacheContext = new SourceCacheContext(); + var testLogger = new TestLogger(); + var framework = NuGetFramework.Parse("net45"); + var context = new RemoteWalkContext(cacheContext, testLogger); + var token = CancellationToken.None; + var edge = new GraphEdge(null, null, null); + var actualIdentity = new LibraryIdentity("X", NuGetVersion.Parse("1.0.0-bEta"), LibraryType.Package); + var dependencies = new[] { new LibraryDependency() { LibraryRange = new LibraryRange("y", VersionRange.All, LibraryDependencyTarget.Package) } }; + var dependencyInfo = LibraryDependencyInfo.Create(actualIdentity, framework, dependencies); + + var remoteProvider = new Mock(); + remoteProvider.Setup(e => e.FindLibraryAsync(range, framework, cacheContext, testLogger, token)) + .ReturnsAsync(actualIdentity); + + remoteProvider.Setup(e => e.GetDependenciesAsync(actualIdentity, framework, cacheContext, testLogger, token)) + .ReturnsAsync(dependencyInfo); + + context.RemoteLibraryProviders.Add(remoteProvider.Object); + + // Act + var result = await ResolverUtility.FindLibraryEntryAsync(range, framework, edge, context, token); + + // Assert + Assert.Equal(LibraryType.Package, result.Data.Match.Library.Type); + Assert.Equal("X", result.Data.Match.Library.Name); + Assert.Equal("1.0.0-bEta", result.Data.Match.Library.Version.ToString()); + Assert.Equal("y", result.Data.Dependencies.Single().Name); + } + + [Fact] + public async Task FindPackage_VerifyMissingVersionPackageReturnsUnresolved() + { + // Arrange + var range = new LibraryRange("x", VersionRange.Parse("1.0.0-beta"), LibraryDependencyTarget.Package); + var cacheContext = new SourceCacheContext(); + var testLogger = new TestLogger(); + var framework = NuGetFramework.Parse("net45"); + var context = new RemoteWalkContext(cacheContext, testLogger); + var edge = new GraphEdge(null, null, null); + + var remoteProvider = new Mock(); + context.RemoteLibraryProviders.Add(remoteProvider.Object); + + // Act + var result = await ResolverUtility.FindLibraryEntryAsync(range, framework, edge, context, CancellationToken.None); + + // Assert + Assert.Equal(LibraryType.Unresolved, result.Data.Match.Library.Type); + Assert.Equal("x", result.Data.Match.Library.Name); + Assert.Equal("1.0.0-beta", result.Data.Match.Library.Version.ToString()); + } + } +} diff --git a/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/RemoteDependencyWalkerTests.cs b/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/RemoteDependencyWalkerTests.cs index 1df46de4e66..4502f0915ef 100644 --- a/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/RemoteDependencyWalkerTests.cs +++ b/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/RemoteDependencyWalkerTests.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using NuGet.Common; +using NuGet.Configuration; using NuGet.Frameworks; using NuGet.LibraryModel; using NuGet.Protocol.Core.Types; @@ -828,6 +829,8 @@ public bool IsHttp } } + public PackageSource Source => new PackageSource("Test"); + public Task CopyToAsync( LibraryIdentity match, Stream stream, @@ -850,7 +853,7 @@ public Task FindLibraryAsync( return Task.FromResult(packages.FindBestMatch(libraryRange.VersionRange, i => i?.Version)); } - public Task> GetDependenciesAsync( + public Task GetDependenciesAsync( LibraryIdentity match, NuGetFramework targetFramework, SourceCacheContext cacheContext, @@ -860,9 +863,10 @@ public Task> GetDependenciesAsync( List dependencies; if (_graph.TryGetValue(match, out dependencies)) { - return Task.FromResult>(dependencies); + return Task.FromResult(LibraryDependencyInfo.Create(match, targetFramework, dependencies)); } - return Task.FromResult(Enumerable.Empty()); + + return Task.FromResult(LibraryDependencyInfo.Create(match, targetFramework, Enumerable.Empty())); } public TestPackage Package(string id, string version) diff --git a/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/ResolverFacts.cs b/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/ResolverFacts.cs index 868e128d96a..92ca3c7ecf1 100644 --- a/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/ResolverFacts.cs +++ b/test/NuGet.Core.Tests/NuGet.DependencyResolver.Core.Tests/ResolverFacts.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using NuGet.Common; +using NuGet.Configuration; using NuGet.DependencyResolver.Tests; using NuGet.Frameworks; using NuGet.LibraryModel; @@ -116,6 +117,8 @@ public void AddLibrary(LibraryIdentity identity) public bool IsHttp => true; + public PackageSource Source => new PackageSource("Test"); + public Task CopyToAsync( LibraryIdentity match, Stream stream, @@ -141,14 +144,14 @@ public async Task FindLibraryAsync( return _libraries.FindBestMatch(libraryRange.VersionRange, l => l?.Version); } - public Task> GetDependenciesAsync( + public Task GetDependenciesAsync( LibraryIdentity match, NuGetFramework targetFramework, SourceCacheContext cacheContext, ILogger logger, CancellationToken cancellationToken) { - return Task.FromResult(Enumerable.Empty()); + return Task.FromResult(LibraryDependencyInfo.Create(match, targetFramework, Enumerable.Empty())); } } } diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/HttpFileSystemBasedFindPackageByIdResourceTests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/HttpFileSystemBasedFindPackageByIdResourceTests.cs index cc6ef36c307..b80d85f2ab6 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/HttpFileSystemBasedFindPackageByIdResourceTests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/HttpFileSystemBasedFindPackageByIdResourceTests.cs @@ -61,7 +61,7 @@ public async Task HttpFileSystemBasedFindPackageById_GetOriginalIdentity() var resource = await repo.GetResourceAsync(); // Act - var identity = await resource.GetOriginalIdentityAsync( + var info = await resource.GetDependencyInfoAsync( "DEEPEQUAL", new NuGetVersion("1.4.0.1-RC"), cacheContext, @@ -70,8 +70,8 @@ public async Task HttpFileSystemBasedFindPackageById_GetOriginalIdentity() // Assert Assert.IsType(resource); - Assert.Equal("DeepEqual", identity.Id); - Assert.Equal("1.4.0.1-rc", identity.Version.ToNormalizedString()); + Assert.Equal("DeepEqual", info.PackageIdentity.Id); + Assert.Equal("1.4.0.1-rc", info.PackageIdentity.Version.ToNormalizedString()); } } } diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/NuGetv3LocalRepositoryTests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/NuGetv3LocalRepositoryTests.cs index fdad8973013..db9efd1ca38 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/NuGetv3LocalRepositoryTests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/NuGetv3LocalRepositoryTests.cs @@ -257,5 +257,34 @@ await SimpleTestPackageUtility.CreateFolderFeedV3( Assert.Equal("2.0.0-BETA", package.Version.ToNormalizedString()); } } + + [Fact] + public async Task NuGetv3LocalRepository_FindPackage_VerifyNuspecsCached() + { + // Arrange + using (var workingDir = TestDirectory.Create()) + { + var id = "Foo"; + var target = new NuGetv3LocalRepository(workingDir); + await SimpleTestPackageUtility.CreateFolderFeedV3( + workingDir, + PackageSaveMode.Defaultv3, + new SimpleTestPackageContext(id, "1.0.0"), + new SimpleTestPackageContext(id, "2.0.0-Beta")); + + // Act + var package1 = target.FindPackage(id, NuGetVersion.Parse("2.0.0-beta")); + var package2 = target.FindPackage(id, NuGetVersion.Parse("2.0.0-BETA")); + var package3 = target.FindPackage(id, NuGetVersion.Parse("2.0.0-beta")); + + // Assert + Assert.True(ReferenceEquals(package1, package3)); + Assert.True(ReferenceEquals(package1.Nuspec, package2.Nuspec)); + Assert.True(ReferenceEquals(package1.Nuspec, package3.Nuspec)); + + // These should contain different versions + Assert.False(ReferenceEquals(package1, package2)); + } + } } } diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV2FindPackageByIdResourceTests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV2FindPackageByIdResourceTests.cs index aaf441e2797..cad96ee9f9b 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV2FindPackageByIdResourceTests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV2FindPackageByIdResourceTests.cs @@ -93,7 +93,7 @@ public async Task RemoteV2FindPackageById_GetOriginalIdentity_IdInResponse() var resource = await repo.GetResourceAsync(); // Act - var identity = await resource.GetOriginalIdentityAsync( + var info = await resource.GetDependencyInfoAsync( "XUNIT", new NuGetVersion("2.2.0-BETA1-build3239"), cacheContext, @@ -102,8 +102,8 @@ public async Task RemoteV2FindPackageById_GetOriginalIdentity_IdInResponse() // Assert Assert.IsType(resource); - Assert.Equal("xunit", identity.Id); - Assert.Equal("2.2.0-beta1-build3239", identity.Version.ToNormalizedString()); + Assert.Equal("xunit", info.PackageIdentity.Id); + Assert.Equal("2.2.0-beta1-build3239", info.PackageIdentity.Version.ToNormalizedString()); } } } @@ -151,7 +151,7 @@ public async Task RemoteV2FindPackageById_GetOriginalIdentity_IdNotInResponse() var resource = await repo.GetResourceAsync(); // Act - var identity = await resource.GetOriginalIdentityAsync( + var info = await resource.GetDependencyInfoAsync( "WINDOWSAZURE.STORAGE", new NuGetVersion("6.2.2-PREVIEW"), cacheContext, @@ -160,8 +160,8 @@ public async Task RemoteV2FindPackageById_GetOriginalIdentity_IdNotInResponse() // Assert Assert.IsType(resource); - Assert.Equal("WindowsAzure.Storage", identity.Id); - Assert.Equal("6.2.2-preview", identity.Version.ToNormalizedString()); + Assert.Equal("WindowsAzure.Storage", info.PackageIdentity.Id); + Assert.Equal("6.2.2-preview", info.PackageIdentity.Version.ToNormalizedString()); } } } diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV3FindPackageByIdResourceTests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV3FindPackageByIdResourceTests.cs index 4ee862f0d67..edea228cfb7 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV3FindPackageByIdResourceTests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/RemoteV3FindPackageByIdResourceTests.cs @@ -1,7 +1,11 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using NuGet.Protocol.Core.Types; @@ -18,18 +22,45 @@ public class RemoteV3FindPackageByIdResourceTests public async Task RemoteV3FindPackageById_GetOriginalIdentity_IdInResponse() { // Arrange - var responses = new Dictionary(); - responses.Add("http://testsource.com/v3/index.json", JsonData.IndexWithoutFlatContainer); - responses.Add("https://api.nuget.org/v3/registration0/deepequal/index.json", JsonData.DeepEqualRegistationIndex); - - var repo = StaticHttpHandler.CreateSource("http://testsource.com/v3/index.json", Repository.Provider.GetCoreV3(), responses); - var logger = new TestLogger(); - using (var cacheContext = new SourceCacheContext()) + using (var workingDir = TestDirectory.Create()) { + var source = "http://testsource.com/v3/index.json"; + var package = SimpleTestPackageUtility.CreateFullPackage(workingDir, "DeepEqual", "1.4.0.1-rc"); + var packageBytes = File.ReadAllBytes(package.FullName); + + var responses = new Dictionary>> + { + { + source, + _ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new TestContent(JsonData.IndexWithoutFlatContainer) + }) + }, + { + "https://api.nuget.org/v3/registration0/deepequal/index.json", + _ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new TestContent(JsonData.DeepEqualRegistationIndex) + }) + }, + { + "https://api.nuget.org/packages/deepequal.1.4.0.1-rc.nupkg", + _ => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(packageBytes) + }) + } + }; + + var repo = StaticHttpHandler.CreateSource(source, Repository.Provider.GetCoreV3(), responses); + + var logger = new TestLogger(); + // Act var resource = await repo.GetResourceAsync(); - var identity = await resource.GetOriginalIdentityAsync( + var info = await resource.GetDependencyInfoAsync( "DEEPEQUAL", new NuGetVersion("1.4.0.1-RC"), cacheContext, @@ -38,8 +69,8 @@ public async Task RemoteV3FindPackageById_GetOriginalIdentity_IdInResponse() // Assert Assert.IsType(resource); - Assert.Equal("DeepEqual", identity.Id); - Assert.Equal("1.4.0.1-rc", identity.Version.ToNormalizedString()); + Assert.Equal("DeepEqual", info.PackageIdentity.Id); + Assert.Equal("1.4.0.1-rc", info.PackageIdentity.Version.ToNormalizedString()); } } } diff --git a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/Utility/FindPackagesByIdNupkgDownloaderTests.cs b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/Utility/FindPackagesByIdNupkgDownloaderTests.cs index 6f53d6bf497..1e79adb8d9c 100644 --- a/test/NuGet.Core.Tests/NuGet.Protocol.Tests/Utility/FindPackagesByIdNupkgDownloaderTests.cs +++ b/test/NuGet.Core.Tests/NuGet.Protocol.Tests/Utility/FindPackagesByIdNupkgDownloaderTests.cs @@ -68,7 +68,7 @@ public async Task GetNuspecReaderFromNupkgAsync_ThrowsWhenNupkgIsNotFound( tc.StatusCode = statusCode; // Act & Assert - var exception = await Assert.ThrowsAsync( + var exception = await Assert.ThrowsAsync( () => tc.Target.GetNuspecReaderFromNupkgAsync( tc.Identity, tc.NupkgUrl, diff --git a/test/TestUtilities/Test.Utility/SimpleTestSetup/SimpleTestPathContext.cs b/test/TestUtilities/Test.Utility/SimpleTestSetup/SimpleTestPathContext.cs index ea477295af8..18728e91e5c 100644 --- a/test/TestUtilities/Test.Utility/SimpleTestSetup/SimpleTestPathContext.cs +++ b/test/TestUtilities/Test.Utility/SimpleTestSetup/SimpleTestPathContext.cs @@ -2,11 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Threading.Tasks; using System.Xml.Linq; namespace NuGet.Test.Utility @@ -30,6 +26,8 @@ public class SimpleTestPathContext : IDisposable public string FallbackFolder { get; } + public string HttpCacheFolder { get; } + public bool CleanUp { get; set; } = true; public SimpleTestPathContext() @@ -42,6 +40,7 @@ public SimpleTestPathContext() NuGetConfig = Path.Combine(WorkingDirectory, "NuGet.Config"); PackageSource = Path.Combine(WorkingDirectory.Path, "source"); FallbackFolder = Path.Combine(WorkingDirectory.Path, "fallback"); + HttpCacheFolder = Path.Combine(WorkingDirectory.Path, "v3-cache"); Directory.CreateDirectory(SolutionRoot); Directory.CreateDirectory(UserPackagesFolder);