[ReleasePrep][2021.05.23]RI dev into main

src/GitHubVulnerabilities2Db/Gallery/ThrowingTelemetryService.cs

@@ -66,7 +66,7 @@ public void TrackDownloadCountDecreasedDuringRefresh(string packageId, string pa
             throw new NotImplementedException();
         }
 
-        public void TrackDownloadJsonRefreshDuration(long milliseconds)
+        public void TrackDownloadJsonRefreshDuration(TimeSpan duration)
         {
             throw new NotImplementedException();
         }
@@ -370,5 +370,10 @@ public void TrackVerifyPackageKeyEvent(string packageId, string packageVersion,
         {
             throw new NotImplementedException();
         }
+
+        public void TrackVulnerabilitiesCacheRefreshDuration(TimeSpan duration)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

src/NuGetGallery.Services/PackageManagement/IPackageVulnerabilitiesManagementService.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System.Threading.Tasks;
+using System.Linq;
 using NuGet.Services.Entities;
 
 namespace NuGetGallery

src/NuGetGallery.Services/Telemetry/ITelemetryService.cs

@@ -16,7 +16,7 @@ public interface ITelemetryService
 
         void TrackGetPackageRegistrationDownloadCountFailed(string packageId);
 
-        void TrackDownloadJsonRefreshDuration(long milliseconds);
+        void TrackDownloadJsonRefreshDuration(TimeSpan duration);
 
         void TrackDownloadCountDecreasedDuringRefresh(string packageId, string packageVersion, long oldCount, long newCount);
 
@@ -404,5 +404,11 @@ void TrackABTestEvaluated(
             bool isAuthenticated,
             int testBucket,
             int testPercentage);
+
+        /// <summary>
+        /// Track how long it takes to populate the vulnerabilities cache
+        /// </summary>
+        /// <param name="duration">Refresh duration for vulnerabilities cache</param>
+        void TrackVulnerabilitiesCacheRefreshDuration(TimeSpan duration);
     }
 }

src/NuGetGallery.Services/Telemetry/TelemetryService.cs

@@ -91,6 +91,7 @@ public class Events
             public const string ABTestEvaluated = "ABTestEvaluated";
             public const string PackagePushDisconnect = "PackagePushDisconnect";
             public const string SymbolPackagePushDisconnect = "SymbolPackagePushDisconnect";
+            public const string VulnerabilitiesCacheRefreshDurationMs = "VulnerabilitiesCacheRefreshDurationMs";
         }
 
         private readonly IDiagnosticsSource _diagnosticsSource;
@@ -260,9 +261,9 @@ public void TrackGetPackageRegistrationDownloadCountFailed(string packageId)
             });
         }
 
-        public void TrackDownloadJsonRefreshDuration(long milliseconds)
+        public void TrackDownloadJsonRefreshDuration(TimeSpan duration)
         {
-            TrackMetric(Events.DownloadJsonRefreshDuration, milliseconds, properties => { });
+            TrackMetric(Events.DownloadJsonRefreshDuration, duration.TotalMilliseconds, properties => { });
         }
 
         public void TrackDownloadCountDecreasedDuringRefresh(string packageId, string packageVersion, long oldCount, long newCount)
@@ -1103,6 +1104,11 @@ public void TrackSymbolPackagePushDisconnectEvent()
             TrackMetric(Events.SymbolPackagePushDisconnect, 1, p => { });
         }
 
+        public void TrackVulnerabilitiesCacheRefreshDuration(TimeSpan duration)
+        {
+            TrackMetric(Events.VulnerabilitiesCacheRefreshDurationMs, duration.TotalMilliseconds, properties => { });
+        }
+
         /// <summary>
         /// We use <see cref="ITelemetryClient.TrackMetric(string, double, IDictionary{string, string})"/> instead of
         /// <see cref="ITelemetryClient.TrackEvent(string, IDictionary{string, string}, IDictionary{string, double})"/>

src/NuGetGallery/App_Start/AppActivator.cs

@@ -15,7 +15,7 @@
 using System.Web.Routing;
 using System.Web.UI;
 using Elmah;
-using Microsoft.WindowsAzure.ServiceRuntime;
+using Microsoft.Extensions.DependencyInjection;
 using NuGetGallery;
 using NuGetGallery.Configuration;
 using NuGetGallery.Diagnostics;
@@ -270,10 +270,17 @@ private static void BackgroundJobsPostStart(IAppConfiguration configuration)
                 {
                     // Perform initial refresh + schedule new refreshes every 15 minutes
                     HostingEnvironment.QueueBackgroundWorkItem(_ => cloudDownloadCountService.RefreshAsync());
-                    jobs.Add(new CloudDownloadCountServiceRefreshJob(TimeSpan.FromMinutes(15), cloudDownloadCountService));
+                    jobs.Add(new CloudDownloadCountServiceRefreshJob(TimeSpan.FromMinutes(15),
+                        cloudDownloadCountService));
                 }
             }
 
+            // Perform initial refresh for vulnerabilities cache + schedule new refreshes every 30 minutes
+            var packageVulnerabilitiesCacheService = DependencyResolver.Current.GetService<IPackageVulnerabilitiesCacheService>();
+            var serviceScopeFactory = DependencyResolver.Current.GetService<IServiceScopeFactory>();
+            HostingEnvironment.QueueBackgroundWorkItem(_ => packageVulnerabilitiesCacheService.RefreshCache(serviceScopeFactory));
+            jobs.Add(new PackageVulnerabilitiesCacheRefreshJob(TimeSpan.FromMinutes(30), packageVulnerabilitiesCacheService, serviceScopeFactory));
+
             if (jobs.AnySafe())
             {
                 var jobCoordinator = new NuGetJobCoordinator();

src/NuGetGallery/App_Start/DefaultDependenciesModule.cs

@@ -460,6 +460,11 @@ protected override void Load(ContainerBuilder builder)
                 .As<IPackageVulnerabilitiesManagementService>()
                 .InstancePerLifetimeScope();
 
+            builder.RegisterType<PackageVulnerabilitiesCacheService>()
+                .AsSelf()
+                .As<IPackageVulnerabilitiesCacheService>()
+                .SingleInstance();
+
             services.AddHttpClient();
             services.AddScoped<IGravatarProxyService, GravatarProxyService>();
 

src/NuGetGallery/Infrastructure/Jobs/PackageVulnerabilitiesCacheRefreshJob.cs

@@ -0,0 +1,30 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.Extensions.DependencyInjection;
+using WebBackgrounder;
+
+namespace NuGetGallery
+{
+    public class PackageVulnerabilitiesCacheRefreshJob : Job
+    {
+        private readonly IPackageVulnerabilitiesCacheService _packageVulnerabilitiesCacheService;
+        private IServiceScopeFactory _serviceScopeFactory;
+
+        public PackageVulnerabilitiesCacheRefreshJob(TimeSpan interval, 
+            IPackageVulnerabilitiesCacheService packageVulnerabilitiesCacheService,
+            IServiceScopeFactory serviceScopeFactory)
+            : base("", interval)
+        {
+            _packageVulnerabilitiesCacheService = packageVulnerabilitiesCacheService;
+            _serviceScopeFactory = serviceScopeFactory;
+        }
+
+        public override Task Execute()
+        {
+            return new Task(() => _packageVulnerabilitiesCacheService.RefreshCache(_serviceScopeFactory));
+        }
+    }
+}

src/NuGetGallery/NuGetGallery.csproj

@@ -227,6 +227,7 @@
     <Compile Include="Infrastructure\ABTestEnrollmentState.cs" />
     <Compile Include="Infrastructure\ABTestEnrollmentFactory.cs" />
     <Compile Include="Infrastructure\CookieBasedABTestService.cs" />
+    <Compile Include="Infrastructure\Jobs\PackageVulnerabilitiesCacheRefreshJob.cs" />
     <Compile Include="Infrastructure\RequestValidationExceptionFilter.cs" />
     <Compile Include="Infrastructure\HttpStatusCodeWithHeadersResult.cs" />
     <Compile Include="Infrastructure\IABTestEnrollmentFactory.cs" />
@@ -314,8 +315,10 @@
     <Compile Include="Services\IImageDomainValidator.cs" />
     <Compile Include="Services\IPackageVulnerabilitiesService.cs" />
     <Compile Include="Services\IPackageMetadataValidationService.cs" />
+    <Compile Include="Services\IPackageVulnerabilitiesCacheService.cs" />
     <Compile Include="Services\MarkdownService.cs" />
     <Compile Include="Services\OwnerlessNamespaceIdConflictMessage.cs" />
+    <Compile Include="Services\PackageVulnerabilitiesCacheService.cs" />
     <Compile Include="Services\PackageVulnerabilitiesService.cs" />
     <Compile Include="Services\PackageMetadataValidationService.cs" />
     <Compile Include="Services\ConfigurationIconFileProvider.cs" />

src/NuGetGallery/Services/CloudDownloadCountService.cs

@@ -106,7 +106,7 @@ public async Task RefreshAsync()
                     var stopwatch = Stopwatch.StartNew();
                     await RefreshCoreAsync();
                     stopwatch.Stop();
-                    _telemetryService.TrackDownloadJsonRefreshDuration(stopwatch.ElapsedMilliseconds);
+                    _telemetryService.TrackDownloadJsonRefreshDuration(TimeSpan.FromMilliseconds(stopwatch.ElapsedMilliseconds));
 
                 }
                 catch (WebException ex)

src/NuGetGallery/Services/IPackageVulnerabilitiesCacheService.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using Microsoft.Extensions.DependencyInjection;
+using NuGet.Services.Entities;
+
+namespace NuGetGallery
+{
+    /// <summary>
+    /// This interface is used to implement a basic caching for vulnerabilities querying.
+    /// /// </summary>
+    public interface IPackageVulnerabilitiesCacheService
+    {
+        /// <summary>
+        /// This function is used to get the packages by id dictionary from the cache
+        /// </summary>
+        IReadOnlyDictionary<int, IReadOnlyList<PackageVulnerability>> GetVulnerabilitiesById(string id);
+
+        /// <summary>
+        /// This function will refresh the cache from the database, to be called at regular intervals
+        /// </summary>
+        /// <param name="serviceScopeFactory">The factory which will provide a new service scope for each refresh</param>
+        void RefreshCache(IServiceScopeFactory serviceScopeFactory);
+    }
+}

src/NuGetGallery/Services/PackageVulnerabilitiesCacheService.cs

@@ -0,0 +1,107 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Data.Entity;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.Extensions.DependencyInjection;
+using NuGet.Services.Entities;
+
+namespace NuGetGallery
+{
+    public class PackageVulnerabilitiesCacheService : IPackageVulnerabilitiesCacheService
+    {
+        private IDictionary<string,
+            Dictionary<int, IReadOnlyList<PackageVulnerability>>> _vulnerabilitiesByIdCache
+            = new Dictionary<string, Dictionary<int, IReadOnlyList<PackageVulnerability>>>();
+        private readonly object _refreshLock = new object();
+        private bool _isRefreshing;
+
+        private readonly ITelemetryService _telemetryService;
+
+        public PackageVulnerabilitiesCacheService(ITelemetryService telemetryService)
+        {
+            _telemetryService = telemetryService ?? throw new ArgumentNullException(nameof(telemetryService));
+        }
+
+        public IReadOnlyDictionary<int, IReadOnlyList<PackageVulnerability>> GetVulnerabilitiesById(string id)
+        {
+            if (string.IsNullOrEmpty(id))
+            {
+                throw new ArgumentException("Must have a value.", nameof(id));
+            }
+
+            if (_vulnerabilitiesByIdCache.TryGetValue(id, out var result))
+            {
+                return result;
+            }
+
+            return null;
+        }
+
+        public void RefreshCache(IServiceScopeFactory serviceScopeFactory)
+        {
+            if (serviceScopeFactory == null)
+            {
+                throw new ArgumentNullException(nameof(serviceScopeFactory));
+            }
+
+            bool shouldRefresh = false;
+            lock (_refreshLock)
+            {
+                if (!_isRefreshing)
+                {
+                    _isRefreshing = true;
+                    shouldRefresh = true;
+                }
+            }
+
+            if (shouldRefresh)
+            {
+                try
+                {
+                    var stopwatch = Stopwatch.StartNew();
+
+                    // Create a unique service scope for each refresh to ensure a fresh entities context
+                    using (var serviceScope = serviceScopeFactory.CreateScope())
+                    {
+                        var serviceProvider = serviceScope.ServiceProvider;
+                        var entitiesContext = serviceProvider.GetService<IEntitiesContext>();
+
+                        // We need to build a dictionary of dictionaries. Breaking it down:
+                        // - this give us a list of all vulnerable package version ranges
+                        _vulnerabilitiesByIdCache = entitiesContext.Set<VulnerablePackageVersionRange>()
+                            .Include(x => x.Vulnerability)
+                            // - from these we want a list in this format: (<id>, (<package key>, <vulnerability>))
+                            //   which will allow us to look up the dictionary by id, and return a dictionary of version -> vulnerability
+                            .SelectMany(x => x.Packages.Select(p => new
+                            { PackageId = x.PackageId ?? string.Empty, KeyVulnerability = new { PackageKey = p.Key, x.Vulnerability } }))
+                            .GroupBy(ikv => ikv.PackageId, ikv => ikv.KeyVulnerability)
+                            // - build the outer dictionary, keyed by <id> - each inner dictionary is paired with a time of creation (for cache invalidation)
+                            .ToDictionary(ikv => ikv.Key,
+                                ikv =>
+                                    ikv.GroupBy(kv => kv.PackageKey, kv => kv.Vulnerability)
+                                        // - build the inner dictionaries, all under the same <id>, each keyed by <package key>
+                                        .ToDictionary(kv => kv.Key,
+                                            kv => kv.ToList().AsReadOnly() as IReadOnlyList<PackageVulnerability>));
+                    }
+
+                    stopwatch.Stop();
+
+                    _telemetryService.TrackVulnerabilitiesCacheRefreshDuration(TimeSpan.FromMilliseconds(stopwatch.ElapsedMilliseconds));
+                }
+                catch (Exception ex)
+                {
+                    _telemetryService.TraceException(ex);
+                }
+                finally
+                {
+                    _isRefreshing = false;
+                }
+            }
+        }
+    }
+}

src/NuGetGallery/Services/PackageVulnerabilitiesService.cs

@@ -12,36 +12,17 @@ namespace NuGetGallery
 {
     public class PackageVulnerabilitiesService : IPackageVulnerabilitiesService
     {
-        private readonly IEntitiesContext _entitiesContext;
+        private readonly IPackageVulnerabilitiesCacheService _packageVulnerabilitiesCacheService;
 
-        public PackageVulnerabilitiesService(IEntitiesContext entitiesContext)
+        public PackageVulnerabilitiesService(IPackageVulnerabilitiesCacheService packageVulnerabilitiesCacheService)
         {
-            _entitiesContext = entitiesContext ?? throw new ArgumentNullException(nameof(entitiesContext));
+            _packageVulnerabilitiesCacheService = packageVulnerabilitiesCacheService ??
+                                                  throw new ArgumentNullException(
+                                                      nameof(packageVulnerabilitiesCacheService));
         }
 
-        public IReadOnlyDictionary<int, IReadOnlyList<PackageVulnerability>> GetVulnerabilitiesById(string id)
-        {
-            var result = new Dictionary<int, List<PackageVulnerability>>();
-            var packagesMatchingId = _entitiesContext.Packages
-                .Where(p => p.PackageRegistration != null && p.PackageRegistration.Id == id)
-                .Include($"{nameof(Package.VulnerablePackageRanges)}.{nameof(VulnerablePackageVersionRange.Vulnerability)}");
-            foreach (var package in packagesMatchingId)
-            {
-                if (package.VulnerablePackageRanges == null)
-                {
-                    continue;
-                }
-
-                if (package.VulnerablePackageRanges.Any())
-                {
-                    result.Add(package.Key,
-                        package.VulnerablePackageRanges.Select(vr => vr.Vulnerability).ToList());
-                }
-            }
-
-            return !result.Any() ? null :
-                result.ToDictionary(kv => kv.Key, kv => kv.Value as IReadOnlyList<PackageVulnerability>);
-        }
+        public IReadOnlyDictionary<int, IReadOnlyList<PackageVulnerability>> GetVulnerabilitiesById(string id) =>
+            _packageVulnerabilitiesCacheService.GetVulnerabilitiesById(id);
 
         public bool IsPackageVulnerable(Package package)
         {

src/VerifyMicrosoftPackage/Fakes/FakeTelemetryService.cs

@@ -63,7 +63,7 @@ public void TrackDownloadCountDecreasedDuringRefresh(string packageId, string pa
             throw new NotImplementedException();
         }
 
-        public void TrackDownloadJsonRefreshDuration(long milliseconds)
+        public void TrackDownloadJsonRefreshDuration(TimeSpan duration)
         {
             throw new NotImplementedException();
         }
@@ -372,5 +372,10 @@ public void TrackVerifyPackageKeyEvent(string packageId, string packageVersion,
         {
             throw new NotImplementedException();
         }
+
+        public void TrackVulnerabilitiesCacheRefreshDuration(TimeSpan duration)
+        {
+            throw new NotImplementedException();
+        }
     }
 }

tests/NuGetGallery.Facts/NuGetGallery.Facts.csproj

@@ -103,6 +103,7 @@
     <Compile Include="Infrastructure\SearchServiceFactoryFacts.cs" />
     <Compile Include="Services\ImageDomainValidatorFacts.cs" />
     <Compile Include="Services\MarkdownServiceFacts.cs" />
+    <Compile Include="Services\PackageVulnerabilitiesCacheServiceFacts.cs" />
     <Compile Include="Services\PackageVulnerabilitiesServiceFacts.cs" />
     <Compile Include="Services\PackageMetadataValidationServiceFacts.cs" />
     <Compile Include="Services\ConfigurationIconFileProviderFacts.cs" />