[ReleasePrep][2020.08.07] RI of dev into master

src/GalleryTools/Commands/VerifyGitHubVulnerabilitiesCommand.cs

@@ -0,0 +1,246 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Data.Entity;
+using System.Data.SqlClient;
+using System.Linq;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+using GitHubVulnerabilities2Db.Collector;
+using GitHubVulnerabilities2Db.Configuration;
+using GitHubVulnerabilities2Db.GraphQL;
+using GitHubVulnerabilities2Db.Ingest;
+using Microsoft.Extensions.CommandLineUtils;
+using Microsoft.Extensions.Logging.Abstractions;
+using NuGet.Services.Entities;
+using NuGet.Versioning;
+using NuGetGallery;
+
+namespace GalleryTools.Commands
+{
+    /// <summary>
+    /// This command verifies that the <see cref="PackageVulnerability"/> and <see cref="VulnerablePackageVersionRange"/> entities in the 
+    /// database match the <see cref="SecurityAdvisory"/> and <see cref="SecurityVulnerability"/> entities in GitHub's V4 GraphQL API.
+    /// </summary>
+    /// <remarks>
+    /// The verification only expects that advisories that are present in the GitHub API have the same metadata and contain the same ranges in the DB.
+    /// It intentionally does not require that all vulnerabilities in the DB come from GitHub, or that the set of ranges in the DB match the set of ranges in the GitHub API.
+    /// This is so that we can add some additional vulnerabilities or ranges for testing or administrative purposes.
+    /// </remarks>
+    public static class VerifyGitHubVulnerabilitiesCommand
+    {
+        public static void Configure(CommandLineApplication config)
+        {
+            config.Description = "Verify that the gallery database's vulnerability information matches GitHub's feed.";
+            config.HelpOption("-? | -h | --help");
+
+            var gitHubPersonalAccessTokenOption = config.Option(
+                "--token | -t",
+                "The personal access token to use to authenticate with GitHub.",
+                CommandOptionType.SingleValue);
+
+            var connectionStringOption = config.Option(
+                "--connectionstring | -c",
+                "The SQL connectionstring of the target NuGetGallery database.",
+                CommandOptionType.SingleValue);
+
+            config.OnExecute(async () => await ExecuteAsync(
+                connectionStringOption,
+                gitHubPersonalAccessTokenOption));
+        }
+
+        private static async Task<int> ExecuteAsync(
+            CommandOption connectionStringOption,
+            CommandOption gitHubPersonalAccessTokenOption)
+        {
+            if (!connectionStringOption.HasValue())
+            {
+                Console.Error.WriteLine($"The {connectionStringOption.Template} option is required.");
+                return 1;
+            }
+
+            if (!gitHubPersonalAccessTokenOption.HasValue())
+            {
+                Console.Error.WriteLine($"The {connectionStringOption.Template} option is required.");
+                return 1;
+            }
+
+            try
+            {
+                var advisories = await FetchAdvisories(gitHubPersonalAccessTokenOption.Value());
+                await VerifyPackageVulnerabilities(
+                    connectionStringOption.Value(),
+                    advisories);
+
+                Console.WriteLine("DONE");
+                return 0;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine(" FAILED");
+                Console.Error.WriteLine(e.Message);
+                return 1;
+            }
+        }
+
+        private static async Task<IReadOnlyList<SecurityAdvisory>> FetchAdvisories(
+            string token)
+        {
+            Console.Write("Fetching vulnerabilities from GitHub...");
+
+            var config = new GitHubVulnerabilities2DbConfiguration
+            {
+                GitHubPersonalAccessToken = token
+            };
+
+            var queryService = new QueryService(
+                config,
+                new HttpClient());
+
+            var advisoryQueryService = new AdvisoryQueryService(
+                queryService,
+                new AdvisoryQueryBuilder(),
+                NullLogger<AdvisoryQueryService>.Instance);
+
+            var advisories = await advisoryQueryService.GetAdvisoriesSinceAsync(DateTimeOffset.MinValue, CancellationToken.None);
+            Console.WriteLine($" FOUND {advisories.Count} advisories.");
+            return advisories;
+        }
+
+        private static async Task VerifyPackageVulnerabilities(
+            string connectionString,
+            IReadOnlyList<SecurityAdvisory> advisories)
+        {
+            Console.WriteLine("Fetching vulnerabilities from DB...");
+
+            using (var sqlConnection = new SqlConnection(connectionString))
+            {
+                await sqlConnection.OpenAsync();
+                using (var entitiesContext = new EntitiesContext(sqlConnection, readOnly: false))
+                {
+                    var verifier = new PackageVulnerabilityServiceVerifier(entitiesContext);
+                    var ingestor = new AdvisoryIngestor(verifier, new GitHubVersionRangeParser());
+                    await ingestor.IngestAsync(advisories);
+
+                    if (verifier.HasErrors)
+                    {
+                        throw new Exception("DB does not match GitHub API!");
+                    }
+
+                    Console.WriteLine("DB matches GitHub API!");
+                }
+            }
+        }
+
+        public class PackageVulnerabilityServiceVerifier : IPackageVulnerabilityService
+        {
+            private readonly IEntitiesContext _entitiesContext;
+
+            public PackageVulnerabilityServiceVerifier(
+                IEntitiesContext entitiesContext)
+            {
+                _entitiesContext = entitiesContext ?? throw new ArgumentNullException(nameof(entitiesContext));
+            }
+
+            public void ApplyExistingVulnerabilitiesToPackage(Package package)
+            {
+                throw new NotImplementedException();
+            }
+
+            public Task UpdateVulnerabilityAsync(PackageVulnerability vulnerability, bool withdrawn)
+            {
+                Console.WriteLine($"Verifying vulnerability {vulnerability.GitHubDatabaseKey}.");
+                var existingVulnerability = _entitiesContext.Vulnerabilities
+                    .Include(v => v.AffectedRanges)
+                    .SingleOrDefault(v => v.GitHubDatabaseKey == vulnerability.GitHubDatabaseKey);
+
+                if (withdrawn || !vulnerability.AffectedRanges.Any())
+                {
+                    if (existingVulnerability != null)
+                    {
+                        Console.Error.WriteLine(withdrawn ? 
+                            $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey} was withdrawn and should not be in DB!" :
+                            $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey} affects no packages and should not be in DB!");
+                        HasErrors = true;
+                    }
+
+                    return Task.CompletedTask;
+                }
+
+                if (existingVulnerability == null)
+                {
+                    Console.Error.WriteLine($"Cannot find vulnerability {vulnerability.GitHubDatabaseKey} in DB!");
+                    HasErrors = true;
+                    return Task.CompletedTask;
+                }
+
+                if (existingVulnerability.Severity != vulnerability.Severity)
+                {
+                    Console.Error.WriteLine(
+                        $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey
+                        }, severity does not match! GitHub: {vulnerability.Severity}, DB: {existingVulnerability.Severity}");
+                    HasErrors = true;
+                }
+
+                if (existingVulnerability.AdvisoryUrl != vulnerability.AdvisoryUrl)
+                {
+                    Console.Error.WriteLine(
+                        $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey
+                        }, advisory URL does not match! GitHub: {vulnerability.AdvisoryUrl}, DB: { existingVulnerability.AdvisoryUrl}");
+                    HasErrors = true;
+                }
+
+                foreach (var range in vulnerability.AffectedRanges)
+                {
+                    Console.WriteLine($"Verifying range affecting {range.PackageId} {range.PackageVersionRange}.");
+                    var existingRange = existingVulnerability.AffectedRanges
+                        .SingleOrDefault(r => r.PackageId == range.PackageId && r.PackageVersionRange == range.PackageVersionRange);
+
+                    if (existingRange == null)
+                    {
+                        Console.Error.WriteLine(
+                            $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey
+                            }, cannot find range {range.PackageId} {range.PackageVersionRange} in DB!");
+                        HasErrors = true;
+                        continue;
+                    }
+
+                    if (existingRange.FirstPatchedPackageVersion != range.FirstPatchedPackageVersion)
+                    {
+                        Console.Error.WriteLine(
+                            $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey
+                            }, range {range.PackageId} {range.PackageVersionRange}, first patched version does not match! GitHub: {
+                            range.FirstPatchedPackageVersion}, DB: {range.FirstPatchedPackageVersion}");
+                        HasErrors = true;
+                    }
+
+                    var packages = _entitiesContext.Packages
+                        .Where(p => p.PackageRegistration.Id == range.PackageId)
+                        .Include(p => p.Vulnerabilities)
+                        .ToList();
+
+                    var versionRange = VersionRange.Parse(range.PackageVersionRange);
+                    foreach (var package in packages)
+                    {
+                        var version = NuGetVersion.Parse(package.NormalizedVersion);
+                        if (versionRange.Satisfies(version) != package.Vulnerabilities.Contains(existingRange))
+                        {
+                            Console.Error.WriteLine(
+                                $@"Vulnerability advisory {vulnerability.GitHubDatabaseKey
+                                }, range {range.PackageId} {range.PackageVersionRange}, package {package.NormalizedVersion
+                                } is not properly marked vulnerable to vulnerability!");
+                            HasErrors = true;
+                        }
+                    }
+                }
+
+                return Task.CompletedTask;
+            }
+
+            public bool HasErrors { get; private set; }
+        }
+    }
+}

src/GalleryTools/GalleryTools.csproj

@@ -52,6 +52,7 @@
     <Compile Include="Commands\ReflowCommand.cs" />
     <Compile Include="Commands\UpdateIsLatestCommand.cs" />
     <Compile Include="Commands\VerifyApiKeyCommand.cs" />
+    <Compile Include="Commands\VerifyGitHubVulnerabilitiesCommand.cs" />
     <Compile Include="Program.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\AssemblyInfo.*.cs" />
@@ -61,6 +62,10 @@
     <None Include="App.config" />
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\GitHubVulnerabilities2Db\GitHubVulnerabilities2Db.csproj">
+      <Project>{26bb718a-e1c1-4e70-9008-fb8ee7a7f7e5}</Project>
+      <Name>GitHubVulnerabilities2Db</Name>
+    </ProjectReference>
     <ProjectReference Include="..\NuGet.Services.Entities\NuGet.Services.Entities.csproj">
       <Project>{6262f4fc-29be-4226-b676-db391c89d396}</Project>
       <Name>NuGet.Services.Entities</Name>

src/GalleryTools/Program.cs

@@ -21,6 +21,7 @@ public static int Main(params string[] args)
             commandLineApplication.Command("filldevdeps", BackfillDevelopmentDependencyCommand.Configure);
             commandLineApplication.Command("verifyapikey", VerifyApiKeyCommand.Configure);
             commandLineApplication.Command("updateIsLatest", UpdateIsLatestCommand.Configure);
+            commandLineApplication.Command("verifyVulnerabilities", VerifyGitHubVulnerabilitiesCommand.Configure);
 
             try
             {

src/GitHubVulnerabilities2Db/Collector/AdvisoryCollector.cs

@@ -32,7 +32,9 @@ public AdvisoryCollector(
 
         public async Task<bool> ProcessAsync(CancellationToken token)
         {
-            var advisories = await _queryService.GetAdvisoriesSinceAsync(_cursor, token);
+            await _cursor.Load(token);
+            var lastUpdated = _cursor.Value;
+            var advisories = await _queryService.GetAdvisoriesSinceAsync(lastUpdated, token);
             var hasAdvisories = advisories != null && advisories.Any();
             _logger.LogInformation("Found {AdvisoryCount} new advisories to process", advisories?.Count() ?? 0);
             if (hasAdvisories)

src/GitHubVulnerabilities2Db/Collector/AdvisoryQueryService.cs

@@ -8,7 +8,6 @@
 using System.Threading.Tasks;
 using GitHubVulnerabilities2Db.GraphQL;
 using Microsoft.Extensions.Logging;
-using NuGet.Services.Cursor;
 
 namespace GitHubVulnerabilities2Db.Collector
 {
@@ -28,10 +27,8 @@ public AdvisoryQueryService(
             _logger = logger ?? throw new ArgumentNullException(nameof(logger));
         }
 
-        public async Task<IReadOnlyList<SecurityAdvisory>> GetAdvisoriesSinceAsync(ReadCursor<DateTimeOffset> cursor, CancellationToken token)
+        public async Task<IReadOnlyList<SecurityAdvisory>> GetAdvisoriesSinceAsync(DateTimeOffset lastUpdated, CancellationToken token)
         {
-            await cursor.Load(token);
-            var lastUpdated = cursor.Value;
             _logger.LogInformation("Fetching advisories updated since {LastUpdated}", lastUpdated);
             var firstQuery = _queryBuilder.CreateSecurityAdvisoriesQuery(lastUpdated);
             var firstResponse = await _queryService.QueryAsync(firstQuery, token);

src/GitHubVulnerabilities2Db/Collector/IAdvisoryQueryService.cs

@@ -15,6 +15,6 @@ namespace GitHubVulnerabilities2Db.Collector
     /// </summary>
     public interface IAdvisoryQueryService
     {
-        Task<IReadOnlyList<SecurityAdvisory>> GetAdvisoriesSinceAsync(ReadCursor<DateTimeOffset> cursor, CancellationToken token);
+        Task<IReadOnlyList<SecurityAdvisory>> GetAdvisoriesSinceAsync(DateTimeOffset lastUpdated, CancellationToken token);
     }
 }

src/GitHubVulnerabilities2Db/Job.cs

@@ -5,12 +5,14 @@
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Web;
 using Autofac;
 using GitHubVulnerabilities2Db.Collector;
 using GitHubVulnerabilities2Db.Configuration;
 using GitHubVulnerabilities2Db.Gallery;
 using GitHubVulnerabilities2Db.GraphQL;
 using GitHubVulnerabilities2Db.Ingest;
+using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
@@ -21,6 +23,8 @@
 using NuGet.Services.Storage;
 using NuGetGallery;
 using NuGetGallery.Auditing;
+using NuGetGallery.Configuration;
+using NuGetGallery.Diagnostics;
 using NuGetGallery.Security;
 
 namespace GitHubVulnerabilities2Db
@@ -109,6 +113,19 @@ protected void ConfigureGalleryServices(ContainerBuilder containerBuilder)
             containerBuilder
                 .RegisterType<PackageUpdateService>()
                 .As<IPackageUpdateService>();
+
+            containerBuilder.RegisterType<AppConfiguration>()
+                .As<IAppConfiguration>()
+                .SingleInstance();
+
+            var contentService = new FakeContentService();
+            containerBuilder.RegisterInstance(contentService)
+                .As<IContentService>()
+                .SingleInstance();
+
+            containerBuilder.RegisterType<ContentObjectService>()
+                .As<IContentObjectService>()
+                .SingleInstance();
         }
 
         protected void ConfigureQueryServices(ContainerBuilder containerBuilder)
@@ -165,4 +182,18 @@ private DurableCursor CreateCursor(IComponentContext ctx, Func<GitHubVulnerabili
             return new DurableCursor(storage.ResolveUri(getBlobName(config)), storage, DateTimeOffset.MinValue);
         }
     }
+
+    public class FakeContentService : IContentService
+    {
+        public void ClearCache()
+        {
+            //no-op
+        }
+
+        public Task<IHtmlString> GetContentItemAsync(string name, TimeSpan expiresIn)
+        {
+            // no-op
+            return Task.FromResult((IHtmlString)null);
+        }
+    }
 }