Implement multithreaded driver analyze command w/deterministic output

scripts/Projects.psm1

@@ -10,13 +10,13 @@
 $Frameworks = @{}
 
 # .NET Framework versions for which we build.
-$Frameworks.NetFx = @("net461")
+$Frameworks.NetFx = @("net472")
 
 # Frameworks for which we build libraries.
-$Frameworks.Library = @("netstandard2.0") + $Frameworks.NetFx
+$Frameworks.Library = @("netstandard2.1") + $Frameworks.NetFx
 
 # Frameworks for which we build applications.
-$Frameworks.Application = @("netcoreapp2.1") + $Frameworks.NetFx
+$Frameworks.Application = @("netcoreapp3.1") + $Frameworks.NetFx
 
 $Frameworks.All = ($Frameworks.Library + $Frameworks.Application | Select -Unique)
 

scripts/Run-Tests.ps1

@@ -38,7 +38,7 @@ $failedTestProjects = @()
 foreach ($project in $Projects.Tests) {
     foreach ($framework in $Frameworks.Application) {
 
-        if ((-not $AppVeyor) -and ($framework -ne "netcoreapp2.1")) { continue; }
+        if ($framework -ne "netcoreapp3.1") { continue; }
 
         Write-Information "Running tests in ${project}: $framework..."
         Push-Location $SourceRoot\$project

src/Nuget/Sarif.Multitool.nuspec

@@ -21,38 +21,28 @@
   </metadata>
   <files>
     <!-- The subfolder layout for the different TargetFrameworks is intentionally inconsistent -->
-    <!-- net461 and netcoreapp2.1 pack to tools\TargetFramework\* for backcompat -->
-    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net461\Sarif.Multitool.exe.config"
-          target="tools\net461"
+    <!-- net472 and netcoreapp2.1 pack to tools\TargetFramework\* for backcompat -->
+    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net472\Sarif.Multitool.exe.config"
+          target="tools\net472"
           />
-    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net461\**"
-          target="tools\net461"
-          exclude="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net461\Sarif*;bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net461\WorkItems.dll;"
+    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net472\**"
+          target="tools\net472"
+          exclude="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net472\Sarif*;bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\net472\WorkItems.dll;"
           />
-    <file src="bld\bin\Signing\net461\**"
-          target="tools\net461"
-          />
-    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp2.1\**"
-          target="tools\netcoreapp2.1"
-          exclude="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp2.1\Sarif*.dll;bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp2.1\WorkItems.dll"
-          />
-    <file src="bld\bin\Signing\netcoreapp2.1\**"
-          target="tools\netcoreapp2.1"
-          />
-    <file src="bld\bin\Signing\netstandard2.0\**"
-          target="tools\netcoreapp2.1"
+    <file src="bld\bin\Signing\net472\**"
+          target="tools\net472"
           />
 
-    <!-- netcoreapp2.1 packs to tools\TargetFramework\any\* for compatibility with dotnet tool install -->
-    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp2.1\**"
-          target="tools\netcoreapp2.1\any"
-          exclude="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp2.1\Sarif*.dll;bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp2.1\WorkItems.dll"
+    <!-- netcoreapp3.1 packs to tools\TargetFramework\any\* for compatibility with dotnet tool install -->
+    <file src="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp3.1\**"
+          target="tools\netcoreapp3.1\any"
+          exclude="bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp3.1\Sarif*.dll;bld\bin\$platform$_$configuration$\Sarif.Multitool\Publish\netcoreapp3.1\WorkItems.dll"
           />
-    <file src="bld\bin\Signing\netcoreapp2.1\**"
-          target="tools\netcoreapp2.1\any"
+    <file src="bld\bin\Signing\netcoreapp3.1\**"
+          target="tools\netcoreapp3.1\any"
           />
-    <file src="bld\bin\Signing\netstandard2.0\**"
-          target="tools\netcoreapp2.1\any"
+    <file src="bld\bin\Signing\netstandard2.1\**"
+          target="tools\netcoreapp3.1\any"
           />
 
     <file src="triskelion.png" />

src/Sarif.Converters/Sarif.Converters.csproj

@@ -8,8 +8,8 @@
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), build.props))\build.props" />
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0</TargetFrameworks>
-    <TargetFrameworks Condition="$(OS) == 'Windows_NT'">$(TargetFrameworks);net461</TargetFrameworks>
+    <TargetFrameworks>netstandard2.1</TargetFrameworks>
+    <TargetFrameworks Condition="$(OS) == 'Windows_NT'">$(TargetFrameworks);net472</TargetFrameworks>
   </PropertyGroup>
 
   <ItemGroup>
@@ -39,7 +39,8 @@
 
   <ItemGroup>
     <PackageReference Include="CsvHelper" Version="15.0.5" />
-    <PackageReference Include="System.Collections.Immutable" Version="1.5.0" />
+    <PackageReference Include="System.Collections.Immutable" Version="5.0.0" />
+    <PackageReference Include="System.IO.Compression" Version="4.3.0" />
   </ItemGroup>
 
   <ItemGroup>

src/Sarif.Driver/Sarif.Driver.csproj

@@ -8,8 +8,8 @@
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), build.props))\build.props" />
 
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0</TargetFrameworks>
-    <TargetFrameworks Condition="$(OS) == 'Windows_NT'">$(TargetFrameworks);net461</TargetFrameworks>
+    <TargetFrameworks>netstandard2.1</TargetFrameworks>
+    <TargetFrameworks Condition="$(OS) == 'Windows_NT'">$(TargetFrameworks);net472</TargetFrameworks>
   </PropertyGroup>
 
   <ItemGroup>
@@ -34,7 +34,9 @@
 
   <ItemGroup>
     <PackageReference Include="CommandLineParser" Version="2.7.82" />
-    <PackageReference Include="System.Composition" Version="1.2.0" />
+    <PackageReference Include="Microsoft.Extensions.FileSystemGlobbing" Version="3.1.9" />
+    <PackageReference Include="System.Composition" Version="5.0.0" />
+    <PackageReference Include="System.Threading.Channels" Version="5.0.0" />
   </ItemGroup>
 
 </Project>

src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs

@@ -28,7 +28,7 @@ public abstract class AnalyzeCommandBase<TContext, TOptions> : PlugInDriverComma
 
         private Tool _tool;
         private TContext _rootContext;
-        private ResultsCachingLogger _resultsCachingLogger;
+        private CacheByFileHashLogger _cacheByFileHashLogger;
         private IDictionary<string, HashData> _pathToHashDataMap;
 
         public Exception ExecutionException { get; set; }
@@ -55,15 +55,15 @@ public string DefaultConfigurationPath
             }
         }
 
-        public override int Run(TOptions analyzeOptions)
+        public override int Run(TOptions options)
         {
             // 0. Initialize an common logger that drives all outputs. This
             //    object drives logging for console, statistics, etc.
-            using (AggregatingLogger logger = InitializeLogger(analyzeOptions))
+            using (AggregatingLogger logger = InitializeLogger(options))
             {
                 try
                 {
-                    Analyze(analyzeOptions, logger);
+                    Analyze(options, logger);
                 }
                 catch (ExitApplicationException<ExitReason> ex)
                 {
@@ -86,30 +86,30 @@ public override int Run(TOptions analyzeOptions)
 
             bool succeeded = (RuntimeErrors & ~RuntimeConditions.Nonfatal) == RuntimeConditions.None;
 
-            if (analyzeOptions.RichReturnCode)
+            if (options.RichReturnCode)
             {
                 return (int)RuntimeErrors;
             }
 
             return succeeded ? SUCCESS : FAILURE;
         }
 
-        private void Analyze(TOptions analyzeOptions, AggregatingLogger logger)
+        private void Analyze(TOptions options, AggregatingLogger logger)
         {
             // 0. Log analysis initiation
             logger.AnalysisStarted();
 
             // 1. Create context object to pass to skimmers. The logger
             //    and configuration objects are common to all context
             //    instances and will be passed on again for analysis.
-            _rootContext = CreateContext(analyzeOptions, logger, RuntimeErrors);
+            _rootContext = CreateContext(options, logger, RuntimeErrors);
 
             // 2. Perform any command line argument validation beyond what
             //    the command line parser library is capable of.
-            ValidateOptions(_rootContext, analyzeOptions);
+            ValidateOptions(_rootContext, options);
 
             // 3. Produce a comprehensive set of analysis targets 
-            ISet<string> targets = CreateTargetsSet(analyzeOptions);
+            ISet<string> targets = CreateTargetsSet(options);
 
             // 4. Proactively validate that we can locate and 
             //    access all analysis targets. Helper will return
@@ -118,23 +118,23 @@ private void Analyze(TOptions analyzeOptions, AggregatingLogger logger)
             targets = ValidateTargetsExist(_rootContext, targets);
 
             // 5. Initialize report file, if configured.
-            InitializeOutputFile(analyzeOptions, _rootContext, targets);
+            InitializeOutputFile(options, _rootContext, targets);
 
             // 6. Instantiate skimmers.
             ISet<Skimmer<TContext>> skimmers = CreateSkimmers(_rootContext);
 
             // 7. Initialize configuration. This step must be done after initializing
             //    the skimmers, as rules define their specific context objects and
             //    so those assemblies must be loaded.
-            InitializeConfiguration(analyzeOptions, _rootContext);
+            InitializeConfiguration(options, _rootContext);
 
             // 8. Initialize skimmers. Initialize occurs a single time only. This
             //    step needs to occurs after initializing configuration in order
             //    to allow command-line override of rule settings
             skimmers = InitializeSkimmers(skimmers, _rootContext);
 
             // 9. Run all analysis
-            AnalyzeTargets(analyzeOptions, skimmers, _rootContext, targets);
+            AnalyzeTargets(options, skimmers, _rootContext, targets);
 
             // 10. For test purposes, raise an unhandled exception if indicated
             if (RaiseUnhandledExceptionInDriverCode)
@@ -250,15 +250,10 @@ internal AggregatingLogger InitializeLogger(AnalyzeOptionsBase analyzeOptions)
                 logger.Loggers.Add(_consoleLogger);
             }
 
-            if (analyzeOptions.ComputeFileHashes)
+            if ((analyzeOptions.DataToInsert.ToFlags() & OptionallyEmittedData.Hashes) != 0)
             {
-                _resultsCachingLogger = new ResultsCachingLogger(analyzeOptions.Verbose);
-                logger.Loggers.Add(_resultsCachingLogger);
-            }
-
-            if (analyzeOptions.Statistics)
-            {
-                logger.Loggers.Add(new StatisticsLogger());
+                _cacheByFileHashLogger = new CacheByFileHashLogger(analyzeOptions.Verbose);
+                logger.Loggers.Add(_cacheByFileHashLogger);
             }
 
             return logger;
@@ -300,19 +295,21 @@ protected virtual TContext CreateContext(
             TOptions options,
             IAnalysisLogger logger,
             RuntimeConditions runtimeErrors,
+            PropertiesDictionary policy = null,
             string filePath = null)
         {
             var context = new TContext
             {
                 Logger = logger,
-                RuntimeErrors = runtimeErrors
+                RuntimeErrors = runtimeErrors,
+                Policy = policy
             };
 
             if (filePath != null)
             {
                 context.TargetUri = new Uri(filePath);
 
-                if (options.ComputeFileHashes)
+                if ((options.DataToInsert.ToFlags() & OptionallyEmittedData.Hashes) != 0)
                 {
                     if (_pathToHashDataMap != null && _pathToHashDataMap.TryGetValue(filePath, out HashData hashData))
                     {
@@ -356,7 +353,7 @@ internal string GetConfigurationFileName(TOptions options, bool unitTestFileExis
 
         protected virtual void InitializeConfiguration(TOptions options, TContext context)
         {
-            context.Policy = new PropertiesDictionary();
+            context.Policy ??= new PropertiesDictionary();
 
             string configurationFileName = GetConfigurationFileName(options);
             if (string.IsNullOrEmpty(configurationFileName))
@@ -401,10 +398,6 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context, ISe
                         OptionallyEmittedData dataToInsert = analyzeOptions.DataToInsert.ToFlags();
                         OptionallyEmittedData dataToRemove = analyzeOptions.DataToRemove.ToFlags();
 
-                        // This code is required in order to support the obsolete ComputeFileHashes argument
-                        // on the analyze command-line.
-                        if (analyzeOptions.ComputeFileHashes) { dataToInsert |= OptionallyEmittedData.Hashes; }
-
                         SarifLogger sarifLogger;
 
                         if (analyzeOptions.SarifOutputVersion != SarifVersion.OneZeroZero)
@@ -576,7 +569,7 @@ protected virtual void AnalyzeTargets(
                 ThrowExitApplicationException(rootContext, ExitReason.NoRulesLoaded);
             }
 
-            if (options.ComputeFileHashes)
+            if ((options.DataToInsert.ToFlags() & OptionallyEmittedData.Hashes) != 0)
             {
                 // If analysis is persisted to a disk log file, we will have already
                 // computed all file hashes and stored them to _pathToHashDataMap.
@@ -599,13 +592,17 @@ protected virtual TContext DetermineApplicabilityAndAnalyze(
             string target,
             ISet<string> disabledSkimmers)
         {
-            TContext context = CreateContext(options, rootContext.Logger, rootContext.RuntimeErrors, target);
-            context.Policy = rootContext.Policy;
+            TContext context = CreateContext(
+                options,
+                rootContext.Logger,
+                rootContext.RuntimeErrors,
+                rootContext.Policy,
+                target);
 
-            if (options.ComputeFileHashes)
+            if ((options.DataToInsert.ToFlags() & OptionallyEmittedData.Hashes) != 0)
             {
-                _resultsCachingLogger.HashToResultsMap.TryGetValue(context.Hashes.Sha256, out List<Tuple<ReportingDescriptor, Result>> cachedResultTuples);
-                _resultsCachingLogger.HashToNotificationsMap.TryGetValue(context.Hashes.Sha256, out List<Notification> cachedNotifications);
+                _cacheByFileHashLogger.HashToResultsMap.TryGetValue(context.Hashes.Sha256, out List<Tuple<ReportingDescriptor, Result>> cachedResultTuples);
+                _cacheByFileHashLogger.HashToNotificationsMap.TryGetValue(context.Hashes.Sha256, out List<Notification> cachedNotifications);
 
                 bool replayCachedData = (cachedResultTuples != null || cachedNotifications != null);
 
@@ -727,7 +724,8 @@ protected virtual void AnalyzeTarget(IEnumerable<Skimmer<TContext>> skimmers, TC
                 }
                 catch (Exception ex)
                 {
-                    RuntimeErrors |= Errors.LogUnhandledRuleExceptionAnalyzingTarget(disabledSkimmers, context, ex);
+                    Errors.LogUnhandledRuleExceptionAnalyzingTarget(disabledSkimmers, context, ex);
+                    RuntimeErrors |= context.RuntimeErrors;
                 }
             }
         }
@@ -794,8 +792,8 @@ protected virtual ISet<Skimmer<TContext>> InitializeSkimmers(ISet<Skimmer<TConte
         {
             SortedSet<Skimmer<TContext>> disabledSkimmers = new SortedSet<Skimmer<TContext>>(SkimmerIdComparer<TContext>.Instance);
 
-            // ONE-TIME initialization of skimmers. Do not call 
-            // Initialize more than once per skimmer instantiation
+            // ONE-TIME initialization of skimmers. Do not
+            // call more than once per skimmer instantiation.
             foreach (Skimmer<TContext> skimmer in skimmers)
             {
                 try