diff --git a/src/ReleaseHistory.md b/src/ReleaseHistory.md index 5c6ba6545..0a1aa4e3c 100644 --- a/src/ReleaseHistory.md +++ b/src/ReleaseHistory.md @@ -2,25 +2,34 @@ ## **v3.2.0** (UNRELEASED) +* BREAKING: Correct casing of `LogMissingreportingConfiguration` helper to `LogMissingReportingConfiguration`. [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* BREAKING: Change type of `MaxFileSizeInKilobytes` from int to long in `IAnalysisContext` and other classes. [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) * BREAKING: For `Guid` properties defined in SARIF spec, updated Json schema to use `uuid`, and updated C# object model to use `Guid?` instead of `string`. [#2555](https://github.com/microsoft/sarif-sdk/pull/2555) -* FEATURE: Provide mechanism to populate `SarifLogger` with a `FileRegionsCache` instance. -* BUGFIX: Another attempt to resolve 'InvalidOperationException' with message `Collection was modified; enumeration operation may not execute` in `MultithreadedAnalyzeCommandBase`, raised when analyzing with the `--hashes` switch. [#2459](https://github.com/microsoft/sarif-sdk/pull/2549). There was a previous attempt to fix this in [#2447](https://github.com/microsoft/sarif-sdk/pull/2447). -* FEATURE: Allow initialization of file regions cache in `InsertOptionalDataVisitor` (previously initialized exclusively from `FileRegionsCache.Instance`). -* BUGFIX: Resolve issue where `match-results-forward` command fails to generate VersionControlDetails data. [#2487](https://github.com/microsoft/sarif-sdk/pull/2487) -* BUGFIX: Remove duplicated rule definitions when executing `match-results-forward` commands for results with sub-rule ids. [#2486](https://github.com/microsoft/sarif-sdk/pull/2486) -* BUGFIX: Update `merge` command to properly produce runs by tool and version when passed the `--merge-runs` argument. [#2488](https://github.com/microsoft/sarif-sdk/pull/2488) -* BUGFIX: Eliminate `IOException` and `DirectoryNotFoundException` exceptions thrown by `merge` command when splitting by rule (due to invalid file characters in rule ids). [#2513](https://github.com/microsoft/sarif-sdk/pull/2513) -* BUGFIX: Fix classes inside NotYetAutoGenerated folder missing `virtual` keyword for public methods and properties, by regenerate and manually sync the changes. [#2537](https://github.com/microsoft/sarif-sdk/pull/2537) -* FEATURE: Allow external set of `MaxFileSizeInKilobytes`, which will allow SDK users to change the value. (Default value is 1024) [#2578](https://github.com/microsoft/sarif-sdk/pull/2578) -* FEATURE: Add a Github validation rule `GH1007`, which requires flattened result message so GHAS code scanning can ingest the log. [#2580](https://github.com/microsoft/sarif-sdk/issues/2580) -* BUGFIX: MSBuild Converter now accepts case insensitive keywords and supports PackageValidator msbuild log output. [#2579](https://github.com/microsoft/sarif-sdk/pull/2579) +* BREAKING: Mark `AnalyzeCommandBase` as obsolete. This type will be removed in the next significant update. [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* BREAKING: `LogUnhandledEngineException` no longer has a return value (and updates the `RuntimeErrors` context property directly as other helpers do). [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* BUGFIX : Resolve hangs due to unhandled exceptions during multithreaded analysis file enumeration phase. [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* BUGFIX : Another attempt to resolve 'InvalidOperationException' with message `Collection was modified; enumeration operation may not execute` in `MultithreadedAnalyzeCommandBase`, raised when analyzing with the `--hashes` switch. [#2459](https://github.com/microsoft/sarif-sdk/pull/2549). There was a previous attempt to fix this in [#2447](https://github.com/microsoft/sarif-sdk/pull/2447). +* BUGFIX : Resolve issue where `match-results-forward` command fails to generate VersionControlDetails data. [#2487](https://github.com/microsoft/sarif-sdk/pull/2487) +* BUGFIX : Remove duplicated rule definitions when executing `match-results-forward` commands for results with sub-rule ids. [#2486](https://github.com/microsoft/sarif-sdk/pull/2486) +* BUGFIX : Update `merge` command to properly produce runs by tool and version when passed the `--merge-runs` argument. [#2488](https://github.com/microsoft/sarif-sdk/pull/2488) +* BUGFIX : Eliminate `IOException` and `DirectoryNotFoundException` exceptions thrown by `merge` command when splitting by rule (due to invalid file characters in rule ids). [#2513](https://github.com/microsoft/sarif-sdk/pull/2513) +* BUGFIX : Fix classes inside NotYetAutoGenerated folder missing `virtual` keyword for public methods and properties, by regenerate and manually sync the changes. [#2537](https://github.com/microsoft/sarif-sdk/pull/2537) +* BUGFIX : MSBuild Converter now accepts case insensitive keywords and supports PackageValidator msbuild log output. [#2579](https://github.com/microsoft/sarif-sdk/pull/2579) * BUGFIX: Eliminate `NullReferenceException` when file hashing fails (due to file locked or other errors reading the file). [#2596](https://github.com/microsoft/sarif-sdk/pull/2596) +* FEATURE : Provide `PluginDriver` property (`AdditionalOptionsProvider`) that allows additional options to be exported (typically for command-line arguments). [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* FEATURE : Provide `LogFileSkippedDueToSize` that fires a warning notification if any file is skipped due to exceeding size threshold. [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* FEATURE : Provide overridable `ShouldEnqueue` predicate method to filter files from driver processing. [#2559](https://github.com/microsoft/sarif-sdk/pull/2599) +* FEATURE : Allow external set of `MaxFileSizeInKilobytes`, which will allow SDK users to change the value. (Default value is 1024) [#2578](https://github.com/microsoft/sarif-sdk/pull/2578) +* FEATURE : Add a Github validation rule `GH1007`, which requires flattened result message so GHAS code scanning can ingest the log. [#2580](https://github.com/microsoft/sarif-sdk/issues/2580) +* FEATURE : Provide mechanism to populate `SarifLogger` with a `FileRegionsCache` instance. +* FEATURE : Allow initialization of file regions cache in `InsertOptionalDataVisitor` (previously initialized exclusively from `FileRegionsCache.Instance`). ## **v3.1.0** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/3.1.0) | [Driver](https://www.nuget.org/packages/Sarif.Driver/3.1.0) | [Converters](https://www.nuget.org/packages/Sarif.Converters/3.1.0) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/3.1.0) | [Multitool Library](https://www.nuget.org/packages/Sarif.Multitool.Library/3.1.0) * BUGFIX: Loosen `System.Collections.Immutable` minimum version requirement to 1.5.0. [#2504](https://github.com/microsoft/sarif-sdk/pull/2533) ## **v3.0.0** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/3.0.0) | [Driver](https://www.nuget.org/packages/Sarif.Driver/3.0.0) | [Converters](https://www.nuget.org/packages/Sarif.Converters/3.0.0) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/3.0.0) | [Multitool Library](https://www.nuget.org/packages/Sarif.Multitool.Library/3.0.0) + * BUGFIX: Loosen Newtonsoft.JSON minimum version requirement to 6.0.8 (for .NET framework) or 9.0.1 (for all other compilations) for Sarif.Sdk. Sarif.Converts requires 8.0.1, minimally, for .NET framework compilations. * BUGFIX: Broaden set of supported .NET frameworks for compatibility reasons. Sarif.Sdk, Sarif.Driver and Sarif.WorkItems requires net461. diff --git a/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs index a5182286b..697d50a8c 100644 --- a/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs @@ -12,6 +12,7 @@ namespace Microsoft.CodeAnalysis.Sarif.Driver { + [Obsolete("AnalyzeCommandBase will be deprecated entirely soon. Use MultithreadedAnalyzeCommandBase instead.")] public abstract class AnalyzeCommandBase : PluginDriverCommand where TContext : IAnalysisContext, new() where TOptions : AnalyzeOptionsBase @@ -97,12 +98,13 @@ public override int Run(TOptions options) catch (Exception ex) { // These exceptions escaped our net and must be logged here - RuntimeErrors |= Errors.LogUnhandledEngineException(_rootContext, ex); + Errors.LogUnhandledEngineException(_rootContext, ex); ExecutionException = ex; return FAILURE; } finally { + RuntimeErrors |= _rootContext.RuntimeErrors; logger.AnalysisStopped(RuntimeErrors); } } @@ -207,7 +209,7 @@ protected virtual void ValidateOptions(TContext context, TOptions analyzeOptions succeeded &= ValidateFile(context, analyzeOptions.OutputFilePath, DefaultPolicyName, shouldExist: null); succeeded &= ValidateInvocationPropertiesToLog(context, analyzeOptions.InvocationPropertiesToLog); succeeded &= analyzeOptions.ValidateOutputOptions(context); - succeeded &= analyzeOptions.MaxFileSizeInKilobytes > 0; + succeeded &= context.MaxFileSizeInKilobytes >= 0; if (!succeeded) { @@ -217,6 +219,18 @@ protected virtual void ValidateOptions(TContext context, TOptions analyzeOptions } } + protected virtual bool ShouldEnqueue(string file, TContext context) + { + bool shouldEnqueue = IsTargetWithinFileSizeLimit(file, context.MaxFileSizeInKilobytes, out long fileSizeInKb); + + if (!shouldEnqueue) + { + Warnings.LogFileSkippedDueToSize(context, file, fileSizeInKb); + } + + return shouldEnqueue; + } + internal AggregatingLogger InitializeLogger(AnalyzeOptionsBase analyzeOptions) { _tool = Tool.CreateFromAssemblyData(); @@ -257,7 +271,7 @@ private ISet CreateTargetsSet(TOptions analyzeOptions) foreach (string file in fileSpecifier.Files) { // Only include files that are below the max size limit. - if (IsTargetWithinFileSizeLimit(file, _rootContext.MaxFileSizeInKilobytes)) + if (ShouldEnqueue(file, _rootContext)) { targets.Add(file); } @@ -288,10 +302,13 @@ protected virtual TContext CreateContext( { Logger = logger, RuntimeErrors = runtimeErrors, - Policy = policy + Policy = policy ?? new PropertiesDictionary() }; - context.MaxFileSizeInKilobytes = options.MaxFileSizeInKilobytes; + context.MaxFileSizeInKilobytes = + options.MaxFileSizeInKilobytes >= 0 + ? options.MaxFileSizeInKilobytes + : AnalyzeContextBase.MaxFileSizeInKilobytesDefaultValue; if (filePath != null) { diff --git a/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs b/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs index 2862b4172..272f834ae 100644 --- a/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs +++ b/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs @@ -11,6 +11,17 @@ namespace Microsoft.CodeAnalysis.Sarif.Driver [Verb("analyze", HelpText = "Analyze one or more binary files for security and correctness issues.")] public abstract class AnalyzeOptionsBase : CommonOptionsBase { + public AnalyzeOptionsBase() + { + // TODO: these defaults need to be converted to the configuration + // property pattern as followed by MaxFileSizeInKilobytes. + Traces = new string[] { }; + Kind = new List { ResultKind.Fail }; + Level = new List { FailureLevel.Warning, FailureLevel.Error }; + + MaxFileSizeInKilobytes = AnalyzeContextBase.MaxFileSizeInKilobytesDefaultValue; + } + [Value(0, HelpText = "One or more specifiers to a file, directory, or filter pattern that resolves to one or more binaries to analyze.")] public IEnumerable TargetFileSpecifiers { get; set; } @@ -118,7 +129,7 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase [Option( "max-file-size-in-kb", HelpText = "The maximum file size (in kilobytes) that will be analyzed.", - Default = 1024)] - public int MaxFileSizeInKilobytes { get; set; } = 1024; + Default = -1)] + public long MaxFileSizeInKilobytes { get; set; } } } diff --git a/src/Sarif.Driver/Sdk/ExportConfigurationCommandBase.cs b/src/Sarif.Driver/Sdk/ExportConfigurationCommandBase.cs index e41a90366..dc6dded75 100644 --- a/src/Sarif.Driver/Sdk/ExportConfigurationCommandBase.cs +++ b/src/Sarif.Driver/Sdk/ExportConfigurationCommandBase.cs @@ -16,11 +16,17 @@ public override int Run(ExportConfigurationOptions exportOptions) try { - PropertiesDictionary allOptions = new PropertiesDictionary(); + var allOptions = new PropertiesDictionary(); // The export command could be updated in the future to accept an arbitrary set // of analyzers for which to build an options XML file suitable for configuring them. - ImmutableArray providers = CompositionUtilities.GetExports(RetrievePluginAssemblies(DefaultPluginAssemblies, exportOptions.PluginFilePaths)); + var providers = new List(CompositionUtilities.GetExports(RetrievePluginAssemblies(DefaultPluginAssemblies, exportOptions.PluginFilePaths))); + + if (AdditionalOptionsProvider != null) + { + providers.Add(AdditionalOptionsProvider); + } + foreach (IOptionsProvider provider in providers) { IOption sampleOption = null; diff --git a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs index db75cd351..902122b9d 100644 --- a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs @@ -48,7 +48,7 @@ public abstract class MultithreadedAnalyzeCommandBase : Plug public static bool RaiseUnhandledExceptionInDriverCode { get; set; } - public virtual FileFormat ConfigurationFormat { get { return FileFormat.Json; } } + public virtual FileFormat ConfigurationFormat => FileFormat.Json; protected MultithreadedAnalyzeCommandBase(IFileSystem fileSystem = null) { @@ -89,13 +89,14 @@ public override int Run(TOptions options) if (!(ex is ExitApplicationException)) { // These exceptions escaped our net and must be logged here - RuntimeErrors |= Errors.LogUnhandledEngineException(_rootContext, ex); + Errors.LogUnhandledEngineException(_rootContext, ex); } ExecutionException = ex; return FAILURE; } finally { + RuntimeErrors |= _rootContext.RuntimeErrors; logger.AnalysisStopped(RuntimeErrors); } } @@ -127,12 +128,9 @@ public override int Run(TOptions options) } } - if (options.RichReturnCode) - { - return (int)RuntimeErrors; - } - - return succeeded ? SUCCESS : FAILURE; + return options.RichReturnCode + ? (int)RuntimeErrors + : succeeded ? SUCCESS : FAILURE; } finally { @@ -214,7 +212,7 @@ private void MultithreadedAnalyzeTargets(TOptions options, // 1: First we initiate an asynchronous operation to locate disk files for // analysis, as specified in analysis configuration (file names, wildcards). - Task enumerateFilesOnDisk = EnumerateFilesOnDiskAsync(options, rootContext); + Task enumerateFilesOnDisk = EnumerateFilesOnDiskAsync(options); // 2: Files found on disk are put in a specific sort order, after which a // reference to each scan target is put into a channel for hashing, @@ -297,9 +295,14 @@ private async Task LogScanResultsAsync(TContext rootContext) } catch (Exception e) { - context?.Dispose(); + if (context != null) + { + RuntimeErrors |= context.RuntimeErrors; + context?.Dispose(); + } context = default; - RuntimeErrors |= Errors.LogUnhandledEngineException(rootContext, e); + Errors.LogUnhandledEngineException(rootContext, e); + RuntimeErrors |= rootContext.RuntimeErrors; ThrowExitApplicationException(context, ExitReason.ExceptionWritingToLogFile, e); } } @@ -311,7 +314,7 @@ private async Task LogScanResultsAsync(TContext rootContext) return true; } - private void LogCachingLogger(TContext rootContext, TContext context, bool clone = false) + private static void LogCachingLogger(TContext rootContext, TContext context, bool clone = false) { var cachingLogger = (CachingLogger)context.Logger; IDictionary> results = cachingLogger.Results; @@ -363,107 +366,133 @@ private void LogCachingLogger(TContext rootContext, TContext context, bool clone } } - private async Task EnumerateFilesOnDiskAsync(TOptions options, TContext rootContext) + protected virtual bool ShouldEnqueue(string file, TContext context) { - this._fileContextsCount = 0; - this._fileContexts = new ConcurrentDictionary(); + bool shouldEnqueue = IsTargetWithinFileSizeLimit(file, context.MaxFileSizeInKilobytes, out long fileSizeInKb); - // INTERESTING BREAKPOINT: debug 'ERR997.NoValidAnalysisTargets : No valid analysis targets were specified.' - // Set a conditional breakpoint on 'matchExpression.Name' to filter by specific rules. - // Set a conditional breakpoint on 'searchText' to filter on specific target text patterns. - foreach (string specifier in options.TargetFileSpecifiers) + if (!shouldEnqueue) { - string normalizedSpecifier = Environment.ExpandEnvironmentVariables(specifier); + Warnings.LogFileSkippedDueToSize(context, file, fileSizeInKb); + } - if (Uri.TryCreate(specifier, UriKind.RelativeOrAbsolute, out Uri uri)) + return shouldEnqueue; + } + + private async Task EnumerateFilesOnDiskAsync(TOptions options) + { + try + { + this._fileContextsCount = 0; + this._fileContexts = new ConcurrentDictionary(); + + // INTERESTING BREAKPOINT: debug 'ERR997.NoValidAnalysisTargets : No valid analysis targets were specified.' + // Set a conditional breakpoint on 'matchExpression.Name' to filter by specific rules. + // Set a conditional breakpoint on 'searchText' to filter on specific target text patterns. + foreach (string specifier in options.TargetFileSpecifiers) { - if (uri.IsAbsoluteUri && (uri.IsFile || uri.IsUnc)) + string normalizedSpecifier = Environment.ExpandEnvironmentVariables(specifier); + + if (Uri.TryCreate(specifier, UriKind.RelativeOrAbsolute, out Uri uri)) { - normalizedSpecifier = uri.LocalPath; + if (uri.IsAbsoluteUri && (uri.IsFile || uri.IsUnc)) + { + normalizedSpecifier = uri.LocalPath; + } } - } - string filter = Path.GetFileName(normalizedSpecifier); - string directory = Path.GetDirectoryName(normalizedSpecifier); + string filter = Path.GetFileName(normalizedSpecifier); + string directory = Path.GetDirectoryName(normalizedSpecifier); - if (directory.Length == 0) - { - directory = $".{Path.DirectorySeparatorChar}"; - } + if (directory.Length == 0) + { + directory = $".{Path.DirectorySeparatorChar}"; + } - directory = Path.GetFullPath(directory); - var directories = new Queue(); + directory = Path.GetFullPath(directory); + var directories = new Queue(); - if (!FileSystem.DirectoryExists(directory)) - { - continue; - } + if (!FileSystem.DirectoryExists(directory)) + { + continue; + } - if (options.Recurse) - { - EnqueueAllDirectories(directories, directory); - } - else - { - directories.Enqueue(directory); - } + if (options.Recurse) + { + EnqueueAllDirectories(directories, directory); + } + else + { + directories.Enqueue(directory); + } - var sortedFiles = new SortedSet(); + var sortedFiles = new SortedSet(); - while (directories.Count > 0) - { - sortedFiles.Clear(); + while (directories.Count > 0) + { + sortedFiles.Clear(); - directory = Path.GetFullPath(directories.Dequeue()); + directory = Path.GetFullPath(directories.Dequeue()); #if NETFRAMEWORK - // .NET Framework: Directory.Enumerate with empty filter returns NO files. - // .NET Core: Directory.Enumerate with empty filter returns all files in directory. - // We will standardize on the .NET Core behavior. - if (string.IsNullOrEmpty(filter)) - { - filter = "*"; - } + // .NET Framework: Directory.Enumerate with empty filter returns NO files. + // .NET Core: Directory.Enumerate with empty filter returns all files in directory. + // We will standardize on the .NET Core behavior. + if (string.IsNullOrEmpty(filter)) + { + filter = "*"; + } #endif - foreach (string file in FileSystem.DirectoryEnumerateFiles(directory, filter, SearchOption.TopDirectoryOnly)) - { - // Only include files that are below the max size limit. - if (IsTargetWithinFileSizeLimit(file, _rootContext.MaxFileSizeInKilobytes)) + foreach (string file in FileSystem.DirectoryEnumerateFiles(directory, filter, SearchOption.TopDirectoryOnly)) { - sortedFiles.Add(file); - continue; + // Only include files that are below the max size limit. + if (ShouldEnqueue(file, _rootContext)) + { + sortedFiles.Add(file); + continue; + } + + if (!IsTargetWithinFileSizeLimit(file, _rootContext.MaxFileSizeInKilobytes, out long fileSizeInKb)) + { + _ignoredFilesCount++; + } } - _ignoredFilesCount++; - } - foreach (string file in sortedFiles) - { - _fileContexts.TryAdd( - _fileContextsCount, - CreateContext(options, - new CachingLogger(options.Level, options.Kind), - rootContext.RuntimeErrors, - rootContext.Policy, - filePath: file) - ); - - await readyToHashChannel.Writer.WriteAsync(_fileContextsCount++); + foreach (string file in sortedFiles) + { + _fileContexts.TryAdd( + _fileContextsCount, + CreateContext(options, + new CachingLogger(options.Level, options.Kind), + _rootContext.RuntimeErrors, + _rootContext.Policy, + filePath: file) + ); + + await readyToHashChannel.Writer.WriteAsync(_fileContextsCount++); + } } } } - - readyToHashChannel.Writer.Complete(); + catch (Exception e) + { + Errors.LogUnhandledEngineException(_rootContext, e); + ThrowExitApplicationException(_rootContext, ExitReason.UnhandledExceptionInEngine); + } + finally + { + readyToHashChannel.Writer.Complete(); + } if (_ignoredFilesCount > 0) { - Warnings.LogOneOrMoreFilesSkippedDueToSize(rootContext); + Warnings.LogOneOrMoreFilesSkippedDueToSize(_rootContext); } if (_fileContextsCount == 0) { - Errors.LogNoValidAnalysisTargets(rootContext); - ThrowExitApplicationException(rootContext, ExitReason.NoValidAnalysisTargets); + Errors.LogNoValidAnalysisTargets(_rootContext); + ThrowExitApplicationException(_rootContext, ExitReason.NoValidAnalysisTargets); } return true; @@ -573,7 +602,8 @@ protected virtual void ValidateOptions(TOptions options, TContext context) succeeded &= ValidateOutputFileCanBeCreated(context, options.OutputFilePath, options.Force); succeeded &= ValidateInvocationPropertiesToLog(context, options.InvocationPropertiesToLog); succeeded &= options.ValidateOutputOptions(context); - succeeded &= options.MaxFileSizeInKilobytes > 0; + + succeeded &= context.MaxFileSizeInKilobytes >= 0; if (!succeeded) { @@ -607,12 +637,16 @@ protected virtual TContext CreateContext(TOptions options, { var context = new TContext { + Policy = policy ?? new PropertiesDictionary(), Logger = logger, RuntimeErrors = runtimeErrors, - Policy = policy }; - context.MaxFileSizeInKilobytes = options.MaxFileSizeInKilobytes; + context.MaxFileSizeInKilobytes = + options.MaxFileSizeInKilobytes >= 0 + ? options.MaxFileSizeInKilobytes + : AnalyzeContextBase.MaxFileSizeInKilobytesDefaultValue; + if (filePath != null) { @@ -676,7 +710,7 @@ protected virtual void InitializeConfiguration(TOptions options, TContext contex private void InitializeOutputFile(TOptions analyzeOptions, TContext context) { string filePath = analyzeOptions.OutputFilePath; - AggregatingLogger aggregatingLogger = (AggregatingLogger)context.Logger; + var aggregatingLogger = (AggregatingLogger)context.Logger; if (!string.IsNullOrEmpty(filePath)) { @@ -744,7 +778,7 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context) } } - private IEnumerable GenerateSensitiveTokensList() + private static IEnumerable GenerateSensitiveTokensList() { var result = new List { @@ -814,7 +848,7 @@ protected virtual ISet> CreateSkimmers(TOptions options, TCont return result; } - private SupportedPlatform GetCurrentRunningOS() + private static SupportedPlatform GetCurrentRunningOS() { // RuntimeInformation is not present in NET452. #if NET452 @@ -897,7 +931,7 @@ protected virtual TContext DetermineApplicabilityAndAnalyze( return context; } - CachingLogger logger = (CachingLogger)context.Logger; + var logger = (CachingLogger)context.Logger; logger.AnalyzingTarget(context); if (logger.CacheFinalized) @@ -1108,7 +1142,7 @@ internal void CheckIncompatibleRules(IEnumerable> skimmers, TC protected virtual ISet> InitializeSkimmers(ISet> skimmers, TContext context) { - SortedSet> disabledSkimmers = new SortedSet>(SkimmerIdComparer.Instance); + var disabledSkimmers = new SortedSet>(SkimmerIdComparer.Instance); // ONE-TIME initialization of skimmers. Do not call // Initialize more than once per skimmer instantiation diff --git a/src/Sarif.Driver/Sdk/PluginDriverCommand.cs b/src/Sarif.Driver/Sdk/PluginDriverCommand.cs index 76d19f26f..1ca392ea0 100644 --- a/src/Sarif.Driver/Sdk/PluginDriverCommand.cs +++ b/src/Sarif.Driver/Sdk/PluginDriverCommand.cs @@ -19,12 +19,21 @@ namespace Microsoft.CodeAnalysis.Sarif.Driver { public abstract class PluginDriverCommand : DriverCommand { + // The plugin assemblies that contain IOptionProvider instances. public virtual IEnumerable DefaultPluginAssemblies { get { return null; } set { throw new InvalidOperationException(); } } + // An additional IOptionsProvider instance, typically, the one + // that exposes a client tool command-line interface. + public virtual IOptionsProvider AdditionalOptionsProvider + { + get { return null; } + set { throw new InvalidOperationException(); } + } + public IEnumerable RetrievePluginAssemblies(IEnumerable defaultPluginAssemblies, IEnumerable pluginFilePaths) { if (pluginFilePaths == null) @@ -41,11 +50,11 @@ public IEnumerable RetrievePluginAssemblies(IEnumerable defa return assemblies; } - public bool IsTargetWithinFileSizeLimit(string path, int maxFileSize) + internal bool IsTargetWithinFileSizeLimit(string path, long maxFileSizeInKB, out long fileSizeInKb) { - long fileSize = FileSystem.FileInfoLength(path) / 1024; - - return (maxFileSize == -1 || fileSize < maxFileSize); + long size = Math.Max(FileSystem.FileInfoLength(path), 1024); + fileSizeInKb = size / 1024; + return fileSizeInKb <= maxFileSizeInKB; } internal static bool ValidateInvocationPropertiesToLog(IAnalysisContext context, IEnumerable propertiesToLog) diff --git a/src/Sarif.Multitool.Library/SarifValidationContext.cs b/src/Sarif.Multitool.Library/SarifValidationContext.cs index 75aa723ce..bfe353f88 100644 --- a/src/Sarif.Multitool.Library/SarifValidationContext.cs +++ b/src/Sarif.Multitool.Library/SarifValidationContext.cs @@ -8,7 +8,7 @@ namespace Microsoft.CodeAnalysis.Sarif.Multitool { - public class SarifValidationContext : IAnalysisContext + public class SarifValidationContext : AnalyzeContextBase { public enum ReportingDescriptorKind { @@ -25,7 +25,7 @@ public SarifValidationContext() CurrentReportingDescriptorKind = ReportingDescriptorKind.None; } - public bool IsValidAnalysisTarget + public override bool IsValidAnalysisTarget { get { @@ -34,31 +34,29 @@ public bool IsValidAnalysisTarget } } - public IAnalysisLogger Logger { get; set; } + public override IAnalysisLogger Logger { get; set; } - public string MimeType + public override string MimeType { get { return "application/sarif-json"; } set { throw new InvalidOperationException(); } } - public bool AnalysisComplete { get; set; } + public override bool AnalysisComplete { get; set; } - public HashData Hashes { get; set; } + public override HashData Hashes { get; set; } - public PropertiesDictionary Policy { get; set; } + public override ReportingDescriptor Rule { get; set; } - public ReportingDescriptor Rule { get; set; } + public override RuntimeConditions RuntimeErrors { get; set; } - public RuntimeConditions RuntimeErrors { get; set; } - - public Exception TargetLoadException { get; set; } + public override Exception TargetLoadException { get; set; } public bool UpdateInputsToCurrentSarif { get; set; } private Uri _uri; - public Uri TargetUri + public override Uri TargetUri { get { @@ -94,11 +92,9 @@ public Uri TargetUri public JToken InputLogToken { get; internal set; } - public DefaultTraces Traces { get; set; } - - public int MaxFileSizeInKilobytes { get; set; } + public override DefaultTraces Traces { get; set; } - public void Dispose() + public override void Dispose() { // Nothing to dispose. } diff --git a/src/Sarif.Multitool.Library/ValidateCommand.cs b/src/Sarif.Multitool.Library/ValidateCommand.cs index c2cb35631..677196f12 100644 --- a/src/Sarif.Multitool.Library/ValidateCommand.cs +++ b/src/Sarif.Multitool.Library/ValidateCommand.cs @@ -14,7 +14,9 @@ namespace Microsoft.CodeAnalysis.Sarif.Multitool { +#pragma warning disable CS0618 public class ValidateCommand : AnalyzeCommandBase +#pragma warning restore CS0618 { private List _defaultPlugInAssemblies; diff --git a/src/Sarif.Multitool/AnalyzeTestContext.cs b/src/Sarif.Multitool/AnalyzeTestContext.cs index 755a01416..bde492a5f 100644 --- a/src/Sarif.Multitool/AnalyzeTestContext.cs +++ b/src/Sarif.Multitool/AnalyzeTestContext.cs @@ -6,33 +6,31 @@ namespace Microsoft.CodeAnalysis.Sarif.Multitool { - public class AnalyzeTestContext : IAnalysisContext +#pragma warning disable CS0618 + public class AnalyzeTestContext : AnalyzeContextBase +#pragma warning restore CS0618 { - public Exception TargetLoadException { get; set; } + public override Exception TargetLoadException { get; set; } - public bool IsValidAnalysisTarget => true; + public override bool IsValidAnalysisTarget => true; - public IAnalysisLogger Logger { get; set; } + public override IAnalysisLogger Logger { get; set; } - public ReportingDescriptor Rule { get; set; } + public override ReportingDescriptor Rule { get; set; } - public PropertiesDictionary Policy { get; set; } + public override string MimeType { get; set; } - public string MimeType { get; set; } + public override HashData Hashes { get; set; } - public HashData Hashes { get; set; } + public override RuntimeConditions RuntimeErrors { get; set; } - public RuntimeConditions RuntimeErrors { get; set; } + public override Uri TargetUri { get; set; } - public Uri TargetUri { get; set; } + public override bool AnalysisComplete { get; set; } - public bool AnalysisComplete { get; set; } + public override DefaultTraces Traces { get; set; } - public DefaultTraces Traces { get; set; } - - public int MaxFileSizeInKilobytes { get; set; } - - public void Dispose() { } + public override void Dispose() { } } } #endif diff --git a/src/Sarif/AnalyzeContextBase.cs b/src/Sarif/AnalyzeContextBase.cs new file mode 100644 index 000000000..54915987a --- /dev/null +++ b/src/Sarif/AnalyzeContextBase.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; + +namespace Microsoft.CodeAnalysis.Sarif +{ + public abstract class AnalyzeContextBase : IAnalysisContext, IOptionsProvider + { + public AnalyzeContextBase() + { + this.Policy = new PropertiesDictionary(); + } + + // All of these properties are persisted to configuration XML and can be + // passed using that mechanism. All command-line arguments are + // candidates to follow this pattern. + public IEnumerable GetOptions() + { + return new[] + { + MaxFileSizeInKilobytesProperty + }; + } + + public abstract Uri TargetUri { get; set; } + public abstract string MimeType { get; set; } + public abstract HashData Hashes { get; set; } + public abstract Exception TargetLoadException { get; set; } + public abstract bool IsValidAnalysisTarget { get; } + public abstract ReportingDescriptor Rule { get; set; } + public PropertiesDictionary Policy { get; set; } + public abstract IAnalysisLogger Logger { get; set; } + public abstract RuntimeConditions RuntimeErrors { get; set; } + public abstract bool AnalysisComplete { get; set; } + public abstract DefaultTraces Traces { get; set; } + + public long MaxFileSizeInKilobytes + { + get { return this.Policy.GetProperty(MaxFileSizeInKilobytesProperty); } + set { this.Policy.SetProperty(MaxFileSizeInKilobytesProperty, value); } + } + + abstract public void Dispose(); + + public static long MaxFileSizeInKilobytesDefaultValue { get; set; } = 1024; + + public static PerLanguageOption MaxFileSizeInKilobytesProperty { get; } = + new PerLanguageOption( + "CoreSettings", nameof(MaxFileSizeInKilobytes), defaultValue: () => MaxFileSizeInKilobytesDefaultValue, + "Scan targets that fall below this size threshold (in kilobytes) will not be analyzed. " + + "It is legal to set this value to 0 (in order to potentially complete an analysis that " + + "records what scan targets would have been analyzed, given current configuration. " + + $"Negative values will be discarded in favor of the default of {MaxFileSizeInKilobytesDefaultValue} KB."); + } +} diff --git a/src/Sarif/Errors.cs b/src/Sarif/Errors.cs index 794b46cf7..35f209180 100644 --- a/src/Sarif/Errors.cs +++ b/src/Sarif/Errors.cs @@ -46,8 +46,10 @@ public static void LogExceptionLoadingTarget(IAnalysisContext context) throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionLoadingTargetFile; + string message = context.TargetLoadException.Message; - string execptionType = context.TargetLoadException.GetType().Name; + string exceptionType = context.TargetLoadException.GetType().Name; // Could not load analysis target '{0}' ({1} : '{2}'). context.Logger.LogConfigurationNotification( @@ -60,10 +62,8 @@ public static void LogExceptionLoadingTarget(IAnalysisContext context) persistExceptionStack: true, messageFormat: null, context.TargetUri.GetFileName(), - execptionType, + exceptionType, message)); - - context.RuntimeErrors |= RuntimeConditions.ExceptionLoadingTargetFile; } public static void LogExceptionInstantiatingSkimmers( @@ -76,6 +76,8 @@ public static void LogExceptionInstantiatingSkimmers( throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionInstantiatingSkimmers; + string plugins = string.Join(", ", skimmerAssemblies.Select(sa => '"' + Path.GetFileName(sa.Location) + '"')); @@ -90,8 +92,6 @@ public static void LogExceptionInstantiatingSkimmers( persistExceptionStack: false, messageFormat: null, plugins)); - - context.RuntimeErrors |= RuntimeConditions.ExceptionInstantiatingSkimmers; } public static void LogNoRulesLoaded(IAnalysisContext context) @@ -101,6 +101,8 @@ public static void LogNoRulesLoaded(IAnalysisContext context) throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.NoRulesLoaded; + // No analysis rules could be instantiated. context.Logger.LogConfigurationNotification( CreateNotification( @@ -111,8 +113,6 @@ public static void LogNoRulesLoaded(IAnalysisContext context) exception: null, persistExceptionStack: false, messageFormat: null)); - - context.RuntimeErrors |= RuntimeConditions.NoRulesLoaded; } public static void LogAllRulesExplicitlyDisabled(IAnalysisContext context) @@ -122,6 +122,8 @@ public static void LogAllRulesExplicitlyDisabled(IAnalysisContext context) throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.NoRulesLoaded; + // All rules were explicitly disabled so there is no work to do. context.Logger.LogConfigurationNotification( CreateNotification( @@ -132,8 +134,6 @@ public static void LogAllRulesExplicitlyDisabled(IAnalysisContext context) exception: null, persistExceptionStack: false, messageFormat: null)); - - context.RuntimeErrors |= RuntimeConditions.NoRulesLoaded; } public static void LogNoPluginsConfigured(IAnalysisContext context) { @@ -142,6 +142,8 @@ public static void LogNoPluginsConfigured(IAnalysisContext context) throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.NoRulesLoaded; + // No analysis plugins were configured, therefore no rules loaded. context.Logger.LogConfigurationNotification( Errors.CreateNotification( @@ -153,7 +155,6 @@ public static void LogNoPluginsConfigured(IAnalysisContext context) persistExceptionStack: false, messageFormat: null)); - context.RuntimeErrors |= RuntimeConditions.NoRulesLoaded; } public static void LogNoValidAnalysisTargets(IAnalysisContext context) @@ -163,6 +164,8 @@ public static void LogNoValidAnalysisTargets(IAnalysisContext context) throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.NoValidAnalysisTargets; + // No valid analysis targets were specified. context.Logger.LogConfigurationNotification( CreateNotification( @@ -174,7 +177,6 @@ public static void LogNoValidAnalysisTargets(IAnalysisContext context) persistExceptionStack: false, messageFormat: null)); - context.RuntimeErrors |= RuntimeConditions.NoValidAnalysisTargets; } public static void LogExceptionCreatingLogFile(IAnalysisContext context, string fileName, Exception exception) @@ -184,6 +186,8 @@ public static void LogExceptionCreatingLogFile(IAnalysisContext context, string throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionCreatingLogFile; + // Could not create output file: '{0}' context.Logger.LogConfigurationNotification( CreateNotification( @@ -195,8 +199,6 @@ public static void LogExceptionCreatingLogFile(IAnalysisContext context, string persistExceptionStack: false, messageFormat: null, fileName)); - - context.RuntimeErrors |= RuntimeConditions.ExceptionCreatingLogFile; } public static void LogMissingFile(IAnalysisContext context, string fileName) @@ -206,6 +208,8 @@ public static void LogMissingFile(IAnalysisContext context, string fileName) throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.MissingFile; + // A required file specified on the command line could not be found: '{0}'. context.Logger.LogConfigurationNotification( CreateNotification( @@ -217,8 +221,6 @@ public static void LogMissingFile(IAnalysisContext context, string fileName) persistExceptionStack: false, messageFormat: null, fileName)); - - context.RuntimeErrors |= RuntimeConditions.MissingFile; } public static void LogExceptionAccessingFile(IAnalysisContext context, string fileName, Exception exception) @@ -228,6 +230,8 @@ public static void LogExceptionAccessingFile(IAnalysisContext context, string fi throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionAccessingFile; + // Could not access a file specified on the command-line: '{0}'. context.Logger.LogConfigurationNotification( CreateNotification( @@ -239,8 +243,6 @@ public static void LogExceptionAccessingFile(IAnalysisContext context, string fi persistExceptionStack: false, messageFormat: null, fileName)); - - context.RuntimeErrors |= RuntimeConditions.ExceptionAccessingFile; } public static void LogInvalidInvocationPropertyName(IAnalysisContext context, string propertyName) @@ -250,6 +252,8 @@ public static void LogInvalidInvocationPropertyName(IAnalysisContext context, st throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.InvalidCommandLineOption; + // '{0}' is not a property of the Invocation object. context.Logger.LogConfigurationNotification( CreateNotification( @@ -261,18 +265,18 @@ public static void LogInvalidInvocationPropertyName(IAnalysisContext context, st persistExceptionStack: false, messageFormat: null, propertyName)); - - context.RuntimeErrors |= RuntimeConditions.InvalidCommandLineOption; } - public static void LogMissingreportingConfiguration(IAnalysisContext context, string reasonForNotAnalyzing) + public static void LogMissingReportingConfiguration(IAnalysisContext context, string reasonForNotAnalyzing) { if (context == null) { throw new ArgumentNullException(nameof(context)); } - Assembly assembly = Assembly.GetEntryAssembly(); + context.RuntimeErrors |= RuntimeConditions.RuleMissingRequiredConfiguration; + + var assembly = Assembly.GetEntryAssembly(); assembly ??= Assembly.GetExecutingAssembly(); string exeName = Path.GetFileName(assembly.Location); @@ -316,8 +320,6 @@ public static void LogMissingreportingConfiguration(IAnalysisContext context, st Level = FailureLevel.Error, Message = new Message { Text = message } }); - - context.RuntimeErrors |= RuntimeConditions.RuleMissingRequiredConfiguration; } public static void LogExceptionLoadingPlugin(string pluginFilePath, IAnalysisContext context, Exception exception) @@ -327,6 +329,8 @@ public static void LogExceptionLoadingPlugin(string pluginFilePath, IAnalysisCon throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionLoadingAnalysisPlugin; + // Could not load plug-in '{0}'. context.Logger.LogConfigurationNotification( CreateNotification( @@ -338,8 +342,6 @@ public static void LogExceptionLoadingPlugin(string pluginFilePath, IAnalysisCon persistExceptionStack: false, messageFormat: null, pluginFilePath)); - - context.RuntimeErrors |= RuntimeConditions.ExceptionLoadingAnalysisPlugin; } public static void LogOutputFileAlreadyExists(IAnalysisContext context, string outputFilePath) @@ -349,6 +351,8 @@ public static void LogOutputFileAlreadyExists(IAnalysisContext context, string o throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.OutputFileAlreadyExists; + context.Logger.LogConfigurationNotification( CreateNotification( context.TargetUri, @@ -359,8 +363,6 @@ public static void LogOutputFileAlreadyExists(IAnalysisContext context, string o persistExceptionStack: false, messageFormat: null, outputFilePath)); - - context.RuntimeErrors |= RuntimeConditions.OutputFileAlreadyExists; } public static void LogTargetParseError(IAnalysisContext context, Region region, string message) @@ -370,6 +372,8 @@ public static void LogTargetParseError(IAnalysisContext context, Region region, throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.TargetParseError; + // {0}({1}): error {2}: {3} context.Logger.LogConfigurationNotification( CreateNotification( @@ -384,8 +388,6 @@ public static void LogTargetParseError(IAnalysisContext context, Region region, region.FormatForVisualStudio(), ERR1000_ParseError, message)); - - context.RuntimeErrors |= RuntimeConditions.TargetParseError; } public static void LogUnhandledRuleExceptionAssessingTargetApplicability( @@ -398,6 +400,8 @@ public static void LogUnhandledRuleExceptionAssessingTargetApplicability( throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionRaisedInSkimmerCanAnalyze; + // An unhandled exception was raised attempting to determine whether '{0}' // is a valid analysis target for check '{1}' (which has been disabled // for the remainder of the analysis). The exception may have resulted @@ -422,8 +426,6 @@ public static void LogUnhandledRuleExceptionAssessingTargetApplicability( disabledSkimmers.Add(context.Rule.Id); } } - - context.RuntimeErrors |= RuntimeConditions.ExceptionRaisedInSkimmerCanAnalyze; } public static void LogUnhandledExceptionInitializingRule(IAnalysisContext context, Exception exception) @@ -433,6 +435,8 @@ public static void LogUnhandledExceptionInitializingRule(IAnalysisContext contex throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionInSkimmerInitialize; + // An unhandled exception was encountered initializing check '{0}', which // has been disabled for the remainder of the analysis. context.Logger.LogToolNotification( @@ -445,8 +449,6 @@ public static void LogUnhandledExceptionInitializingRule(IAnalysisContext contex persistExceptionStack: true, messageFormat: null, context.Rule.Name)); - - context.RuntimeErrors |= RuntimeConditions.ExceptionInSkimmerInitialize; } public static void LogUnhandledRuleExceptionAnalyzingTarget( @@ -459,6 +461,8 @@ public static void LogUnhandledRuleExceptionAnalyzingTarget( throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionInSkimmerAnalyze; + // An unhandled exception of type '{0}' was encountered analyzing // '{0}' for check '{1}' (which has been disabled for the // remainder of the analysis. The exception may have resulted @@ -484,17 +488,17 @@ public static void LogUnhandledRuleExceptionAnalyzingTarget( disabledSkimmers.Add(context.Rule.Id); } } - - context.RuntimeErrors |= RuntimeConditions.ExceptionInSkimmerAnalyze; } - public static RuntimeConditions LogUnhandledEngineException(IAnalysisContext context, Exception exception) + public static void LogUnhandledEngineException(IAnalysisContext context, Exception exception) { if (context == null) { throw new ArgumentNullException(nameof(context)); } + context.RuntimeErrors |= RuntimeConditions.ExceptionInEngine; + // An unhandled exception was raised during analysis. context.Logger.LogToolNotification( CreateNotification( @@ -506,8 +510,6 @@ public static RuntimeConditions LogUnhandledEngineException(IAnalysisContext con persistExceptionStack: true, messageFormat: "{0}", args: new string[] { exception.ToString() })); - - return RuntimeConditions.ExceptionInEngine; } public static void LogIncompatibleRules(IAnalysisContext context, string ruleId, string incompatibleRuleId) diff --git a/src/Sarif/IAnalysisContext.cs b/src/Sarif/IAnalysisContext.cs index 282a08aee..2fd54aa1a 100644 --- a/src/Sarif/IAnalysisContext.cs +++ b/src/Sarif/IAnalysisContext.cs @@ -30,6 +30,6 @@ public interface IAnalysisContext : IDisposable DefaultTraces Traces { get; set; } - int MaxFileSizeInKilobytes { get; set; } + long MaxFileSizeInKilobytes { get; set; } } } diff --git a/src/Sarif/SdkResources.Designer.cs b/src/Sarif/SdkResources.Designer.cs index 52ec43970..0eb9004fd 100644 --- a/src/Sarif/SdkResources.Designer.cs +++ b/src/Sarif/SdkResources.Designer.cs @@ -503,6 +503,15 @@ public static string ValueMustBeAtLeastOne { } } + /// + /// Looks up a localized string similar to '{0}' was not analyzed as its size ({1} kilobytes) exceeds the currently configured threshold ({2} kilobytes).. + /// + public static string WRN997_FileSkippedDueToSize { + get { + return ResourceManager.GetString("WRN997.FileSkippedDueToSize", resourceCulture); + } + } + /// /// Looks up a localized string similar to Option '{0}' is invalid for this command.. /// diff --git a/src/Sarif/SdkResources.resx b/src/Sarif/SdkResources.resx index 737fe06be..1c070b09a 100644 --- a/src/Sarif/SdkResources.resx +++ b/src/Sarif/SdkResources.resx @@ -284,6 +284,9 @@ You can also refer to properties in the result's property bag with 'properties.& One or more files were skipped for analysis due to exceeding size limits (currently configured as {0} KB). The 'max-file-size-in-kb' command-line argument can be used to increase this threshold. + + '{0}' was not analyzed as its size ({1} kilobytes) exceeds the currently configured threshold ({2} kilobytes). + The current configuration enables rules that are not compatible ('{0}' has declared that it is not compatible with '{1}'). You can selectively disable one of the rules using an updated XML configuration (passed by the --config argument). diff --git a/src/Sarif/Warnings.cs b/src/Sarif/Warnings.cs index e6fa2a428..14fbe1863 100644 --- a/src/Sarif/Warnings.cs +++ b/src/Sarif/Warnings.cs @@ -11,6 +11,7 @@ public static class Warnings { // Conditions that may indicate an issue with command-line configuration. public const string Wrn997_InvalidTarget = "WRN997.InvalidTarget"; + public const string Wrn997_FileSkippedDueToSize = "WRN997.FileSkippedDueToSize"; public const string Wrn997_OneOrMoreFilesSkippedDueToSize = "WRN997.OneOrMoreFilesSkippedDueToSize"; public const string Wrn997_ObsoleteOption = "WRN997.ObsoleteOption"; @@ -22,6 +23,30 @@ public static class Warnings // Warnings around dangerous public const string Wrn999_RuleExplicitlyDisabled = "WRN999.RuleExplicitlyDisabled"; + public static void LogFileSkippedDueToSize(IAnalysisContext context, string skippedFile, long fileSizeInKb) + { + if (context == null) + { + throw new ArgumentNullException(nameof(context)); + } + + // '{0}' was skipped as its size ({1} kilobytes) exceeds the currently configured threshold ({2} kilobytes). + context.Logger.LogConfigurationNotification( + Errors.CreateNotification( + context.TargetUri, + Wrn997_FileSkippedDueToSize, + ruleId: null, + FailureLevel.Warning, + exception: null, + persistExceptionStack: false, + messageFormat: null, + skippedFile, + fileSizeInKb.ToString(CultureInfo.CurrentCulture), + context.MaxFileSizeInKilobytes.ToString())); + + context.RuntimeErrors |= RuntimeConditions.OneOrMoreFilesSkippedDueToSize; + } + public static void LogOneOrMoreFilesSkippedDueToSize(IAnalysisContext context) { if (context == null) diff --git a/src/Shared/CommonAssemblyInfo.cs b/src/Shared/CommonAssemblyInfo.cs index 98cf86773..d902352be 100644 --- a/src/Shared/CommonAssemblyInfo.cs +++ b/src/Shared/CommonAssemblyInfo.cs @@ -16,6 +16,7 @@ // This reference necessary for the MOQ test engine. [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] +[assembly: InternalsVisibleTo("Sarif.Driver, PublicKey=0024000004800000940000000602000000240000525341310004000001000100433fbf156abe9718142bdbd48a440e779a1b708fd21486ee0ae536f4c548edf8a7185c1e3ac89ceef76c15b8cc2497906798779a59402f9b9e27281fb15e7111566cdc9a9f8326301d45320623c5222089cf4d0013f365ae729fb0a9c9d15138042825cd511a0f3d4887a7b92f4c2749f81b410813d297b73244cf64995effb1")] // By default, we expose all internal product data to test binaries [assembly: InternalsVisibleTo("Test.FunctionalTests.Sarif, PublicKey=0024000004800000940000000602000000240000525341310004000001000100433fbf156abe9718142bdbd48a440e779a1b708fd21486ee0ae536f4c548edf8a7185c1e3ac89ceef76c15b8cc2497906798779a59402f9b9e27281fb15e7111566cdc9a9f8326301d45320623c5222089cf4d0013f365ae729fb0a9c9d15138042825cd511a0f3d4887a7b92f4c2749f81b410813d297b73244cf64995effb1")] diff --git a/src/Test.FunctionalTests.Sarif/Multitool/BaselineOptionTests.cs b/src/Test.FunctionalTests.Sarif/Multitool/BaselineOptionTests.cs index efe891b43..3f76c4462 100644 --- a/src/Test.FunctionalTests.Sarif/Multitool/BaselineOptionTests.cs +++ b/src/Test.FunctionalTests.Sarif/Multitool/BaselineOptionTests.cs @@ -115,7 +115,6 @@ protected override string ConstructTestOutputFromInputResource(string inputResou Quiet = true, PrettyPrint = true, Optimize = true, - Kind = new List { ResultKind.Fail }, Level = new List { FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note, FailureLevel.None }, }; diff --git a/src/Test.FunctionalTests.Sarif/Multitool/ValidateCommandTests.cs b/src/Test.FunctionalTests.Sarif/Multitool/ValidateCommandTests.cs index 96f581266..51efeb726 100644 --- a/src/Test.FunctionalTests.Sarif/Multitool/ValidateCommandTests.cs +++ b/src/Test.FunctionalTests.Sarif/Multitool/ValidateCommandTests.cs @@ -508,7 +508,6 @@ protected override string ConstructTestOutputFromInputResource(string inputResou UpdateInputsToCurrentSarif = updateInputsToCurrentSarif, PrettyPrint = true, Optimize = true, - Kind = new List { ResultKind.Fail }, Level = new List { FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note, FailureLevel.None }, }; diff --git a/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs b/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs index 50b253fbc..7c7188926 100644 --- a/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs +++ b/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs @@ -1,6 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. - +#pragma warning disable CS0618 using System; using System.Collections.Generic; using System.Data.Common; @@ -9,7 +9,6 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Text; -using System.Threading.Tasks; using FluentAssertions; @@ -20,7 +19,6 @@ using Microsoft.CodeAnalysis.Test.Utilities.Sarif; using Microsoft.Coyote; -using Microsoft.Coyote.Specifications; using Microsoft.Coyote.SystematicTesting; using Moq; @@ -42,6 +40,7 @@ public class AnalyzeCommandBaseTests public AnalyzeCommandBaseTests(ITestOutputHelper output) { this.Output = output; + Output.WriteLine($"The seed that will be used is: {TestRule.s_seed}"); } [Fact] @@ -763,69 +762,59 @@ public void MultithreadedAnalyzeCommandBase_TargetFileSizeTestCases() dynamic[] testCases = new[] { new { - expectedExitReason = ExitReason.InvalidCommandLineOption, - fileSize = (long)ulong.MinValue, - maxFileSize = int.MinValue - }, - new { - expectedExitReason = ExitReason.InvalidCommandLineOption, - fileSize = (long)ulong.MinValue, - maxFileSize = -1 + expectedExitReason = ExitReason.NoValidAnalysisTargets, + fileSize = (long)1023, + maxFileSize = (long)0 }, new { - expectedExitReason = ExitReason.InvalidCommandLineOption, - fileSize = (long)ulong.MinValue, - maxFileSize = 0 + expectedExitReason = ExitReason.NoValidAnalysisTargets, + fileSize = (long)0, + maxFileSize = (long)0 }, new { expectedExitReason = ExitReason.None, fileSize = (long)ulong.MinValue, - maxFileSize = 1 + maxFileSize = (long)1 }, new { expectedExitReason = ExitReason.None, fileSize = (long)ulong.MinValue, - maxFileSize = 2000 + maxFileSize = (long)2000 }, new { expectedExitReason = ExitReason.None, fileSize = (long)ulong.MinValue, - maxFileSize = 1000 + maxFileSize = (long)1000 }, new { expectedExitReason = ExitReason.None, fileSize = (long)ulong.MinValue, - maxFileSize = int.MaxValue + maxFileSize = long.MaxValue }, new { expectedExitReason = ExitReason.NoValidAnalysisTargets, fileSize = (long)20000, - maxFileSize = 1 + maxFileSize = (long)1 }, new { expectedExitReason = ExitReason.None, fileSize = (long)20000, - maxFileSize = int.MaxValue + maxFileSize = long.MaxValue }, new { expectedExitReason = ExitReason.None, fileSize = (long)10, - maxFileSize = 10 + maxFileSize = (long)10 }, new { - expectedExitReason = ExitReason.InvalidCommandLineOption, - fileSize = long.MaxValue, - maxFileSize = int.MinValue - }, - new { - expectedExitReason = ExitReason.InvalidCommandLineOption, + expectedExitReason = ExitReason.NoValidAnalysisTargets, fileSize = long.MaxValue, - maxFileSize = 0 + maxFileSize = (long)0 }, new { - expectedExitReason = ExitReason.NoValidAnalysisTargets, - fileSize = long.MaxValue, - maxFileSize = int.MaxValue + expectedExitReason = ExitReason.None, + fileSize = long.MaxValue - 1, + maxFileSize = long.MaxValue }, }; @@ -866,8 +855,6 @@ public void MultithreadedAnalyzeCommandBase_TargetFileSizeTestCases() bool expectedToBeWithinLimits = testCase.maxFileSize == -1 || testCase.fileSize / 1024 < testCase.maxFileSize; - Output.WriteLine($"The seed that will be used is: {TestRule.s_seed}"); - var options = new TestAnalyzeOptions { TargetFileSpecifiers = new[] { specifier }, @@ -1752,8 +1739,6 @@ private static void RunResultsCachingTestCase(ResultsCachingTestCase testCase, TestRuleBehaviors = testCase.TestRuleBehaviors, OutputFilePath = testCase.PersistLogFileToDisk ? Guid.NewGuid().ToString() : null, TargetFileSpecifiers = new string[] { Guid.NewGuid().ToString() }, - Kind = new List { ResultKind.Fail }, - Level = new List { FailureLevel.Warning, FailureLevel.Error }, }; EnhanceOptions(options, enhancedOptions); @@ -2254,3 +2239,4 @@ private void PostUriTestHelper(string postUri, int expectedReturnCode, RuntimeCo } } } +#pragma warning restore CS0618 diff --git a/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs b/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs index 0a6418a2a..4f83ee1e2 100644 --- a/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs +++ b/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs @@ -9,7 +9,9 @@ namespace Microsoft.CodeAnalysis.Sarif.Driver { +#pragma warning disable CS0618 public class TestAnalyzeCommand : AnalyzeCommandBase, ITestAnalyzeCommand +#pragma warning restore CS0618 { public TestAnalyzeCommand(IFileSystem fileSystem = null) : base(fileSystem) { @@ -31,9 +33,8 @@ protected override TestAnalysisContext CreateContext( { TestAnalysisContext context = base.CreateContext(options, logger, runtimeErrors, policy, filePath); - if (context.Policy == null) + if (policy == null) { - context.Policy ??= new PropertiesDictionary(); context.Policy.SetProperty(TestRule.Behaviors, options.TestRuleBehaviors); } diff --git a/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs b/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs index 8e9b2bf7a..107655a38 100644 --- a/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs +++ b/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs @@ -27,15 +27,9 @@ protected override TestAnalysisContext CreateContext( string filePath = null) { TestAnalysisContext context = base.CreateContext(options, logger, runtimeErrors, policy, filePath); - - if (context.Policy == null) - { - context.Policy = new PropertiesDictionary(); - context.Policy.SetProperty(TestRule.Behaviors, options.TestRuleBehaviors.AccessibleWithinContextOnly()); - } + context.Policy.SetProperty(TestRule.Behaviors, options.TestRuleBehaviors.AccessibleWithinContextOnly()); TestRuleBehaviors behaviors = context.Policy.GetProperty(TestRule.Behaviors); - context.IsValidAnalysisTarget = !behaviors.HasFlag(TestRuleBehaviors.RegardAnalysisTargetAsInvalid); context.TargetLoadException = diff --git a/src/Test.UnitTests.Sarif.Multitool.Library/BaselineOptionTests.cs b/src/Test.UnitTests.Sarif.Multitool.Library/BaselineOptionTests.cs index 5cee68657..d82cc0a50 100644 --- a/src/Test.UnitTests.Sarif.Multitool.Library/BaselineOptionTests.cs +++ b/src/Test.UnitTests.Sarif.Multitool.Library/BaselineOptionTests.cs @@ -60,8 +60,6 @@ public void ValidateCommandWithBaseline_Invalid_WhenOutputFileIsIsAbsent() SchemaFilePath = SchemaFilePath, TargetFileSpecifiers = new string[] { logFilePath }, BaselineSarifFile = baseLineFilePath, - Kind = new List { ResultKind.Fail }, - Level = new List { FailureLevel.Warning, FailureLevel.Error } }; int returnCode = validateCommand.Run(options); @@ -88,7 +86,8 @@ public void ValidateCommandWithBaseline_ResultShouldHaveBaselineStatus() Force = true, BaselineSarifFile = baselineFilePath, Kind = new List { ResultKind.Fail }, - Level = new List { FailureLevel.Warning, FailureLevel.Error } + Level = new List { FailureLevel.Warning, FailureLevel.Error }, + MaxFileSizeInKilobytes = 1024, }; // Verify command returned success @@ -121,9 +120,7 @@ public void ValidateCommandWithBaseline_OutputOneZeroZero_ResultShouldHaveBaseli OutputFilePath = outputPath, Force = true, BaselineSarifFile = baselineFilePath, - Kind = new List { ResultKind.Fail }, - Level = new List { FailureLevel.Warning, FailureLevel.Error }, - SarifOutputVersion = SarifVersion.OneZeroZero + SarifOutputVersion = SarifVersion.OneZeroZero, }; // Verify command returned success diff --git a/src/Test.Utilities.Sarif/TestAnalysisContext.cs b/src/Test.Utilities.Sarif/TestAnalysisContext.cs index fdf56aad1..6111171f4 100644 --- a/src/Test.Utilities.Sarif/TestAnalysisContext.cs +++ b/src/Test.Utilities.Sarif/TestAnalysisContext.cs @@ -35,7 +35,7 @@ public class TestAnalysisContext : IAnalysisContext public bool Disposed { get; private set; } - public int MaxFileSizeInKilobytes { get; set; } + public long MaxFileSizeInKilobytes { get; set; } public void Dispose() { diff --git a/src/Test.Utilities.Sarif/TestAnalyzeOptions.cs b/src/Test.Utilities.Sarif/TestAnalyzeOptions.cs index b6cc9fde9..a225feaa7 100644 --- a/src/Test.Utilities.Sarif/TestAnalyzeOptions.cs +++ b/src/Test.Utilities.Sarif/TestAnalyzeOptions.cs @@ -9,12 +9,6 @@ namespace Microsoft.CodeAnalysis.Sarif { public class TestAnalyzeOptions : AnalyzeOptionsBase { - public TestAnalyzeOptions() - { - Kind = new List { ResultKind.Fail }; - Level = new List { FailureLevel.Warning, FailureLevel.Error }; - } - public TestRuleBehaviors TestRuleBehaviors { get; set; } public bool DisableCheck { get; set; }