From b0dd7750465b9076ab1df44ba8a00636bbe4e498 Mon Sep 17 00:00:00 2001 From: "Michael C. Fanning" Date: Wed, 1 Feb 2023 17:01:15 -0800 Subject: [PATCH] Add rolling has optional data enum value. Remove SarifLogger tool parameter. (#2614) * Add rolling has optional data enum value. Remove SarifLogger tool parameter. * Remove unnecessary tool argument from all log construction. * Formatting updates. * Fine-tune extensions serialization. * Don't serialize empty guid values. * Add hash computation helpers. * Update release notes. * Fix compute hash function callback logic. * Formatting changes. --- src/ReleaseHistory.md | 7 +- src/Samples/Sarif.Sdk.Sample/Program.cs | 1 - src/Samples/SarifTrim/Program.cs | 2 +- src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs | 5 +- .../Sdk/MultithreadedAnalyzeCommandBase.cs | 5 +- src/Sarif/Core/Artifact.cs | 2 +- src/Sarif/Core/Tool.cs | 13 ++-- src/Sarif/Core/ToolComponent.cs | 10 +++ src/Sarif/HashUtilities.cs | 67 +++++++++++-------- src/Sarif/OptionallyEmittedData.cs | 9 ++- src/Sarif/Writers/SarifLogger.cs | 16 +++-- src/Sarif/Writers/SarifOneZeroZeroLogger.cs | 2 - 12 files changed, 84 insertions(+), 55 deletions(-) diff --git a/src/ReleaseHistory.md b/src/ReleaseHistory.md index a21b4f30f..c1a65b3da 100644 --- a/src/ReleaseHistory.md +++ b/src/ReleaseHistory.md @@ -1,8 +1,9 @@ # SARIF Package Release History (SDK, Driver, Converters, and Multitool) ## **v3.2.0** (UNRELEASED) -* BRK: `SarifLogger` updates version details differently. [#2661](https://github.com/microsoft/sarif-sdk/pull/2611) -* BRK: Add `ToolComponent` argument to `IAnalysisLogger.Log(ReportingDescriptor, Result)` method. [#2661](https://github.com/microsoft/sarif-sdk/pull/2611) +* BRK: `SarifLogger` no longer allows providing a `Tool` instance. Use the `run` parameter instead (and populate it with any custom `Tool` object). [#2614](https://github.com/microsoft/sarif-sdk/pull/2614) +* BRK: `SarifLogger` updates version details differently. [#2611](https://github.com/microsoft/sarif-sdk/pull/2611) +* BRK: Add `ToolComponent` argument to `IAnalysisLogger.Log(ReportingDescriptor, Result)` method. [#2611](https://github.com/microsoft/sarif-sdk/pull/2611) * BRK: Rename `--normalize-for-github` argument to `--normalize-for-ghas` for `convert` command and mark `--normalize-for-github` as obsolete. [#2581](https://github.com/microsoft/sarif-sdk/pull/2581) * BRK: Update `IAnalysisContext.LogToolNotification` method to add `ReportingDescriptor` parameter. This is required in order to populated `AssociatedRule` data in `Notification` instances. The new method has an option value of null for the `associatedRule` parameter to maximize build compatibility. [#2604](https://github.com/microsoft/sarif-sdk/pull/2604) * BRK: Correct casing of `LogMissingreportingConfiguration` helper to `LogMissingReportingConfiguration`. [#2599](https://github.com/microsoft/sarif-sdk/pull/2599) @@ -33,6 +34,8 @@ * NEW: Add `--normalize-for-ghas` argument to the `rewrite` command to ensure rewritten SARIF is compatible with GitHub Advanced Security (GHAS) ingestion requirements. [#2581](https://github.com/microsoft/sarif-sdk/pull/2581) * NEW: Allow per-line rolling (partial) hash computation for a file. [#2605](https://github.com/microsoft/sarif-sdk/pull/2605) * NEW: `SarifLogger` now supports extensions rules data when logging (by providing a `ToolComponent` instance to the result logging method). [#2661](https://github.com/microsoft/sarif-sdk/pull/2611) +* NEW: `SarifLogger` provides a `ComputeHashData` callback to provide hash data for in-memory scan targets. [#2614](https://github.com/microsoft/sarif-sdk/pull/2614) +* NEW: Provide `HashUtilities.ComputeHashes(Stream)` and `ComputeHashesForText(string) helpers. [#2614](https://github.com/microsoft/sarif-sdk/pull/2614) ## **v3.1.0** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/3.1.0) | [Driver](https://www.nuget.org/packages/Sarif.Driver/3.1.0) | [Converters](https://www.nuget.org/packages/Sarif.Converters/3.1.0) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/3.1.0) | [Multitool Library](https://www.nuget.org/packages/Sarif.Multitool.Library/3.1.0) * BUG: Loosen `System.Collections.Immutable` minimum version requirement to 1.5.0. [#2504](https://github.com/microsoft/sarif-sdk/pull/2533) diff --git a/src/Samples/Sarif.Sdk.Sample/Program.cs b/src/Samples/Sarif.Sdk.Sample/Program.cs index 3f20b8f7f..1280cb02f 100644 --- a/src/Samples/Sarif.Sdk.Sample/Program.cs +++ b/src/Samples/Sarif.Sdk.Sample/Program.cs @@ -328,7 +328,6 @@ private static void SerializeSarifResult(string filePath, int numResult, bool us OptionallyEmittedData.TextFiles | // Embed source file content directly in the log file -- great for portability of the log! OptionallyEmittedData.Hashes | OptionallyEmittedData.RegionSnippets, - tool: null, run: run, kinds: new ResultKind[] { ResultKind.Fail, ResultKind.Pass }, levels: new FailureLevel[] { FailureLevel.Error, FailureLevel.Warning, FailureLevel.Note }, diff --git a/src/Samples/SarifTrim/Program.cs b/src/Samples/SarifTrim/Program.cs index 7c56ef038..2b5dc09a2 100644 --- a/src/Samples/SarifTrim/Program.cs +++ b/src/Samples/SarifTrim/Program.cs @@ -53,7 +53,7 @@ private static void Main(string[] args) consolidator.RemoveWebResponses = (removeParts.Contains("WebResponses")); // Consolidate the SarifLog per settings - using (SarifLogger logger = new SarifLogger(outputFilePath, LogFilePersistenceOptions.OverwriteExistingOutputFile, tool: run.Tool, run: run)) + using (SarifLogger logger = new SarifLogger(outputFilePath, LogFilePersistenceOptions.OverwriteExistingOutputFile, run: run)) { foreach (Result result in run.Results) { diff --git a/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs index 85712f776..d6945f962 100644 --- a/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs @@ -411,7 +411,8 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context) { Id = analyzeOptions.AutomationId, Guid = analyzeOptions.AutomationGuid - } + }, + Tool = _tool, }; if (analyzeOptions.SarifOutputVersion != SarifVersion.OneZeroZero) @@ -421,7 +422,6 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context) logFilePersistenceOptions, dataToInsert, dataToRemove, - tool: _tool, run: _run, analysisTargets: null, quiet: analyzeOptions.Quiet, @@ -437,7 +437,6 @@ private void InitializeOutputFile(TOptions analyzeOptions, TContext context) logFilePersistenceOptions, dataToInsert, dataToRemove, - tool: _tool, run: _run, analysisTargets: null, invocationTokensToRedact: GenerateSensitiveTokensList(), diff --git a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs index efdb5e64d..f68116ef5 100644 --- a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs @@ -770,7 +770,8 @@ public virtual void InitializeOutputFile(TOptions analyzeOptions, TContext conte { Id = analyzeOptions.AutomationId, Guid = analyzeOptions.AutomationGuid - } + }, + Tool = Tool, }; if (analyzeOptions.SarifOutputVersion != SarifVersion.OneZeroZero) @@ -779,7 +780,6 @@ public virtual void InitializeOutputFile(TOptions analyzeOptions, TContext conte logFilePersistenceOptions, dataToInsert, dataToRemove, - tool: Tool, run: _run, analysisTargets: null, quiet: analyzeOptions.Quiet, @@ -795,7 +795,6 @@ public virtual void InitializeOutputFile(TOptions analyzeOptions, TContext conte logFilePersistenceOptions, dataToInsert, dataToRemove, - tool: Tool, run: _run, analysisTargets: null, invocationTokensToRedact: GenerateSensitiveTokensList(), diff --git a/src/Sarif/Core/Artifact.cs b/src/Sarif/Core/Artifact.cs index 5f729a6bc..2f1962cee 100644 --- a/src/Sarif/Core/Artifact.cs +++ b/src/Sarif/Core/Artifact.cs @@ -72,7 +72,7 @@ public static Artifact Create( { HashData hashes = hashData ?? HashUtilities.ComputeHashes(filePath); - // The hash utilities will return null data in some text contexts. + // The hash utilities will return null data in some test contexts. if (hashes != null) { artifact.Hashes = new Dictionary diff --git a/src/Sarif/Core/Tool.cs b/src/Sarif/Core/Tool.cs index 940c0bf1f..4d7d2b22b 100644 --- a/src/Sarif/Core/Tool.cs +++ b/src/Sarif/Core/Tool.cs @@ -42,9 +42,9 @@ public static Tool CreateFromAssemblyData(Assembly assembly = null, Driver = new ToolComponent { Name = name, - FullName = name + " " + version.ToString(), - Version = fileVersion.FileVersion, - DottedQuadFileVersion = dottedQuadFileVersion, + FullName = name + " " + (omitSemanticVersion ? null : version.ToString()), + Version = omitSemanticVersion ? null : fileVersion.FileVersion, + DottedQuadFileVersion = omitSemanticVersion ? null : dottedQuadFileVersion, SemanticVersion = omitSemanticVersion ? null : fileVersion.ProductVersion, Organization = string.IsNullOrEmpty(fileVersion.CompanyName) ? null : fileVersion.CompanyName, Product = string.IsNullOrEmpty(fileVersion.ProductName) ? null : fileVersion.ProductName, @@ -64,7 +64,7 @@ internal static string ParseFileVersion(string fileVersion) private static void SetDriverPropertiesFromFileVersionInfo(ToolComponent driver, FileVersionInfo fileVersion) { - if (!string.IsNullOrEmpty(fileVersion.Comments)) { driver.SetProperty("Comments", fileVersion.Comments); } + if (!string.IsNullOrEmpty(fileVersion.Comments)) { driver.SetProperty("comments", fileVersion.Comments); } } /// @@ -107,5 +107,10 @@ public ToolComponent GetToolComponentFromReference(ToolComponentReference refere // Neither specified? Driver. return this.Driver; } + + public bool ShouldSerializeExtensions() + { + return this.Extensions?.HasAtLeastOneNonNullValue() == true; + } } } diff --git a/src/Sarif/Core/ToolComponent.cs b/src/Sarif/Core/ToolComponent.cs index 3f2084deb..15d70b1ba 100644 --- a/src/Sarif/Core/ToolComponent.cs +++ b/src/Sarif/Core/ToolComponent.cs @@ -55,6 +55,16 @@ public ReportingDescriptor GetRuleByGuid(Guid ruleGuid) return rule; } + public bool ShouldSerializeGuid() + { + return this.Guid != default; + } + + public bool ShouldSerializeName() + { + return !string.IsNullOrWhiteSpace(this.Name); + } + public bool ShouldSerializeRules() { return this.Rules.HasAtLeastOneNonNullValue(); diff --git a/src/Sarif/HashUtilities.cs b/src/Sarif/HashUtilities.cs index d3fcda95b..ea1440ecd 100644 --- a/src/Sarif/HashUtilities.cs +++ b/src/Sarif/HashUtilities.cs @@ -100,42 +100,53 @@ public static HashData ComputeHashes(string fileName, IFileSystem fileSystem = n // and record exception details. if (stream == null) { return null; } - using (var bufferedStream = new BufferedStream(stream, 1024 * 32)) - { - string md5, sha1, sha256; - byte[] checksum; + return ComputeHashes(stream); + } + } + catch (IOException) { } + catch (UnauthorizedAccessException) { } + return null; + } - using (var md5Cng = MD5.Create()) - { - checksum = md5Cng.ComputeHash(bufferedStream); - md5 = BitConverter.ToString(checksum).Replace("-", string.Empty); - } + public static HashData ComputeHashes(Stream stream) + { + using (var bufferedStream = new BufferedStream(stream, 1024 * 32)) + { + string md5, sha1, sha256; + byte[] checksum; - stream.Seek(0, SeekOrigin.Begin); - bufferedStream.Seek(0, SeekOrigin.Begin); + using (var md5Cng = MD5.Create()) + { + checksum = md5Cng.ComputeHash(bufferedStream); + md5 = BitConverter.ToString(checksum).Replace("-", string.Empty); + } - using (var sha1Cng = SHA1.Create()) - { - checksum = sha1Cng.ComputeHash(bufferedStream); - sha1 = BitConverter.ToString(checksum).Replace("-", string.Empty); - } + stream.Seek(0, SeekOrigin.Begin); + bufferedStream.Seek(0, SeekOrigin.Begin); - stream.Seek(0, SeekOrigin.Begin); - bufferedStream.Seek(0, SeekOrigin.Begin); + using (var sha1Cng = SHA1.Create()) + { + checksum = sha1Cng.ComputeHash(bufferedStream); + sha1 = BitConverter.ToString(checksum).Replace("-", string.Empty); + } - using (var sha256Cng = SHA256.Create()) - { - checksum = sha256Cng.ComputeHash(bufferedStream); - sha256 = BitConverter.ToString(checksum).Replace("-", string.Empty); - } + stream.Seek(0, SeekOrigin.Begin); + bufferedStream.Seek(0, SeekOrigin.Begin); - return new HashData(md5, sha1, sha256); - } + using (var sha256Cng = SHA256.Create()) + { + checksum = sha256Cng.ComputeHash(bufferedStream); + sha256 = BitConverter.ToString(checksum).Replace("-", string.Empty); } + + return new HashData(md5, sha1, sha256); } - catch (IOException) { } - catch (UnauthorizedAccessException) { } - return null; + } + + public static HashData ComputeHashesForText(string text) + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); + return ComputeHashes(stream); } public static string ComputeSha256Hash(string fileName) diff --git a/src/Sarif/OptionallyEmittedData.cs b/src/Sarif/OptionallyEmittedData.cs index 04d2e9b6f..f13d8f29a 100644 --- a/src/Sarif/OptionallyEmittedData.cs +++ b/src/Sarif/OptionallyEmittedData.cs @@ -66,7 +66,7 @@ public enum OptionallyEmittedData : int // both versions exist FlattenedMessages = 0x100, - // SARIF Results may each have a GUID assigned to uniquely identify them. + // Assign a unique GUID to every SARIF result. Guids = 0x200, // Specifies the repository, branch, and other information describing the source controlled @@ -80,16 +80,19 @@ public enum OptionallyEmittedData : int // Enrich SARIF log with git blame information GitBlameInformation = 0x1000, - // Enrich SARIF log with partial fingerprint based on the hash value of the context region snippet (sha256) + // Enrich SARIF log with partial fingerprint based on the hash value of the context region snippet (sha256). ContextRegionSnippetPartialFingerprints = 0x2000, + // Enrich SARIF log with partial fingerprint based on the CodeQL rolling hash algorithm. + RollingHashPartialFingerprints = 0x4000, + // A special enum value that indicates that insertion should overwrite any existing // information in the SARIF log file. In the absence of this setting, any existing // data that would otherwise have been overwritten by the insert operation will // be preserved. OverwriteExistingData = 0x40000000, - // Insert Everything - should include every flag except the overwrite and git blame information ones + // Insert Everything - should include every flag except the overwrite and git blame information ones. All = ~OverwriteExistingData & ~GitBlameInformation } } diff --git a/src/Sarif/Writers/SarifLogger.cs b/src/Sarif/Writers/SarifLogger.cs index 822dfb6fb..b64c69711 100644 --- a/src/Sarif/Writers/SarifLogger.cs +++ b/src/Sarif/Writers/SarifLogger.cs @@ -34,7 +34,6 @@ public SarifLogger(string outputFilePath, LogFilePersistenceOptions logFilePersistenceOptions = DefaultLogFilePersistenceOptions, OptionallyEmittedData dataToInsert = OptionallyEmittedData.None, OptionallyEmittedData dataToRemove = OptionallyEmittedData.None, - Tool tool = null, Run run = null, IEnumerable analysisTargets = null, IEnumerable invocationTokensToRedact = null, @@ -50,7 +49,6 @@ public SarifLogger(string outputFilePath, logFilePersistenceOptions, dataToInsert, dataToRemove, - tool, run, analysisTargets, invocationTokensToRedact, @@ -69,7 +67,6 @@ public SarifLogger(TextWriter textWriter, LogFilePersistenceOptions logFilePersistenceOptions = DefaultLogFilePersistenceOptions, OptionallyEmittedData dataToInsert = OptionallyEmittedData.None, OptionallyEmittedData dataToRemove = OptionallyEmittedData.None, - Tool tool = null, Run run = null, IEnumerable analysisTargets = null, IEnumerable invocationTokensToRedact = null, @@ -126,9 +123,7 @@ public SarifLogger(TextWriter textWriter, defaultFileEncoding, AnalysisTargetToHashDataMap); - tool ??= Tool.CreateFromAssemblyData(); - - _run.Tool = tool; + _run.Tool ??= Tool.CreateFromAssemblyData(); _dataToInsert = dataToInsert; _dataToRemove = dataToRemove; _issueLogJsonWriter.Initialize(_run); @@ -264,6 +259,8 @@ private void EnhanceRun(IEnumerable analysisTargets, _run.Invocations.Add(invocation); } + public Func ComputeHashData { get; set; } + public IDictionary AnalysisTargetToHashDataMap { get; } public IDictionary RuleToReportingDescriptorReferenceMap { get; } @@ -510,7 +507,12 @@ private void CaptureArtifact(ArtifactLocation fileLocation) } HashData hashData = null; - AnalysisTargetToHashDataMap?.TryGetValue(fileLocation.Uri.OriginalString, out hashData); + if ((AnalysisTargetToHashDataMap == null || + !AnalysisTargetToHashDataMap.TryGetValue(fileLocation.Uri.OriginalString, out hashData)) && + ComputeFileHashes && ComputeHashData != null) + { + hashData = ComputeHashData(fileLocation.Uri); + } // Ensure Artifact is in Run.Artifacts and ArtifactLocation.Index is set to point to it int index = _run.GetFileIndex(fileLocation, diff --git a/src/Sarif/Writers/SarifOneZeroZeroLogger.cs b/src/Sarif/Writers/SarifOneZeroZeroLogger.cs index 19eb8a7b5..d797318c2 100644 --- a/src/Sarif/Writers/SarifOneZeroZeroLogger.cs +++ b/src/Sarif/Writers/SarifOneZeroZeroLogger.cs @@ -25,7 +25,6 @@ public SarifOneZeroZeroLogger( LogFilePersistenceOptions logFilePersistenceOptions = SarifLogger.DefaultLogFilePersistenceOptions, OptionallyEmittedData dataToInsert = OptionallyEmittedData.None, OptionallyEmittedData dataToRemove = OptionallyEmittedData.None, - Tool tool = null, Run run = null, IEnumerable analysisTargets = null, IEnumerable invocationTokensToRedact = null, @@ -39,7 +38,6 @@ public SarifOneZeroZeroLogger( dataToInsert: dataToInsert, dataToRemove: dataToRemove, defaultFileEncoding: defaultFileEncoding, - tool: tool, run: run, analysisTargets: analysisTargets, invocationTokensToRedact: invocationTokensToRedact,