Skip to content

Commit

Permalink
Reduced files array build (#1191)
Browse files Browse the repository at this point in the history 
* Sarif and Sarif.Converters now building

* Files array (#1188)

* Add fileIndex property to file object (#1186)

* Fix up tests

* PR feedback to improve schema comment

* Logical locations notes (#1185) (#1187)

* Respond to a small # of PR comments related to recent logical locations change.

* Fix visibility on helper

* Fix up v1 transformation with keys that collide

* Preserve decorated name data

* Rebaseline test for decorated name propagation

* Respond to PR feedback. Update tests to close test holes.

* Rebaseline updated test

* Test key collision in annotated code locations.

* Update baseline

* DRY out converters to isolate shared code.

* Restore essential change in schema that converts files dictionary to an array.

* Simplify ShouldSerialize logic

* Remove unused namespaces

* Respond to PR feedback.

* Respond to PR feedback
  • Loading branch information
@michaelcfanning
michaelcfanning authored Dec 31, 2018
1 parent 543fa50 commit 621ec57
Show file tree
Hide file tree
Showing 38 changed files with 255 additions and 442 deletions.
38 changes: 8 additions & 30 deletions src/Sarif.Converters/AndroidStudioConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Xml;

Expand Down Expand Up @@ -59,41 +58,20 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
XmlResolver = null
};

ISet<Result> results;
IList<Result> results;
using (XmlReader xmlReader = XmlReader.Create(input, settings))
{
results = ProcessAndroidStudioLog(xmlReader);
}

var tool = new Tool
{
Name = "AndroidStudio"
};

var fileInfoFactory = new FileInfoFactory(null, dataToInsert);
Dictionary<string, FileData> fileDictionary = fileInfoFactory.Create(results);


var run = new Run()
{
Tool = tool,
ColumnKind = ColumnKind.Utf16CodeUnits
Tool = new Tool { Name = ToolFormat.AndroidStudio },
ColumnKind = ColumnKind.Utf16CodeUnits,
LogicalLocations = this.LogicalLocations,
};

output.Initialize(run);

if (fileDictionary != null && fileDictionary.Any())
{
output.WriteFiles(fileDictionary);
}

if (LogicalLocations != null && LogicalLocations.Any())
{
output.WriteLogicalLocations(LogicalLocations);
}

output.OpenResults();
output.WriteResults(results);
output.CloseResults();
PersistResults(output, results, run);
}

/// <summary>Processes an Android Studio log and writes issues therein to an instance of
Expand All @@ -102,9 +80,9 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
/// <returns>
/// A list of the <see cref="Result"/> objects translated from the AndroidStudio format.
/// </returns>
private ISet<Result> ProcessAndroidStudioLog(XmlReader xmlReader)
private IList<Result> ProcessAndroidStudioLog(XmlReader xmlReader)
{
var results = new HashSet<Result>(Result.ValueComparer);
var results = new List<Result>();

int problemsDepth = xmlReader.Depth;
xmlReader.ReadStartElement(_strings.Problems);
Expand Down
25 changes: 2 additions & 23 deletions src/Sarif.Converters/ClangAnalyzerConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
using System.Collections.Generic;
using System.IO;
using System.Xml;
using Microsoft.CodeAnalysis.Sarif.Writers;

namespace Microsoft.CodeAnalysis.Sarif.Converters
{
Expand Down Expand Up @@ -58,27 +57,7 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
}
}

var tool = new Tool
{
Name = "Clang"
};

var fileInfoFactory = new FileInfoFactory(MimeType.DetermineFromFileExtension, dataToInsert);
Dictionary<string, FileData> fileDictionary = fileInfoFactory.Create(results);


var run = new Run()
{
Tool = tool
};

output.Initialize(run);

if (fileDictionary != null && fileDictionary.Count > 0) { output.WriteFiles(fileDictionary); }

output.OpenResults();
output.WriteResults(results);
output.CloseResults();
PersistResults(output, results, "Clang Analyzer");
}
finally
{
Expand Down Expand Up @@ -155,7 +134,7 @@ private Result CreateResult(IDictionary<string, object> issueData)
int fileNumber = FindInt(location, "file");
if (_files != null && fileNumber < _files.Count)
{
fileName = _files[fileNumber] as string;
fileName = (string)_files[fileNumber];
}
}

Expand Down
21 changes: 2 additions & 19 deletions src/Sarif.Converters/CppCheckConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
using System.IO;
using System.Xml;

using Microsoft.CodeAnalysis.Sarif.Writers;

namespace Microsoft.CodeAnalysis.Sarif.Converters
{
public class CppCheckConverter : ToolFileConverterBase
Expand Down Expand Up @@ -114,27 +112,12 @@ private void ProcessCppCheckLog(XmlReader reader, IResultLogWriter output, Optio

reader.ReadEndElement(); // </results>

var tool = new Tool
{
Name = "CppCheck",
Version = version,
};

var fileInfoFactory = new FileInfoFactory(uri => MimeType.Cpp, dataToInsert);
Dictionary<string, FileData> fileDictionary = fileInfoFactory.Create(results);

var run = new Run()
{
Tool = tool
Tool = new Tool { Name = "CppCheck", Version = version }
};

output.Initialize(run);

if (fileDictionary != null && fileDictionary.Count > 0) { output.WriteFiles(fileDictionary); }

output.OpenResults();
output.WriteResults(results);
output.CloseResults();
PersistResults(output, results, run);
}
}
}
19 changes: 2 additions & 17 deletions src/Sarif.Converters/FortifyConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Xml;
using Microsoft.CodeAnalysis.Sarif.Writers;

namespace Microsoft.CodeAnalysis.Sarif.Converters
{
Expand Down Expand Up @@ -85,14 +84,6 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
}
}

var tool = new Tool
{
Name = "Fortify"
};

var fileInfoFactory = new FileInfoFactory(MimeType.DetermineFromFileExtension, dataToInsert);
Dictionary<string, FileData> fileDictionary = fileInfoFactory.Create(results);

var run = new Run()
{
Id = new RunAutomationDetails
Expand All @@ -102,16 +93,10 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
Text = runDescription
}
},
Tool = tool
Tool = new Tool { Name = "Fortify" }
};

output.Initialize(run);

if (fileDictionary != null && fileDictionary.Count > 0) { output.WriteFiles(fileDictionary); }

output.OpenResults();
output.WriteResults(results);
output.CloseResults();
PersistResults(output, results, run);
}

/// <summary>Converts a Fortify result to a static analysis results interchange format result.</summary>
Expand Down
38 changes: 8 additions & 30 deletions src/Sarif.Converters/FortifyFprConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,8 @@ internal class FortifyFprConverter : ToolFileConverterBase
private string _automationId;
private string _originalUriBasePath;
private List<Result> _results = new List<Result>();
private List<Notification> _toolNotifications;
private Dictionary<string, FileData> _fileDictionary;
private Dictionary<string, IRule> _ruleDictionary;
private Dictionary<string, Rule> _ruleDictionary;
private Dictionary<ThreadFlowLocation, string> _tflToNodeIdDictionary;
private Dictionary<ThreadFlowLocation, string> _tflToSnippetIdDictionary;
private Dictionary<Location, string> _locationToSnippetIdDictionary;
Expand All @@ -50,9 +49,8 @@ public FortifyFprConverter()
_strings = new FortifyFprStrings(_nameTable);

_results = new List<Result>();
_toolNotifications = new List<Notification>();
_fileDictionary = new Dictionary<string, FileData>();
_ruleDictionary = new Dictionary<string, IRule>();
_ruleDictionary = new Dictionary<string, Rule>();
_tflToNodeIdDictionary = new Dictionary<ThreadFlowLocation, string>();
_tflToSnippetIdDictionary = new Dictionary<ThreadFlowLocation, string>();
_locationToSnippetIdDictionary = new Dictionary<Location, string>();
Expand Down Expand Up @@ -89,8 +87,8 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
};

_invocation = new Invocation();
_invocation.ToolNotifications = new List<Notification>();
_results.Clear();
_toolNotifications.Clear();
_fileDictionary.Clear();
_ruleDictionary.Clear();
_tflToNodeIdDictionary.Clear();
Expand All @@ -116,7 +114,8 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
InstanceId = _automationId + "/"
},
Tool = tool,
Invocations = new[] { _invocation }
Invocations = new[] { _invocation },
Resources = new Resources { Rules = _ruleDictionary}
};

if (!string.IsNullOrWhiteSpace(_originalUriBasePath))
Expand All @@ -128,28 +127,7 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
};
}

output.Initialize(run);

(output as ResultLogJsonWriter).WriteInvocations(run.Invocations);

if (_fileDictionary.Any())
{
output.WriteFiles(_fileDictionary);
}

output.OpenResults();
output.WriteResults(_results);
output.CloseResults();

if (_ruleDictionary.Any())
{
output.WriteRules(_ruleDictionary);
}

if (_toolNotifications.Any())
{
output.WriteToolNotifications(_toolNotifications);
}
PersistResults(output, _results, run);
}

private void ParseFprFile(Stream input)
Expand Down Expand Up @@ -871,7 +849,7 @@ private void ParseErrors()
string errorCode = _reader.GetAttribute(_strings.CodeAttribute);
string message = _reader.ReadElementContentAsString();

_toolNotifications.Add(new Notification
_invocation.ToolNotifications.Add(new Notification
{
Id = errorCode,
Level = NotificationLevel.Error,
Expand Down Expand Up @@ -930,7 +908,7 @@ private void AddMessagesToResults()
{
foreach (Result result in _results)
{
IRule rule;
Rule rule;
if (_ruleDictionary.TryGetValue(result.RuleId, out rule))
{
Message message = rule.ShortDescription ?? rule.FullDescription;
Expand Down
36 changes: 8 additions & 28 deletions src/Sarif.Converters/FxCopConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
using System.Reflection;
using System.Xml;
using System.Xml.Schema;
using Microsoft.CodeAnalysis.Sarif.Writers;

namespace Microsoft.CodeAnalysis.Sarif.Converters
{
Expand Down Expand Up @@ -53,46 +52,27 @@ public override void Convert(Stream input, IResultLogWriter output, OptionallyEm
reader.ResultRead += (FxCopLogReader.Context current) => { results.Add(CreateResult(current)); };
reader.Read(context, input);

Tool tool = new Tool
{
Name = "FxCop"
};

var fileInfoFactory = new FileInfoFactory(MimeType.DetermineFromFileExtension, dataToInsert);
Dictionary<string, FileData> fileDictionary = fileInfoFactory.Create(results);

var run = new Run()
{
Tool = tool
Tool = new Tool { Name = "FxCop"}
};

output.Initialize(run);

if (fileDictionary != null && fileDictionary.Any())
{
output.WriteFiles(fileDictionary);
}

if (LogicalLocations != null && LogicalLocations.Any())
{
output.WriteLogicalLocations(LogicalLocations);
}

output.OpenResults();
output.WriteResults(results);
output.CloseResults();

if (rules.Count > 0)
{
IDictionary<string, IRule> rulesDictionary = new Dictionary<string, IRule>();
IDictionary<string, Rule> rulesDictionary = new Dictionary<string, Rule>();

foreach (Rule rule in rules)
{
rulesDictionary[rule.Id] = rule;
}

output.WriteRules(rulesDictionary);
run.Resources = new Resources
{
Rules = rulesDictionary
};
}

PersistResults(output, results, run);
}

internal Rule CreateRule(FxCopLogReader.Context context)
Expand Down
Loading

0 comments on commit 621ec57

Please sign in to comment.