Skip to content

Commit

Permalink
FlatSharp compiler enhancements (#389)
Browse files Browse the repository at this point in the history 
* FlatSharp compiler enhancements

* Fix build break

* Ensure temp directory is on the same drive as the compiler

* Basic Debugger Display stuff.
  • Loading branch information
@jamescourtney
jamescourtney authored Aug 3, 2023
1 parent 4456849 commit 2c53aa0
Show file tree
Hide file tree
Showing 21 changed files with 304 additions and 122 deletions.
8 changes: 7 additions & 1 deletion src/FlatSharp.Compiler/CompilerOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ public record CompilerOptions
public bool? NullableWarnings { get; set; }

[Option("gen-poolable", Hidden = false, Default = false, HelpText = "EXPERIMENTAL: Generate extra code to enable object pooling for allocation reductions.")]
public bool? GeneratePoolableObjects { get; set; }
public bool GeneratePoolableObjects { get; set; }

[Option("deserializers", Hidden = false, HelpText = "Specifies deserializers for FlatSharp to generate. Can help to reduce size of generated code.", Separator = ';')]
public IList<FlatBufferDeserializationOption> Deserializers
Expand All @@ -59,6 +59,12 @@ public IList<FlatBufferDeserializationOption> Deserializers
}
}

[Option("class-definitions-only", Hidden = false, HelpText = "Emits only class and data definitions. No serializers.")]
public bool ClassDefinitionsOnly { get; set; }

[Option("input-files-only", Hidden = false, HelpText = "Only outputs type definitions for expicitely passed input files. Does not process any included files.")]
public bool SpecifiedFilesOnly { get; set; }

[Option("unity-assembly-path", HelpText = "Path to assembly (e.g. UnityEngine.dll) which enables Unity support.")]
public string? UnityAssemblyPath { get; set; }

Expand Down
50 changes: 27 additions & 23 deletions src/FlatSharp.Compiler/FlatSharp.Compiler.targets
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,46 +90,50 @@
</ItemGroup>

<Target Name="FlatSharpFbsCompile" BeforeTargets="ResolveAssemblyReferences" Condition=" '@(FlatSharpSchema)' != '' ">
<ProcessFlatSharpSchema Inputs="@(FlatSharpSchema)">
<Output TaskParameter="IncludeDirectories" PropertyName="Includes" />
</ProcessFlatSharpSchema>

<!-- find compiler and set base command -->
<PropertyGroup>
<CompilerPath>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)\..\tools\net6.0\FlatSharp.Compiler.dll'))</CompilerPath>
<CompilerPath Condition=" '$(FlatSharpCompilerPath)' != '' ">$(FlatSharpCompilerPath)</CompilerPath>
<CompilerCommand>dotnet &quot;$(CompilerPath)&quot; --input &quot;@(FlatSharpSchema)&quot; --includes &quot;$(Includes)&quot; --output $(IntermediateOutputPath)</CompilerCommand>
</PropertyGroup>

<!-- Initialize FlatSharpNullable if not set -->
<PropertyGroup Condition=" '$(FlatSharpNullable)' == '' ">
<FlatSharpNullable>false</FlatSharpNullable>
<FlatSharpNullable Condition=" '$(Nullable)' == 'enable' ">true</FlatSharpNullable>
</PropertyGroup>

<PropertyGroup Condition=" '$(FlatSharpPoolable)' == '' ">
<FlatSharpPoolable>false</FlatSharpPoolable>
<!-- Append nullability to command line -->
<PropertyGroup>
<CompilerCommand>$(CompilerCommand) --nullable-warnings $(FlatSharpNullable)</CompilerCommand>
</PropertyGroup>

<PropertyGroup Condition=" '$(FlatSharpDeserializers)' != '' ">
<FlatSharpDeserializers>--deserializers &quot;$(FlatSharpDeserializers)&quot;</FlatSharpDeserializers>
<PropertyGroup Condition=" '$(FlatSharpPoolable)' == 'true' ">
<CompilerCommand>$(CompilerCommand) --gen-poolable</CompilerCommand>
</PropertyGroup>

<PropertyGroup Condition=" '$(FlatSharpNameNormalization)' == '' ">
<FlatSharpNameNormalization>true</FlatSharpNameNormalization>
<PropertyGroup Condition=" '$(FlatSharpDeserializers)' != '' ">
<CompilerCommand>$(CompilerCommand) --deserializers &quot;$(FlatSharpDeserializers)&quot;</CompilerCommand>
</PropertyGroup>

<PropertyGroup>
<CompilerPath>$([System.IO.Path]::GetFullPath('$(MSBuildThisFileDirectory)\..\tools\net6.0\FlatSharp.Compiler.dll'))</CompilerPath>
<PropertyGroup Condition=" '$(FlatSharpNameNormalization)' != '' ">
<CompilerCommand>$(CompilerCommand) --normalize-field-names $(FlatSharpNameNormalization)</CompilerCommand>
</PropertyGroup>

<PropertyGroup Condition=" '$(FlatSharpCompilerPath)' != '' ">
<CompilerPath>$(FlatSharpCompilerPath)</CompilerPath>
<PropertyGroup Condition=" '$(FlatSharpClassDefinitionsOnly)' == 'true' ">
<CompilerCommand>$(CompilerCommand) --class-definitions-only</CompilerCommand>
</PropertyGroup>

<ProcessFlatSharpSchema Inputs="@(FlatSharpSchema)">
<Output TaskParameter="IncludeDirectories" PropertyName="Includes" />
</ProcessFlatSharpSchema>

<PropertyGroup>
<CompilerCommand>dotnet &quot;$(CompilerPath)&quot; --nullable-warnings $(FlatSharpNullable) --normalize-field-names $(FlatSharpNameNormalization) --gen-poolable $(FlatSharpPoolable) --input &quot;@(FlatSharpSchema)&quot; --includes &quot;$(Includes)&quot; $(FlatSharpDeserializers) --output $(IntermediateOutputPath)</CompilerCommand>
<PropertyGroup Condition=" '$(FlatSharpInputFilesOnly)' == 'true' ">
<CompilerCommand>$(CompilerCommand) --input-files-only</CompilerCommand>
</PropertyGroup>

<Message
Text="$(CompilerCommand)"
Importance="high" />

<Exec
Command="$(CompilerCommand)"
CustomErrorRegularExpression=".*" />
<Message Text="$(CompilerCommand)" Importance="high" />
<Exec Command="$(CompilerCommand)" CustomErrorRegularExpression=".*" />

<ItemGroup>
<GeneratedFbs Include="$(IntermediateOutputPath)*.generated.cs" />
Expand Down
173 changes: 93 additions & 80 deletions src/FlatSharp.Compiler/FlatSharpCompiler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public class FlatSharpCompiler

internal static CompilerOptions? CommandLineOptions { get; private set; }

private static readonly string RootDirectory = Path.GetDirectoryName(typeof(FlatSharpCompiler).Assembly.Location)!;

private static string AssemblyVersion => typeof(ISchemaMutator).Assembly.GetCustomAttribute<AssemblyFileVersionAttribute>()?.Version ?? "unknown";

[ExcludeFromCodeCoverage]
Expand Down Expand Up @@ -108,7 +110,7 @@ private static int RunCompiler(CompilerOptions options)
{
try
{
List<byte[]> bfbs = GetBfbs(options);
var bfbs = GetBfbs(options);

string inputHash = ComputeInputHash(bfbs, options);

Expand Down Expand Up @@ -230,7 +232,7 @@ private static bool IsInputUnchanged(string outputFullPath, string inputHash)
}

[ExcludeFromCodeCoverage]
private static string ComputeInputHash(List<byte[]> bfbs, CompilerOptions options)
private static string ComputeInputHash(List<(byte[] bfbs, string _)> bfbs, CompilerOptions options)
{
static void MergeHashes(byte[] hash, byte[] temp)
{
Expand All @@ -253,7 +255,7 @@ static void MergeHashes(byte[] hash, byte[] temp)
// Merge each of the schema files.
foreach (var schema in bfbs)
{
MergeHashes(hashBytes, hash.ComputeHash(schema));
MergeHashes(hashBytes, hash.ComputeHash(schema.bfbs));
}

inputHash += "." + Convert.ToBase64String(hashBytes);
Expand All @@ -268,7 +270,10 @@ internal static (Assembly, string) CompileAndLoadAssemblyWithCode(
CompilerOptions options,
IEnumerable<Assembly>? additionalReferences = null)
{
string temp = Path.GetTempFileName() + ".fbs";
string tempDirectory = Path.Combine(Path.GetPathRoot(RootDirectory)!, "flatsharptemp", Guid.NewGuid().ToString("n"));
Directory.CreateDirectory(tempDirectory);

string temp = Path.Combine(tempDirectory, "schema.fbs");
File.WriteAllText(temp, fbsSchema);

try
Expand All @@ -278,6 +283,7 @@ internal static (Assembly, string) CompileAndLoadAssemblyWithCode(
finally
{
File.Delete(temp);
Directory.Delete(tempDirectory, true);
}
}

Expand Down Expand Up @@ -349,7 +355,7 @@ private static (Assembly, string) CompileAndLoadAssemblyWithCode(

options.InputFiles = fbsFiles.Select(x => x.FullName);

List<byte[]> bfbs = GetBfbs(options);
List<(byte[], string)> bfbs = GetBfbs(options);
string cSharp = CreateCSharp(bfbs, "hash", options);

var (assembly, formattedText, _) = RoslynSerializerGenerator.CompileAssembly(cSharp, true, additionalRefs);
Expand All @@ -362,7 +368,7 @@ private static (Assembly, string) CompileAndLoadAssemblyWithCode(
}
}

private static List<byte[]> GetBfbs(CompilerOptions options)
private static List<(byte[] bfbs, string fbsPath)> GetBfbs(CompilerOptions options)
{
string flatcPath;

Expand All @@ -378,107 +384,107 @@ private static List<byte[]> GetBfbs(CompilerOptions options)
flatcPath = options.FlatcPath;
}

List<(byte[] bfbs, string fbsPath)> results = new();
string temp = Path.GetTempPath();
string dirName = $"flatsharpcompiler_temp_{Guid.NewGuid():n}";
string outputDir = Path.Combine(temp, dirName);

Directory.CreateDirectory(outputDir);

using var p = new Process
try
{
StartInfo = new ProcessStartInfo
foreach (var inputFile in options.InputFiles)
{
CreateNoWindow = true,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
FileName = flatcPath,
}
};
string inputFullPath = PathHelpers.NormalizePathName(inputFile);

var args = new List<string>
{
"-b",
"--schema",
"--bfbs-comments",
"--bfbs-builtins",
"--bfbs-filenames",
Path.GetDirectoryName(typeof(ISchemaMutator).Assembly.Location!)!,
"--no-warnings",
"-o",
outputDir,
};
using var p = new Process
{
StartInfo = new ProcessStartInfo
{
CreateNoWindow = true,
RedirectStandardError = true,
RedirectStandardOutput = true,
UseShellExecute = false,
FileName = flatcPath,
}
};

if (!string.IsNullOrEmpty(options.IncludesDirectory))
{
// One or more includes directory has been specified
foreach (var includePath in options.IncludesDirectory.Split(';', StringSplitOptions.RemoveEmptyEntries))
{
args.AddRange(new[]
var args = new List<string>
{
"-I",
new DirectoryInfo(includePath).FullName,
});
}
}
"-b",
"--schema",
"--bfbs-comments",
"--bfbs-builtins",
"--bfbs-filenames",
RootDirectory,
"--no-warnings",
"-o",
outputDir,
};

if (!string.IsNullOrEmpty(options.IncludesDirectory))
{
// One or more includes directory has been specified
foreach (var includePath in options.IncludesDirectory.Split(';', StringSplitOptions.RemoveEmptyEntries))
{
args.AddRange(new[]
{
"-I",
new DirectoryInfo(includePath).FullName,
});
}
}

foreach (var item in options.InputFiles)
{
args.Add(new FileInfo(item).FullName);
}
foreach (var arg in args)
{
p.StartInfo.ArgumentList.Add(arg);
}

foreach (var arg in args)
{
p.StartInfo.ArgumentList.Add(arg);
}
p.StartInfo.ArgumentList.Add(inputFullPath);

try
{
p.EnableRaisingEvents = true;
p.EnableRaisingEvents = true;

var lines = new List<string>();
var lines = new List<string>();

void OnDataReceived(object sender, DataReceivedEventArgs args)
{
if (!string.IsNullOrEmpty(args.Data))
void OnDataReceived(object sender, DataReceivedEventArgs args)
{
lines.Add(args.Data);
if (!string.IsNullOrEmpty(args.Data))
{
lines.Add(args.Data);
}
}
}

p.OutputDataReceived += OnDataReceived;
p.ErrorDataReceived += OnDataReceived;
p.OutputDataReceived += OnDataReceived;
p.ErrorDataReceived += OnDataReceived;

p.Start();
p.BeginOutputReadLine();
p.BeginErrorReadLine();
p.WaitForExit();
p.Start();
p.BeginOutputReadLine();
p.BeginErrorReadLine();
p.WaitForExit();

if (p.ExitCode == 0)
{
List<byte[]> bfbs = new();
foreach (string output in Directory.GetFiles(outputDir, "*.bfbs"))
if (p.ExitCode == 0)
{
bfbs.Add(File.ReadAllBytes(output));
results.Add((
File.ReadAllBytes(Path.Combine(outputDir, Path.GetFileNameWithoutExtension(inputFullPath) + ".bfbs")),
inputFullPath));
}

return bfbs;
}
else
{
foreach (var line in lines)
else
{
ErrorContext.Current.RegisterError(line);
}
foreach (var line in lines)
{
ErrorContext.Current.RegisterError(line);
}

ErrorContext.Current.ThrowIfHasErrors();
throw new InvalidFbsFileException("Unknown error when invoking flatc. Process exited with error, but didn't write any errors.");
ErrorContext.Current.ThrowIfHasErrors();
throw new InvalidFbsFileException("Unknown error when invoking flatc. Process exited with error, but didn't write any errors.");
}
}
}
finally
{
Directory.Delete(outputDir, recursive: true);
}

return results;
}

[ExcludeFromCodeCoverage]
Expand Down Expand Up @@ -519,7 +525,7 @@ private static (string os, string name) GetFlatcPath()
}

private static string CreateCSharp(
List<byte[]> bfbs,
List<(byte[] bfbs, string inputPath)> bfbs,
string inputHash,
CompilerOptions options)
{
Expand Down Expand Up @@ -548,14 +554,15 @@ private static string CreateCSharp(
{
new FieldNameNormalizerSchemaMutator(),
new ExternalTypeSchemaMutator(),
new AbsolutePathSchemaMutator(RootDirectory),
};

Stopwatch sw = Stopwatch.StartNew();
ISerializer<Schema.Schema> mutableSerializer = Instrument("CompileReflectionFbs", options, FlatBufferSerializer.Default.Compile<Schema.Schema>);

foreach (var s in bfbs)
foreach ((byte[] s, string fbsPath) in bfbs)
{
rootModel.UnionWith(ParseSchema(mutableSerializer, s, options, postProcessTransforms, mutators).ToRootModel(options));
rootModel.UnionWith(ParseSchema(mutableSerializer, s, options, postProcessTransforms, mutators).ToRootModel(options, fbsPath));
}

ErrorContext.Current.ThrowIfHasErrors();
Expand Down Expand Up @@ -666,10 +673,16 @@ private static T Instrument<T>(string step, CompilerOptions opts, Func<T> callba
private static Assembly[] BuildAdditionalReferences(CompilerOptions options, IEnumerable<Assembly>? additionalReferences = null)
{
var references = new List<Assembly>();

if (additionalReferences is not null)
{
references.AddRange(additionalReferences);
}

if (options.UnityAssemblyPath is not null)
{
references.Add(Assembly.LoadFrom(options.UnityAssemblyPath));
}

return references.ToArray();
}
Expand Down
Loading

0 comments on commit 2c53aa0

Please sign in to comment.