Skip to content

Commit

Permalink
Add support for Unity NativeArray as vector type (#319)
Browse files Browse the repository at this point in the history
* Add support for Unity NativeArray vector type.

* Ensure NativeArray serialization occurs only on aligned types.
  • Loading branch information
joncham authored Jan 6, 2023
1 parent 2795a75 commit 72a0ecf
Show file tree
Hide file tree
Showing 23 changed files with 725 additions and 17 deletions.
3 changes: 3 additions & 0 deletions src/FlatSharp.Compiler/CompilerOptions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,4 +43,7 @@ public record CompilerOptions

[Option("debug", Hidden = true, Default = false)]
public bool Debug { get; set; }

[Option("unity-assembly-path", HelpText = "Path to assembly (e.g. UnityEngine.dll) which enables Unity support.")]
public string? UnityAssemblyPath { get; set; }
}
28 changes: 24 additions & 4 deletions src/FlatSharp.Compiler/FlatSharpCompiler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ private static int RunCompiler(CompilerOptions options)
Console.Error.WriteLine("FlatSharp compiler: No output directory specified.");
return -1;
}

if (!string.IsNullOrEmpty(options.UnityAssemblyPath) && !File.Exists(options.UnityAssemblyPath))
{
Console.Error.WriteLine("FlatSharp compiler: Unity assembly path specified does not exist.");
return -1;
}

// Read existing file to see if we even need to do any work.
const string outputFileName = "FlatSharp.generated.cs";
Expand Down Expand Up @@ -315,7 +321,7 @@ private static (Assembly, string) CompileAndLoadAssemblyWithCode(
{
try
{
Assembly[] additionalRefs = additionalReferences?.ToArray() ?? Array.Empty<Assembly>();
Assembly[] additionalRefs = BuildAdditionalReferences(options, additionalReferences);

options.InputFiles = fbsFiles.Select(x => x.FullName);

Expand Down Expand Up @@ -527,6 +533,8 @@ private static void CreateCSharp(
}

ErrorContext.Current.ThrowIfHasErrors();

Assembly[] additionalRefs = BuildAdditionalReferences(options);

foreach (var step in steps)
{
Expand All @@ -539,7 +547,7 @@ private static void CreateCSharp(
if (step > CodeWritingPass.Initialization)
{
csharp = writer.ToString();
(assembly, _, _) = RoslynSerializerGenerator.CompileAssembly(csharp, true);
(assembly, _, _) = RoslynSerializerGenerator.CompileAssembly(csharp, true, additionalRefs);
}

writer = new CodeWriter();
Expand All @@ -552,7 +560,7 @@ private static void CreateCSharp(
Options = localOptions,
InputHash = inputHash,
PreviousAssembly = assembly,
TypeModelContainer = TypeModelContainer.CreateDefault(),
TypeModelContainer = TypeModelContainer.CreateDefault().WithUnitySupport(localOptions.UnityAssemblyPath is not null),
});

ErrorContext.Current.ThrowIfHasErrors();
Expand Down Expand Up @@ -580,7 +588,8 @@ private static Schema.Schema ParseSchema(
List<Func<string, string>> postProcessTransforms,
params ISchemaMutator[] mutators)
{
ISerializer<Schema.Schema> mutableSerializer = FlatBufferSerializer.Default
ISerializer<Schema.Schema> mutableSerializer = new FlatBufferSerializer(
new FlatBufferSerializerOptions(), TypeModelContainer.CreateDefault().WithUnitySupport(options.UnityAssemblyPath is not null))
.Compile<Schema.Schema>()
.WithSettings(s => s.UseGreedyMutableDeserialization());

Expand All @@ -599,4 +608,15 @@ private static Schema.Schema ParseSchema(
// Immutable.
return mutableSerializer.WithSettings(s => s.UseGreedyDeserialization()).Parse(temp);
}

private static Assembly[] BuildAdditionalReferences(CompilerOptions options, IEnumerable<Assembly>? additionalReferences = null)
{
var references = new List<Assembly>();
if (additionalReferences is not null)
references.AddRange(additionalReferences);
if (options.UnityAssemblyPath is not null)
references.Add(Assembly.LoadFrom(options.UnityAssemblyPath));

return references.ToArray();
}
}
1 change: 1 addition & 0 deletions src/FlatSharp.Compiler/SchemaModel/PropertyFieldModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,7 @@ private string GetVectorTypeName(string innerType, string? keyType)
VectorType.Memory => $"Memory<{innerType}>",
VectorType.ReadOnlyMemory => $"ReadOnlyMemory<{innerType}>",
VectorType.IIndexedVector => $"IIndexedVector<{keyType ?? "string"}, {innerType}>",
VectorType.UnityNativeArray => $"Unity.Collections.NativeArray<{innerType}>",

// Unqualified ubyte vectors default to Memory.
null => this.Field.Type.ElementType == BaseType.UByte
Expand Down
1 change: 1 addition & 0 deletions src/FlatSharp.Compiler/VectorType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,4 +26,5 @@ public enum VectorType
Memory,
ReadOnlyMemory,
IIndexedVector,
UnityNativeArray
}
18 changes: 18 additions & 0 deletions src/FlatSharp.Runtime/IO/InputBufferExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/

using System.IO;
using System.Runtime.InteropServices;

namespace FlatSharp.Internal;

Expand Down Expand Up @@ -136,6 +137,23 @@ public static ReadOnlyMemory<byte> ReadByteReadOnlyMemoryBlock<TBuffer>(this TBu
return buffer.GetReadOnlyMemory().Slice(uoffset + sizeof(uint), (int)buffer.ReadUInt(uoffset));
}
}

public static Span<TElement> UnsafeReadSpan<TBuffer, TElement>(this TBuffer buffer, int uoffset) where TBuffer : IInputBuffer where TElement : struct
{
checked
{
// The local value stores a uoffset_t, so follow that now.
uoffset = uoffset + buffer.ReadUOffset(uoffset);

// We need to construct a Span<TElement> from byte buffer that:
// 1. starts at correct offset for vector data
// 2. has a length based on *TElement* count not *byte* count
var byteSpanAtDataOffset = buffer.GetSpan().Slice(uoffset + sizeof(uint));
var sourceSpan = MemoryMarshal.Cast<byte, TElement>(byteSpanAtDataOffset).Slice(0, (int)buffer.ReadUInt(uoffset));

return sourceSpan;
}
}

[ExcludeFromCodeCoverage] // Not currently used.
[Conditional("DEBUG")]
Expand Down
29 changes: 29 additions & 0 deletions src/FlatSharp.Runtime/IO/SpanWriterExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
* limitations under the License.
*/

using System.Runtime.InteropServices;

namespace FlatSharp.Internal;

/// <summary>
Expand All @@ -36,6 +38,33 @@ public static void WriteReadOnlyByteMemoryBlock<TSpanWriter>(

memory.Span.CopyTo(span.Slice(vectorStartOffset + sizeof(uint)));
}

public static void UnsafeWriteSpan<TSpanWriter, TElement>(
this TSpanWriter spanWriter,
Span<byte> span,
Span<TElement> buffer,
int offset,
int alignment,
SerializationContext ctx) where TSpanWriter : ISpanWriter where TElement : unmanaged
{
var size = Unsafe.SizeOf<TElement>();

if (size % alignment != 0)
throw new InvalidOperationException($"Unsafe Span serialization does not support types with size {size} that is not a multiple of alignment {alignment}.");

int numberOfItems = buffer.Length;
int vectorStartOffset = ctx.AllocateVector(
itemAlignment: alignment,
numberOfItems,
sizePerItem: size);

spanWriter.WriteUOffset(span, offset, vectorStartOffset);
spanWriter.WriteInt(span, numberOfItems, vectorStartOffset);


var start = span.Slice(vectorStartOffset + sizeof(uint));
MemoryMarshal.Cast<TElement, byte>(buffer).CopyTo(start);
}

/// <summary>
/// Writes the given string.
Expand Down
25 changes: 25 additions & 0 deletions src/FlatSharp.UnityPolyfills/FlatSharp.UnityPolyfills.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<Project Sdk="Microsoft.NET.Sdk">
<Import Project="..\common.props" />

<PropertyGroup>
<TargetFrameworks>netstandard2.0</TargetFrameworks>
<RootNamespace>FlatSharp.UnityPolyfills</RootNamespace>
<Description>FlatSharp.UnityPolyfills is a utility assembly to expose the Unity API needed for FlatSharp support of UnityEngine.Collections.NativeArray.</Description>
<Nullable>annotations</Nullable>
<DefineConstants>$(DefineContants);FLATSHARP_UNITY_POLYFILLS</DefineConstants>
<PackageId>FlatSharp.UnityPolyfills</PackageId>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

<PropertyGroup Condition=" '$(TargetFramework)' == 'net6.0' ">
<Nullable>enable</Nullable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="System.Memory" Version="4.5.5" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\FlatSharp.Runtime\FlatSharp.Runtime.csproj" />
</ItemGroup>
</Project>
166 changes: 166 additions & 0 deletions src/FlatSharp.UnityPolyfills/NativeArray.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
/*
* Copyright 2022 Unity Technologies
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

using System;
using System.Collections;
using System.Collections.Generic;
using System.Runtime.CompilerServices;

namespace Unity.Collections
{
public enum NativeArrayOptions
{
UninitializedMemory = 0,
ClearMemory = 1
}

public enum Allocator
{
Invalid = 0,
None = 1,
Temp = 2,
TempJob = 3,
Persistent = 4,
}

// for testing, we remove the 'where T : struct' constraint and enforce this in UnityNativeArrayVectorTypeModel
public unsafe struct NativeArray<T> : IEnumerable<T>, IEquatable<NativeArray<T>> /* where T : struct */
{
internal Memory<T> m_Buffer;

internal void* m_Data;
internal int m_Length;

public NativeArray(int length, Allocator allocator, NativeArrayOptions options = NativeArrayOptions.ClearMemory)
{
Allocate(length, allocator, out this);
}

public NativeArray(T[] array, Allocator allocator)
{
Allocate(array.Length, allocator, out this);
array.AsSpan().CopyTo(m_Buffer.Span);
}

public NativeArray(NativeArray<T> array, Allocator allocator)
{
Allocate(array.Length, allocator, out this);
array.AsSpan().CopyTo(this.AsSpan());
}

static void Allocate(int length, Allocator allocator, out NativeArray<T> nativeArray)
{
var backing = new T[length];
nativeArray = new NativeArray<T>()
{
m_Buffer = new Memory<T>(backing)
};
}

struct Enumerator : IEnumerator<T>
{
private readonly NativeArray<T> m_Array;
private int m_Index;

public Enumerator(NativeArray<T> nativeArray)
{
m_Array = nativeArray;
m_Index = -1;
}

public void Dispose()
{
}

public bool MoveNext()
{
m_Index++;
return m_Index < m_Array.Length;
}

public void Reset()
{
m_Index = -1;
}

// Let NativeArray indexer check for out of range.
public T Current => m_Array[m_Index];

object IEnumerator.Current => Current;
}

public IEnumerator<T> GetEnumerator()
{
return new Enumerator(this);
}

IEnumerator IEnumerable.GetEnumerator()
{
return GetEnumerator();
}

public bool Equals(NativeArray<T> other)
{
throw new NotImplementedException();
}

public int Length => m_Data != null ? m_Length : m_Buffer.Length;

public T this[int index]
{
get => AsSpan()[index];

set => AsSpan()[index] = value;
}

public readonly Span<T> AsSpan()
{
return m_Data != null ? new Span<T>(m_Data, m_Length) : m_Buffer.Span;
}
}
}

namespace Unity.Collections.LowLevel.Unsafe
{
public static unsafe class NativeArrayUnsafeUtility
{
public static NativeArray<T> ConvertExistingDataToNativeArray<T>(void* dataPointer, int length, Allocator allocator) where T : struct
{
var newArray = new NativeArray<T>
{
m_Data = dataPointer,
m_Length = length,
};

return newArray;
}
}


public static unsafe class NativeArrayUnsafeUtilityEx
{
public static NativeArray<T> ConvertExistingDataToNativeArray<T>(Span<T> span, Allocator allocator) where T : struct
{
var newArray = new NativeArray<T>
{
m_Data = System.Runtime.CompilerServices.Unsafe.AsPointer(ref span.GetPinnableReference()),
m_Length = span.Length,
};

return newArray;
}
}
}
6 changes: 6 additions & 0 deletions src/FlatSharp.sln
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microbench.6.3.3", "Benchma
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FlatSharpPoolableEndToEndTests", "Tests\FlatSharpPoolableEndToEndTests\FlatSharpPoolableEndToEndTests.csproj", "{7E55A248-4BBD-48AD-B27D-A7E0E705DC89}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FlatSharp.UnityPolyfills", "FlatSharp.UnityPolyfills\FlatSharp.UnityPolyfills.csproj", "{E439E4AF-B948-4E6C-8791-1830619EB35B}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -93,6 +95,10 @@ Global
{7E55A248-4BBD-48AD-B27D-A7E0E705DC89}.Debug|Any CPU.Build.0 = Debug|Any CPU
{7E55A248-4BBD-48AD-B27D-A7E0E705DC89}.Release|Any CPU.ActiveCfg = Release|Any CPU
{7E55A248-4BBD-48AD-B27D-A7E0E705DC89}.Release|Any CPU.Build.0 = Release|Any CPU
{E439E4AF-B948-4E6C-8791-1830619EB35B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E439E4AF-B948-4E6C-8791-1830619EB35B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E439E4AF-B948-4E6C-8791-1830619EB35B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E439E4AF-B948-4E6C-8791-1830619EB35B}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
3 changes: 3 additions & 0 deletions src/FlatSharp/FlatBufferSerializer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ namespace FlatSharp;
public sealed class FlatBufferSerializer
{
public static FlatBufferSerializer Default { get; } = new FlatBufferSerializer(new FlatBufferSerializerOptions());

// Test hook
internal static FlatBufferSerializer DefaultWithUnitySupport { get; } = new FlatBufferSerializer(new FlatBufferSerializerOptions(), TypeModelContainer.CreateDefault().WithUnitySupport(true));

private readonly Dictionary<Type, object> serializerCache = new();

Expand Down
Loading

0 comments on commit 72a0ecf

Please sign in to comment.