Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks, monodroid] Marshal Method Classifier (#…
Browse files Browse the repository at this point in the history
…7123)

Context: e1af958
Context: #7163

Changes: dotnet/java-interop@c942ab6...fadbb82

  * dotnet/java-interop@fadbb82c: [generator] Add support for @explicitInterface metadata (#1006)
  * dotnet/java-interop@3e4a3c4f: [Java.Interop.Tools.JavaCallableWrappers] JavaCallableMethodClassifier (#998)

dotnet/java-interop@3e4a3c4f reworked how
`Java.Interop.Tools.JavaCallableWrappers.JavaCallableWrapperGenerator`
can be used to find JNI marshal methods.

Update `Xamarin.Android.Build.Tasks.dll` to provide a
`JavaCallableMethodClassifier` to `JavaCallableWrapperGenerator`,
collecting Cecil `MethodDefinition` instances for JNI marshal methods
which can be created at build time via LLVM Marshal Methods.

Methods which cannot be created at build time include methods with
`[Export]`.

Once candidate LLVM marshal methods are found, the marshal method is
updated to have the [`UnmanagedCallersOnlyAttribute` attribute][0],
and related System.Reflection.Emit-based infrastructure such as the
`Get…Handler()` methods and `cb_…` fields are removed.

As with e1af958, this feature is *not* enabled by default, and
remains a xamarin-android Build time configuration option.

[0]:https://docs.microsoft.com/dotnet/api/system.runtime.interopservices.unmanagedcallersonlyattribute?view=net-6.0
  • Loading branch information
grendello authored Jul 18, 2022
1 parent 822667f commit 186a9fc
Show file tree
Hide file tree
Showing 11 changed files with 658 additions and 32 deletions.
2 changes: 1 addition & 1 deletion external/Java.Interop
74 changes: 59 additions & 15 deletions src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,12 @@ void Run (DirectoryAssemblyResolver res)
if (Directory.Exists (dir.ItemSpec))
res.SearchDirectories.Add (dir.ItemSpec);
}

#if ENABLE_MARSHAL_METHODS
Dictionary<string, HashSet<string>> marshalMethodsAssemblyPaths = null;
if (!Debug) {
marshalMethodsAssemblyPaths = new Dictionary<string, HashSet<string>> (StringComparer.Ordinal);
}
#endif
// Put every assembly we'll need in the resolver
bool hasExportReference = false;
bool haveMonoAndroid = false;
Expand Down Expand Up @@ -159,6 +164,11 @@ void Run (DirectoryAssemblyResolver res)
}

res.Load (assembly.ItemSpec);
#if ENABLE_MARSHAL_METHODS
if (!Debug) {
StoreMarshalAssemblyPath (Path.GetFileNameWithoutExtension (assembly.ItemSpec), assembly);
}
#endif
}

// However we only want to look for JLO types in user code for Java stub code generation
Expand All @@ -172,6 +182,9 @@ void Run (DirectoryAssemblyResolver res)
string name = Path.GetFileNameWithoutExtension (asm.ItemSpec);
if (!userAssemblies.ContainsKey (name))
userAssemblies.Add (name, asm.ItemSpec);
#if ENABLE_MARSHAL_METHODS
StoreMarshalAssemblyPath (name, asm);
#endif
}

// Step 1 - Find all the JLO types
Expand All @@ -182,10 +195,6 @@ void Run (DirectoryAssemblyResolver res)

List<TypeDefinition> allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies, res);

// Step 2 - Generate type maps
// Type mappings need to use all the assemblies, always.
WriteTypeMappings (allJavaTypes, cache);

var javaTypes = new List<TypeDefinition> ();
foreach (TypeDefinition td in allJavaTypes) {
if (!userAssemblies.ContainsKey (td.Module.Assembly.Name.Name) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (td, cache)) {
Expand All @@ -195,11 +204,27 @@ void Run (DirectoryAssemblyResolver res)
javaTypes.Add (td);
}

// Step 3 - Generate Java stub code
var success = CreateJavaSources (javaTypes, cache);
MarshalMethodsClassifier classifier = null;
#if ENABLE_MARSHAL_METHODS
if (!Debug) {
classifier = new MarshalMethodsClassifier (cache, res, Log);
}
#endif
// Step 2 - Generate Java stub code
var success = CreateJavaSources (javaTypes, cache, classifier);
if (!success)
return;

#if ENABLE_MARSHAL_METHODS
if (!Debug) {
var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, marshalMethodsAssemblyPaths, Log);
rewriter.Rewrite (res);
}
#endif
// Step 3 - Generate type maps
// Type mappings need to use all the assemblies, always.
WriteTypeMappings (allJavaTypes, cache);

// We need to save a map of .NET type -> ACW type for resource file fixups
var managed = new Dictionary<string, TypeDefinition> (javaTypes.Count, StringComparer.Ordinal);
var java = new Dictionary<string, TypeDefinition> (javaTypes.Count, StringComparer.Ordinal);
Expand All @@ -210,7 +235,7 @@ void Run (DirectoryAssemblyResolver res)
using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) {
foreach (TypeDefinition type in javaTypes) {
string managedKey = type.FullName.Replace ('/', '.');
string javaKey = JavaNativeTypeManager.ToJniName (type).Replace ('/', '.');
string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.');

acw_map.Write (type.GetPartialAssemblyQualifiedName (cache));
acw_map.Write (';');
Expand Down Expand Up @@ -332,17 +357,30 @@ void Run (DirectoryAssemblyResolver res)
string applicationTemplateFile = "ApplicationRegistration.java";
SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir,
template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ()));

#if ENABLE_MARSHAL_METHODS
void StoreMarshalAssemblyPath (string name, ITaskItem asm)
{
if (Debug) {
return;
}

if (!marshalMethodsAssemblyPaths.TryGetValue (name, out HashSet<string> assemblyPaths)) {
assemblyPaths = new HashSet<string> ();
marshalMethodsAssemblyPaths.Add (name, assemblyPaths);
}

assemblyPaths.Add (asm.ItemSpec);
}
#endif
}

bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCache cache)
bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier)
{
string outputPath = Path.Combine (OutputDirectory, "src");
string monoInit = GetMonoInitSource (AndroidSdkPlatform);
bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll");
bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10;
#if ENABLE_MARSHAL_METHODS
var overriddenMethodDescriptors = new List<OverriddenMethodDescriptor> ();
#endif

bool ok = true;
foreach (var t in javaTypes) {
Expand All @@ -353,15 +391,19 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac

using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) {
try {
var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache) {
var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache, classifier) {
GenerateOnCreateOverrides = generateOnCreateOverrides,
ApplicationJavaClass = ApplicationJavaClass,
MonoRuntimeInitialization = monoInit,
};

jti.Generate (writer);
#if ENABLE_MARSHAL_METHODS
overriddenMethodDescriptors.AddRange (jti.OverriddenMethodDescriptors);
if (!Debug) {
if (classifier.FoundDynamicallyRegisteredMethods) {
Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName ()}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods.");
}
}
#endif
writer.Flush ();

Expand Down Expand Up @@ -397,7 +439,9 @@ bool CreateJavaSources (IEnumerable<TypeDefinition> javaTypes, TypeDefinitionCac
}
}
#if ENABLE_MARSHAL_METHODS
BuildEngine4.RegisterTaskObjectAssemblyLocal (MarshalMethodsRegisterTaskKey, overriddenMethodDescriptors, RegisteredTaskObjectLifetime.Build);
if (!Debug) {
BuildEngine4.RegisterTaskObjectAssemblyLocal (MarshalMethodsRegisterTaskKey, new MarshalMethodsState (classifier.MarshalMethods), RegisteredTaskObjectLifetime.Build);
}
#endif
return ok;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -442,10 +442,12 @@ void AddEnvironment ()
};
appConfigAsmGen.Init ();
#if ENABLE_MARSHAL_METHODS
var marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator () {
var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<MarshalMethodsState> (GenerateJavaStubs.MarshalMethodsRegisterTaskKey, RegisteredTaskObjectLifetime.Build);

var marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator {
NumberOfAssembliesInApk = assemblyCount,
UniqueAssemblyNames = uniqueAssemblyNames,
OverriddenMethodDescriptors = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal<List<Java.Interop.Tools.JavaCallableWrappers.OverriddenMethodDescriptor>> (GenerateJavaStubs.MarshalMethodsRegisterTaskKey, RegisteredTaskObjectLifetime.Build)
MarshalMethods = marshalMethodsState?.MarshalMethods,
};
marshalMethodsAsmGen.Init ();
#endif
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
#if ENABLE_MARSHAL_METHODS
using System;
using System.Collections.Generic;
using System.IO;

using Java.Interop.Tools.Cecil;
using Microsoft.Android.Build.Tasks;
using Microsoft.Build.Utilities;
using Mono.Cecil;
using Xamarin.Android.Tools;

namespace Xamarin.Android.Tasks
{
class MarshalMethodsAssemblyRewriter
{
IDictionary<string, IList<MarshalMethodEntry>> methods;
ICollection<AssemblyDefinition> uniqueAssemblies;
IDictionary <string, HashSet<string>> assemblyPaths;
TaskLoggingHelper log;

public MarshalMethodsAssemblyRewriter (IDictionary<string, IList<MarshalMethodEntry>> methods, ICollection<AssemblyDefinition> uniqueAssemblies, IDictionary <string, HashSet<string>> assemblyPaths, TaskLoggingHelper log)
{
this.methods = methods ?? throw new ArgumentNullException (nameof (methods));
this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies));
this.assemblyPaths = assemblyPaths ?? throw new ArgumentNullException (nameof (assemblyPaths));
this.log = log ?? throw new ArgumentNullException (nameof (log));
}

public void Rewrite (DirectoryAssemblyResolver resolver)
{
MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver);
var unmanagedCallersOnlyAttributes = new Dictionary<AssemblyDefinition, CustomAttribute> ();
foreach (AssemblyDefinition asm in uniqueAssemblies) {
unmanagedCallersOnlyAttributes.Add (asm, CreateImportedUnmanagedCallersOnlyAttribute (asm, unmanagedCallersOnlyAttributeCtor));
}

Console.WriteLine ("Adding the [UnmanagedCallersOnly] attribute to native callback methods and removing unneeded fields+methods");
foreach (IList<MarshalMethodEntry> methodList in methods.Values) {
foreach (MarshalMethodEntry method in methodList) {
Console.WriteLine ($"\t{method.NativeCallback.FullName} (token: 0x{method.NativeCallback.MetadataToken.RID:x})");
method.NativeCallback.CustomAttributes.Add (unmanagedCallersOnlyAttributes [method.NativeCallback.Module.Assembly]);
method.Connector.DeclaringType.Methods.Remove (method.Connector);
method.CallbackField?.DeclaringType.Fields.Remove (method.CallbackField);
}
}

Console.WriteLine ();
Console.WriteLine ("Rewriting assemblies");

var newAssemblyPaths = new List<string> ();
foreach (AssemblyDefinition asm in uniqueAssemblies) {
foreach (string path in GetAssemblyPaths (asm)) {
var writerParams = new WriterParameters {
WriteSymbols = (File.Exists (path + ".mdb") || File.Exists (Path.ChangeExtension (path, ".pdb"))),
};

string output = $"{path}.new";
Console.WriteLine ($"\t{asm.Name} => {output}");
asm.Write (output, writerParams);
newAssemblyPaths.Add (output);
}
}

// Replace old versions of the assemblies only after we've finished rewriting without issues, otherwise leave the new
// versions around.
foreach (string path in newAssemblyPaths) {
string target = Path.Combine (Path.GetDirectoryName (path), Path.GetFileNameWithoutExtension (path));
MoveFile (path, target);

string source = Path.ChangeExtension (path, ".pdb");
if (File.Exists (source)) {
target = Path.ChangeExtension (Path.Combine (Path.GetDirectoryName (source), Path.GetFileNameWithoutExtension (source)), ".pdb");

MoveFile (source, target);
}

source = $"{path}.mdb";
if (File.Exists (source)) {
target = Path.ChangeExtension (path, ".mdb");
MoveFile (source, target);
}
}

Console.WriteLine ();
Console.WriteLine ("Method tokens:");
foreach (IList<MarshalMethodEntry> methodList in methods.Values) {
foreach (MarshalMethodEntry method in methodList) {
Console.WriteLine ($"\t{method.NativeCallback.FullName} (token: 0x{method.NativeCallback.MetadataToken.RID:x})");
}
}

void MoveFile (string source, string target)
{
Console.WriteLine ($"Moving '{source}' => '{target}'");
Files.CopyIfChanged (source, target);
try {
File.Delete (source);
} catch (Exception ex) {
log.LogWarning ($"Unable to delete source file '{source}' when moving it to '{target}'");
}
}
}

ICollection<string> GetAssemblyPaths (AssemblyDefinition asm)
{
if (!assemblyPaths.TryGetValue (asm.Name.Name, out HashSet<string> paths)) {
throw new InvalidOperationException ($"Unable to determine file path for assembly '{asm.Name.Name}'");
}

return paths;
}

MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (DirectoryAssemblyResolver resolver)
{
AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices");
TypeDefinition unmanagedCallersOnlyAttribute = null;
foreach (ModuleDefinition md in asm.Modules) {
foreach (ExportedType et in md.ExportedTypes) {
if (!et.IsForwarder) {
continue;
}

if (String.Compare ("System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute", et.FullName, StringComparison.Ordinal) != 0) {
continue;
}

unmanagedCallersOnlyAttribute = et.Resolve ();
break;
}
}

if (unmanagedCallersOnlyAttribute == null) {
throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type");
}

foreach (MethodDefinition md in unmanagedCallersOnlyAttribute.Methods) {
if (!md.IsConstructor) {
continue;
}

return md;
}

throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor");
}

CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition targetAssembly, MethodDefinition unmanagedCallersOnlyAtributeCtor)
{
return new CustomAttribute (targetAssembly.MainModule.ImportReference (unmanagedCallersOnlyAtributeCtor));
}
}
}
#endif
Loading

0 comments on commit 186a9fc

Please sign in to comment.