From 1aed8ca811670f7b7cde69259220e080b844ed2c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 31 Oct 2023 21:49:14 +0100 Subject: [PATCH 001/143] 3 times the charm... --- .../Tasks/ProcessAssemblies.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 80d16a1f72c..8fb44c9f50d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -58,13 +58,13 @@ public override bool RunTask () } // We only need to "dedup" assemblies when there is more than one RID - if (RuntimeIdentifiers.Length > 1) { - Log.LogDebugMessage ("Deduplicating assemblies per RuntimeIdentifier"); - DeduplicateAssemblies (output, symbols); - } else { + // if (RuntimeIdentifiers.Length > 1) { + // Log.LogDebugMessage ("Deduplicating assemblies per RuntimeIdentifier"); + // DeduplicateAssemblies (output, symbols); + // } else { Log.LogDebugMessage ("Found a single RuntimeIdentifier"); SetMetadataForAssemblies (output, symbols); - } + //} OutputAssemblies = output.ToArray (); ResolvedSymbols = symbols.Values.ToArray (); From daf3ed3f2a9a34fc553786e7ac6483cec8e53886 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 2 Nov 2023 22:02:17 +0100 Subject: [PATCH 002/143] Moving forward --- .../Tasks/GenerateJavaStubs.cs | 35 +- .../Tasks/ProcessAssemblies.cs | 14 +- .../Utilities/AssemblyStoreGeneratorNew.cs | 51 +++ .../LlvmIrGenerator/LlvmIrComposer.cs | 12 +- .../LlvmIrGenerator/LlvmIrGenerator.cs | 397 ++++++++++++++---- .../Utilities/LlvmIrGenerator/LlvmIrModule.cs | 8 +- .../LlvmIrGenerator/LlvmIrVariable.cs | 78 ++++ .../LlvmIrGenerator/MemberInfoUtilities.cs | 10 + .../NativeAssemblerAttribute.cs | 2 + .../Utilities/TypeMapGenerator.cs | 5 - ...peMappingReleaseNativeAssemblyGenerator.cs | 6 +- .../Utilities/XAJavaTypeScanner.cs | 12 +- .../Xamarin.Android.Build.Tasks.targets | 29 -- 13 files changed, 511 insertions(+), 148 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 80f1f25b279..475c86516fb 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -152,10 +152,10 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } // Put every assembly we'll need in the resolver - bool hasExportReference = false; - bool haveMonoAndroid = false; var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + AndroidTargetArch arch; + string? assemblyKey; foreach (var assembly in ResolvedAssemblies) { bool value; @@ -166,11 +166,9 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) bool addAssembly = false; string fileName = Path.GetFileName (assembly.ItemSpec); - if (!hasExportReference && String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - hasExportReference = true; + if (String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { addAssembly = true; - } else if (!haveMonoAndroid && String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - haveMonoAndroid = true; + } else if (String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { addAssembly = true; } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { @@ -181,14 +179,15 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } } + assemblyKey = GetAssemblyKey (assembly, out arch); if (addAssembly) { MaybeAddAbiSpecifcAssembly (assembly, fileName); - if (!allTypemapAssemblies.ContainsKey (assembly.ItemSpec)) { - allTypemapAssemblies.Add (assembly.ItemSpec, assembly); + if (!allTypemapAssemblies.ContainsKey (assemblyKey)) { + allTypemapAssemblies.Add (assemblyKey, assembly); } } - res.Load (MonoAndroidHelper.GetTargetArch (assembly), assembly.ItemSpec); + res.Load (arch, assembly.ItemSpec); } // However we only want to look for JLO types in user code for Java stub code generation @@ -197,10 +196,12 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); continue; } - res.Load (MonoAndroidHelper.GetTargetArch (asm), asm.ItemSpec); + + assemblyKey = GetAssemblyKey (asm, out arch); + res.Load (arch, asm.ItemSpec); MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); - if (!allTypemapAssemblies.ContainsKey (asm.ItemSpec)) { - allTypemapAssemblies.Add (asm.ItemSpec, asm); + if (!allTypemapAssemblies.ContainsKey (assemblyKey)) { + allTypemapAssemblies.Add (assemblyKey, asm); } string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); @@ -420,6 +421,16 @@ void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) items.Add (assembly); } } + + string GetAssemblyKey (ITaskItem assembly, out AndroidTargetArch arch) + { + arch = MonoAndroidHelper.GetTargetArch (assembly); + if (arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: assembly '{assembly.ItemSpec}' doesn't specify its ABI. This is not supported."); + } + + return $"[{arch}]{assembly.ItemSpec}"; + } } AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 8fb44c9f50d..1db37adfb6c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -62,9 +62,9 @@ public override bool RunTask () // Log.LogDebugMessage ("Deduplicating assemblies per RuntimeIdentifier"); // DeduplicateAssemblies (output, symbols); // } else { - Log.LogDebugMessage ("Found a single RuntimeIdentifier"); + // Log.LogDebugMessage ("Found a single RuntimeIdentifier"); SetMetadataForAssemblies (output, symbols); - //} + //} OutputAssemblies = output.ToArray (); ResolvedSymbols = symbols.Values.ToArray (); @@ -101,7 +101,7 @@ public override bool RunTask () void SetAssemblyAbiMetadata (string abi, string assetType, ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) { - if (String.IsNullOrEmpty (abi) || (!isDuplicate && String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0)) { + if (String.IsNullOrEmpty (abi) /* || (!isDuplicate && String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0) */) { return; } @@ -115,10 +115,10 @@ void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol, bool isDupli { string assetType = assembly.GetMetadata ("AssetType"); string rid = assembly.GetMetadata ("RuntimeIdentifier"); - if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { - // Satellite assemblies are abi-agnostic, they shouldn't have the Abi metadata set - return; - } + // if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { + // // Satellite assemblies are abi-agnostic, they shouldn't have the Abi metadata set + // return; + // } SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assetType, assembly, symbol, isDuplicate); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs new file mode 100644 index 00000000000..f02ea69313a --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class AssemblyStoreAssemblyInfoNew +{ + public AndroidTargetArch Arch { get; } + public string InArchivePath { get; } + public string SourceFilePath { get; } + + public string? SymbolsFilePath { get; set; } + public string? ConfigFilePath { get; set; } + + public AssemblyStoreAssemblyInfoNew (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) + { + Arch = MonoAndroidHelper.GetTargetArch (assembly); + if (Arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: assembly item '{assembly}' lacks ABI information metadata"); + } + + SourceFilePath = sourceFilePath; + InArchivePath = inArchiveAssemblyPath; + } +} + +class AssemblyStoreGeneratorNew +{ + readonly TaskLoggingHelper log; + readonly Dictionary> assemblies; + + public AssemblyStoreGeneratorNew (TaskLoggingHelper log) + { + this.log = log; + assemblies = new Dictionary> (); + } + + public void Add (AssemblyStoreAssemblyInfoNew asmInfo) + { + if (!assemblies.TryGetValue (asmInfo.Arch, out List infos)) { + infos = new List (); + assemblies.Add (asmInfo.Arch, infos); + } + + infos.Add (asmInfo); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index 8db94269f32..2baa5c0fcc6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -32,11 +32,19 @@ public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter LlvmIrGenerator generator = LlvmIrGenerator.Create (arch, fileName); generator.Generate (output, module); output.Flush (); + + CleanupAfterGeneration (arch); } - public static ulong GetXxHash (string str, bool is64Bit) + protected virtual void CleanupAfterGeneration (AndroidTargetArch arch) + {} + + public static byte[] StringToBytes (string str) => Encoding.UTF8.GetBytes (str); + + public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (StringToBytes (str), is64Bit); + + public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) { - byte[] stringBytes = Encoding.UTF8.GetBytes (str); if (is64Bit) { return XxHash64.HashToUInt64 (stringBytes); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index bb81b03db0e..5442c35fe77 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -29,6 +29,7 @@ sealed class GeneratorWriteContext public readonly LlvmIrMetadataManager MetadataManager; public string CurrentIndent { get; private set; } = String.Empty; public bool InVariableGroup { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Default; public GeneratorWriteContext (TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager) { @@ -80,31 +81,56 @@ sealed class BasicType public readonly string Name; public readonly ulong Size; public readonly bool IsNumeric; + public readonly bool IsUnsigned; + public readonly bool PreferHex; + public readonly string HexFormat; - public BasicType (string name, ulong size, bool isNumeric = true) + public BasicType (string name, ulong size, bool isNumeric = true, bool isUnsigned = false, bool? preferHex = null) { Name = name; Size = size; IsNumeric = isNumeric; + IsUnsigned = isUnsigned; + + // If hex preference isn't specified, we determine whether the type wants to be represented in + // the hexadecimal notation based on signedness. Unsigned types will be represented in hexadecimal, + // but signed types will remain decimal, as it's easier for humans to see the actual value of the + // variable, given this note from LLVM IR manual: + // + // Note that hexadecimal integers are sign extended from the number of active bits, i.e. the bit width minus the number of leading zeros. So ‘s0x0001’ of type ‘i16’ will be -1, not 1. + // + // See: https://llvm.org/docs/LangRef.html#simple-constants + // + if (preferHex.HasValue) { + PreferHex = preferHex.Value; + } else { + PreferHex = isUnsigned; + } + if (!PreferHex) { + HexFormat = String.Empty; + return; + } + + HexFormat = $"x{size * 2}"; } } public const string IRPointerType = "ptr"; static readonly Dictionary basicTypeMap = new Dictionary { - { typeof (bool), new ("i8", 1, isNumeric: false) }, - { typeof (byte), new ("i8", 1) }, - { typeof (char), new ("i16", 2) }, + { typeof (bool), new ("i1", 1, isNumeric: false, isUnsigned: true, preferHex: false) }, + { typeof (byte), new ("i8", 1, isUnsigned: true) }, + { typeof (char), new ("i16", 2, isUnsigned: true, preferHex: false) }, { typeof (sbyte), new ("i8", 1) }, { typeof (short), new ("i16", 2) }, - { typeof (ushort), new ("i16", 2) }, + { typeof (ushort), new ("i16", 2, isUnsigned: true) }, { typeof (int), new ("i32", 4) }, - { typeof (uint), new ("i32", 4) }, + { typeof (uint), new ("i32", 4, isUnsigned: true) }, { typeof (long), new ("i64", 8) }, - { typeof (ulong), new ("i64", 8) }, + { typeof (ulong), new ("i64", 8, isUnsigned: true) }, { typeof (float), new ("float", 4) }, { typeof (double), new ("double", 8) }, - { typeof (void), new ("void", 0, isNumeric: false) }, + { typeof (void), new ("void", 0, isNumeric: false, preferHex: false) }, }; public string FilePath { get; } @@ -191,6 +217,8 @@ void WriteGlobalVariables (GeneratorWriteContext context) } foreach (LlvmIrGlobalVariable gv in context.Module.GlobalVariables) { + context.NumberFormat = gv.NumberFormat; + if (gv is LlvmIrGroupDelimiterVariable groupDelimiter) { if (!context.InVariableGroup && !String.IsNullOrEmpty (groupDelimiter.Comment)) { context.Output.WriteLine (); @@ -240,8 +268,10 @@ void WriteGlobalVariable (GeneratorWriteContext context, LlvmIrGlobalVariable va context.Output.Write (", align "); ulong alignment; - if (typeInfo.IsAggregate) { - ulong count = GetAggregateValueElementCount (variable); + if (variable.Alignment.HasValue) { + alignment = variable.Alignment.Value; + } else if (typeInfo.IsAggregate) { + ulong count = GetAggregateValueElementCount (context, variable); alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, count * typeInfo.Size); } else if (typeInfo.IsStructure) { alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, typeInfo.Size); @@ -280,7 +310,7 @@ void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, return; } - throw new InvalidOperationException ($"Internal error: variable of type {variable.Type} must not have a null value"); + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}'' of type {variable.Type} must not have a null value"); } if (valueType != variable.Type && !LlvmIrModule.NameValueArrayType.IsAssignableFrom (variable.Type)) { @@ -290,9 +320,9 @@ void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, WriteValue (context, valueType, variable); } - ulong GetAggregateValueElementCount (LlvmIrVariable variable) => GetAggregateValueElementCount (variable.Type, variable.Value, variable as LlvmIrGlobalVariable); + ulong GetAggregateValueElementCount (GeneratorWriteContext context, LlvmIrVariable variable) => GetAggregateValueElementCount (context, variable.Type, variable.Value, variable as LlvmIrGlobalVariable); - ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) + ulong GetAggregateValueElementCount (GeneratorWriteContext context, Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) { if (!type.IsArray ()) { throw new InvalidOperationException ($"Internal error: unknown type {type} when trying to determine aggregate type element count"); @@ -300,6 +330,9 @@ ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVaria if (value == null) { if (globalVariable != null) { + if (globalVariable.ArrayDataProvider != null) { + return globalVariable.ArrayDataProvider.GetTotalDataSize (context.Target); + } return globalVariable.ArrayItemCount; } return 0; @@ -386,9 +419,9 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv if (type.IsArray ()) { Type elementType = type.GetArrayElementType (); - ulong elementCount = GetAggregateValueElementCount (type, value, globalVariable); + ulong elementCount = GetAggregateValueElementCount (context, type, value, globalVariable); - WriteArrayType (context, elementType, elementCount, out typeInfo); + WriteArrayType (context, elementType, elementCount, globalVariable, out typeInfo); return; } @@ -404,6 +437,11 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv } void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, out LlvmTypeInfo typeInfo) + { + WriteArrayType (context, elementType, elementCount, variable: null, out typeInfo); + } + + void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, LlvmIrGlobalVariable? variable, out LlvmTypeInfo typeInfo) { string irType; ulong size; @@ -420,6 +458,35 @@ void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elem } else { irType = GetIRType (elementType, out size, out isPointer); maxFieldAlignment = size; + + if (elementType.IsArray) { + if (variable == null) { + throw new InvalidOperationException ($"Internal error: array of arrays ({elementType}) requires variable to be defined"); + } + + // For the sake of simpler code, we currently assume that all the element arrays are of the same size, because that's the only scenario + // that we use at this time. + var value = variable.Value as ICollection; + if (value == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value of type which implements the ICollection interface"); + } + + if (value.Count == 0) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection"); + } + + Array? firstItem = null; + foreach (object v in value) { + firstItem = (Array)v; + break; + } + + if (firstItem == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection with non-null elements"); + } + + irType = $"[{MonoAndroidHelper.CultureInvariantToString (firstItem.Length)} x {irType}]"; + } } typeInfo = new LlvmTypeInfo ( isPointer: isPointer, @@ -449,12 +516,17 @@ ulong GetStructureMaxFieldAlignment (StructureInfo si) void WriteValue (GeneratorWriteContext context, Type valueType, LlvmIrVariable variable) { + if (variable is LlvmIrGlobalVariable globalVariable && globalVariable.ArrayDataProvider != null) { + WriteStreamedArrayValue (context, globalVariable, globalVariable.ArrayDataProvider); + return; + } + if (variable.Type.IsArray ()) { bool zeroInitialize = false; if (variable is LlvmIrGlobalVariable gv) { zeroInitialize = gv.ZeroInitializeArray || variable.Value == null; } else { - zeroInitialize = GetAggregateValueElementCount (variable) == 0; + zeroInitialize = GetAggregateValueElementCount (context, variable) == 0; } if (zeroInitialize) { @@ -478,6 +550,29 @@ void AssertArraySize (StructureInstance si, StructureMemberInfo smi, ulong lengt throw new InvalidOperationException ($"Invalid array size in field '{smi.Info.Name}' of structure '{si.Info.Name}', expected {expectedLength}, found {length}"); } + void WriteInlineArray (GeneratorWriteContext context, byte[] bytes, bool encodeAsASCII) + { + if (encodeAsASCII) { + context.Output.Write ('c'); + context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + return; + } + + string irType = MapToIRType (typeof(byte)); + bool first = true; + context.Output.Write ("[ "); + foreach (byte b in bytes) { + if (!first) { + context.Output.Write (", "); + } else { + first = false; + } + + context.Output.Write ($"{irType} u0x{b:x02}"); + } + context.Output.Write (" ]"); + } + void WriteValue (GeneratorWriteContext context, StructureInstance structInstance, StructureMemberInfo smi, object? value) { if (smi.IsNativePointer) { @@ -495,8 +590,7 @@ void WriteValue (GeneratorWriteContext context, StructureInstance structInstance // Byte arrays are represented in the same way as strings, without the explicit NUL termination byte AssertArraySize (structInstance, smi, length, smi.ArrayElements); - context.Output.Write ('c'); - context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + WriteInlineArray (context, bytes, encodeAsASCII: false); return; } @@ -549,6 +643,27 @@ bool WriteNativePointerValue (GeneratorWriteContext context, StructureInstance s return false; } + string ToHex (BasicType basicTypeDesc, Type type, object? value) + { + const char prefixSigned = 's'; + const char prefixUnsigned = 'u'; + + string hex; + if (type == typeof(byte)) { + hex = ((byte)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ushort)) { + hex = ((ushort)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(uint)) { + hex = ((uint)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ulong)) { + hex = ((ulong)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else { + throw new NotImplementedException ($"Conversion to hexadecimal from type {type} is not implemented"); + }; + + return $"{(basicTypeDesc.IsUnsigned ? prefixUnsigned : prefixSigned)}0x{hex}"; + } + void WriteValue (GeneratorWriteContext context, Type type, object? value) { if (value is LlvmIrVariable variableRef) { @@ -556,14 +671,26 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (IsNumeric (type)) { - context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); - return; - } + bool isBasic = basicTypeMap.TryGetValue (type, out BasicType basicTypeDesc); + if (isBasic) { + if (basicTypeDesc.IsNumeric) { + bool hex = context.NumberFormat switch { + LlvmIrVariableNumberFormat.Default => basicTypeDesc.PreferHex, + LlvmIrVariableNumberFormat.Decimal => false, + LlvmIrVariableNumberFormat.Hexadecimal => true, + _ => throw new InvalidOperationException ($"Internal error: number format {context.NumberFormat} is unsupported") + }; - if (type == typeof(bool)) { - context.Output.Write ((bool)value ? '1' : '0'); - return; + context.Output.Write ( + hex ? ToHex (basicTypeDesc, type, value) : MonoAndroidHelper.CultureInvariantToString (value) + ); + return; + } + + if (type == typeof(bool)) { + context.Output.Write ((bool)value ? "true" : "false"); + return; + } } if (IsStructureInstance (type)) { @@ -588,8 +715,13 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (type.IsInlineArray ()) { + if (type.IsArray) { + if (type == typeof(byte[])) { + WriteInlineArray (context, (byte[])value, encodeAsASCII: true); + return; + } + throw new NotSupportedException ($"Internal error: array of type {type} is unsupported"); } throw new NotSupportedException ($"Internal error: value type '{type}' is unsupported"); @@ -616,8 +748,20 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write (' '); object? value = GetTypedMemberValue (context, info, smi, instance, smi.MemberType); + LlvmIrVariableNumberFormat numberFormat = smi.Info.GetNumberFormat (); + LlvmIrVariableNumberFormat? savedNumberFormat = null; + + if (numberFormat != LlvmIrVariableNumberFormat.Default && numberFormat != context.NumberFormat) { + savedNumberFormat = context.NumberFormat; + context.NumberFormat = numberFormat; + } + WriteValue (context, instance, smi, value); + if (savedNumberFormat.HasValue) { + context.NumberFormat = savedNumberFormat.Value; + } + if (i < lastMember) { context.Output.Write (", "); } @@ -628,9 +772,6 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst sb.Append (MapManagedTypeToNative (smi)); sb.Append (' '); sb.Append (smi.Info.Name); - if (value != null && smi.MemberType.IsPrimitive && smi.MemberType != typeof(bool)) { - sb.Append ($" (0x{value:x})"); - } comment = sb.ToString (); } WriteCommentLine (context, comment); @@ -641,75 +782,80 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write ('}'); } - void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + void WriteArrayValueStart (GeneratorWriteContext context) { - ICollection entries; - if (variable.Type.ImplementsInterface (typeof(IDictionary))) { - var list = new List (); - foreach (var kvp in (IDictionary)variable.Value) { - list.Add (kvp.Key); - list.Add (kvp.Value); - } - entries = list; - } else { - entries = (ICollection)variable.Value; - } - - if (entries.Count == 0) { - context.Output.Write ("zeroinitializer"); - return; - } - context.Output.WriteLine ('['); context.IncreaseIndent (); + } - Type elementType = variable.Type.GetArrayElementType (); - bool writeIndices = (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; - ulong counter = 0; - string? prevItemComment = null; - uint stride; + void WriteArrayValueEnd (GeneratorWriteContext context) + { + context.DecreaseIndent (); + context.Output.Write (']'); + } + uint GetArrayStride (LlvmIrVariable variable) + { if ((variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayFormatInRows) == LlvmIrVariableWriteOptions.ArrayFormatInRows) { - stride = variable.ArrayStride > 0 ? variable.ArrayStride : 1; - } else { - stride = 1; + return variable.ArrayStride > 0 ? variable.ArrayStride : 1; } + return 1; + } + + void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection? entries, Type elementType, uint stride, bool writeIndices, bool terminateWithComma = false) + { bool first = true; + bool ignoreComments = stride > 1; + string? prevItemComment = null; + ulong counter = 0; - // TODO: implement output in rows - foreach (object entry in entries) { - if (!first) { - context.Output.Write (','); - WritePrevItemCommentOrNewline (); - } else { - first = false; - } + if (entries != null) { + foreach (object entry in entries) { + if (!first) { + context.Output.Write (','); + if (stride == 1 || counter % stride == 0) { + WritePrevItemCommentOrNewline (); + context.Output.Write (context.CurrentIndent); + } else { + context.Output.Write (' '); + } + } else { + context.Output.Write (context.CurrentIndent); + first = false; + } - prevItemComment = null; - if (variable.GetArrayItemCommentCallback != null) { - prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); - } + if (!ignoreComments) { + prevItemComment = null; + if (variable.GetArrayItemCommentCallback != null) { + prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); + } - if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { - prevItemComment = $" {counter}"; - } + if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { + prevItemComment = $" {counter}"; + } + } - counter++; - context.Output.Write (context.CurrentIndent); - WriteType (context, elementType, entry, out _); + counter++; + WriteType (context, elementType, entry, out _); - context.Output.Write (' '); - WriteValue (context, elementType, entry); + context.Output.Write (' '); + WriteValue (context, elementType, entry); + } } - WritePrevItemCommentOrNewline (); - context.DecreaseIndent (); - context.Output.Write (']'); + if (terminateWithComma) { + if (!ignoreComments) { + context.Output.WriteLine (); // must put comma outside the comment + context.Output.Write (context.CurrentIndent); + } + context.Output.Write (','); + } + WritePrevItemCommentOrNewline (); void WritePrevItemCommentOrNewline () { - if (!String.IsNullOrEmpty (prevItemComment)) { + if (!ignoreComments && !String.IsNullOrEmpty (prevItemComment)) { context.Output.Write (' '); WriteCommentLine (context, prevItemComment); } else { @@ -718,6 +864,97 @@ void WritePrevItemCommentOrNewline () } } + bool ArrayWantsToWriteIndices (LlvmIrVariable variable) => (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + + void WriteStreamedArrayValue (GeneratorWriteContext context, LlvmIrGlobalVariable variable, LlvmIrStreamedArrayDataProvider dataProvider) + { + ulong dataSizeSoFar = 0; + ulong totalDataSize = dataProvider.GetTotalDataSize (context.Target); + bool first = true; + + WriteArrayValueStart (context); + while (true) { + (LlvmIrStreamedArrayDataProviderState state, ICollection? data) = dataProvider.GetData (context.Target); + if (state == LlvmIrStreamedArrayDataProviderState.NextSectionNoData) { + continue; + } + + bool mustHaveData = state != LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + if (mustHaveData) { + if (data.Count == 0) { + throw new InvalidOperationException ("Data must be provided for streamed arrays"); + } + + dataSizeSoFar += (ulong)data.Count; + if (dataSizeSoFar > totalDataSize) { + throw new InvalidOperationException ($"Data provider {dataProvider} is trying to write more data than declared"); + } + + if (first) { + first = false; + } else { + context.Output.WriteLine (); + } + string comment = dataProvider.GetSectionStartComment (context.Target); + + if (comment.Length > 0) { + context.Output.Write (context.CurrentIndent); + WriteCommentLine (context, comment); + } + } + + bool lastSection = state == LlvmIrStreamedArrayDataProviderState.LastSection || state == LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + WriteArrayEntries ( + context, + variable, + data, + dataProvider.ArrayElementType, + GetArrayStride (variable), + writeIndices: false, + terminateWithComma: !lastSection + ); + + if (lastSection) { + break; + } + + } + WriteArrayValueEnd (context); + } + + void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + { + ICollection entries; + if (variable.Type.ImplementsInterface (typeof(IDictionary))) { + var list = new List (); + foreach (var kvp in (IDictionary)variable.Value) { + list.Add (kvp.Key); + list.Add (kvp.Value); + } + entries = list; + } else { + entries = (ICollection)variable.Value; + } + + if (entries.Count == 0) { + context.Output.Write ("zeroinitializer"); + return; + } + + WriteArrayValueStart (context); + + WriteArrayEntries ( + context, + variable, + entries, + variable.Type.GetArrayElementType (), + GetArrayStride (variable), + writeIndices: ArrayWantsToWriteIndices (variable) + ); + + WriteArrayValueEnd (context); + } + void WriteLinkage (GeneratorWriteContext context, LlvmIrLinkage linkage) { if (linkage == LlvmIrLinkage.Default) { @@ -1232,8 +1469,6 @@ static string MapManagedTypeToNative (StructureMemberInfo smi) return $"{nativeType}*"; } - static bool IsNumeric (Type type) => basicTypeMap.TryGetValue (type, out BasicType typeDesc) && typeDesc.IsNumeric; - object? GetTypedMemberValue (GeneratorWriteContext context, StructureInfo info, StructureMemberInfo smi, StructureInstance instance, Type expectedType, object? defaultValue = null) { object? value = smi.GetValue (instance.Obj); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs index be1570dafe8..9a50f4ee5cf 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs @@ -464,7 +464,7 @@ void AddStringArrayGlobalVariable (LlvmIrGlobalVariable variable, string? string Register (kvp.Value); } } else if (typeof(ICollection).IsAssignableFrom (variable.Type)) { - foreach (string s in (ICollection)variable.Value) { + foreach (string s in (ICollection)variable.Value) { Register (s); } } else { @@ -473,8 +473,12 @@ void AddStringArrayGlobalVariable (LlvmIrGlobalVariable variable, string? string AddStandardGlobalVariable (variable); - void Register (string value) + void Register (string? value) { + if (value == null) { + return; + } + RegisterString (value, stringGroupName, stringGroupComment, symbolSuffix); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 7f741490ce0..49524f0bf6c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Globalization; namespace Xamarin.Android.Tasks.LLVMIR; @@ -11,6 +12,13 @@ enum LlvmIrVariableWriteOptions ArrayFormatInRows = 0x0002, } +enum LlvmIrVariableNumberFormat +{ + Default, + Hexadecimal, + Decimal, +} + abstract class LlvmIrVariable : IEquatable { public abstract bool Global { get; } @@ -29,6 +37,8 @@ abstract class LlvmIrVariable : IEquatable public object? Value { get; set; } public string? Comment { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Decimal; + /// /// Both global and local variables will want their names to matter in equality checks, but function /// parameters must not take it into account, thus this property. If set to false, @@ -146,6 +156,50 @@ public void AssignNumber (ulong n) } } +enum LlvmIrStreamedArrayDataProviderState +{ + NextSection, + LastSection, + NextSectionNoData, + LastSectionNoData, +} + +abstract class LlvmIrStreamedArrayDataProvider +{ + /// + /// Type of every member of the array returned by . Generator will check + /// every member type against this property, allowing also derived types. + /// + public Type ArrayElementType { get; } + + protected LlvmIrStreamedArrayDataProvider (Type arrayElementType) + { + ArrayElementType = arrayElementType; + } + + /// + /// Whenever returns the generator will call this method to obtain the new section + /// comment, if any, to be output before the actual data. Returning `String.Empty` prevents the comment + /// from being added. + /// + public virtual string GetSectionStartComment (LlvmIrModuleTarget target) => String.Empty; + + /// + /// Provide the next chunk of data for the specified target (ABI). Implementations need to return at least one + /// non-empty collection of data. The returned collection **must** be exactly the size of contained data (e.g. it cannot be + /// a byte array rented from a byte pool, because these can be bigger than requested. When returning the last (or the only) section, + /// must have a value of . + /// Each section may be preceded by a comment, . + /// + public abstract (LlvmIrStreamedArrayDataProviderState status, ICollection data) GetData (LlvmIrModuleTarget target); + + /// + /// Provide the total data size for the specified target (ABI). This needs to be used instead of + /// because a variable instance is created once and shared by all targets, while per-target data sets might have different sizes. + /// + public abstract ulong GetTotalDataSize (LlvmIrModuleTarget target); +} + class LlvmIrGlobalVariable : LlvmIrVariable { /// @@ -162,9 +216,33 @@ class LlvmIrGlobalVariable : LlvmIrVariable /// public virtual LlvmIrVariableOptions? Options { get; set; } + /// + /// If set to `true`, initialize the array with a shortcut zero-initializer statement. Useful when pre-allocating + /// space for runtime use that won't be filled in with any data at the build time. + /// public bool ZeroInitializeArray { get; set; } + + /// + /// Specify number of items in an array. Used in cases when we want to pre-allocate an array without giving it any + /// value, thus making it impossible for the generator to discover the number of items automatically. This is useful + /// when using . This property is used **only** if the variable + /// is `null`. + /// public ulong ArrayItemCount { get; set; } + /// + /// If set, it will override any automatically calculated alignment for this variable + /// + public ulong? Alignment { get; set; } + + /// + /// If set, the provider will be called to obtain all the data to be placed in an array variable. The total amount + /// of data that will be returned by the provider **must** be specified in the property, + /// in order for the generator to properly declare the variable. The generator will verify that the amount of data + /// is exactly that much and throw an exception otherwise. + /// + public LlvmIrStreamedArrayDataProvider? ArrayDataProvider { get; set; } + /// /// Constructs a local variable. is translated to one of the LLVM IR first class types (see /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs index 431d92b6229..713b0c5a1a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs @@ -71,5 +71,15 @@ public static bool InlineArrayNeedsPadding (this MemberInfo mi) return attr.NeedsPadding; } + + public static LlvmIrVariableNumberFormat GetNumberFormat (this MemberInfo mi) + { + var attr = mi.GetCustomAttribute (); + if (attr == null) { + return LlvmIrVariableNumberFormat.Default; + } + + return attr.NumberFormat; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs index 39fc5e094bb..c239255e1ac 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs @@ -33,6 +33,8 @@ class NativeAssemblerAttribute : Attribute /// size to which the member must be padded is specified by /// public bool NeedsPadding { get; set; } + + public LLVMIR.LlvmIrVariableNumberFormat NumberFormat { get; set; } = LLVMIR.LlvmIrVariableNumberFormat.Default; } [AttributeUsage (AttributeTargets.Class, Inherited = true)] diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 3730176e751..4a930cf9c5c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -455,11 +455,6 @@ bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List var state = new ReleaseGenerationState (supportedAbis); foreach (JavaType jt in javaTypes) { - if (!jt.IsABiSpecific) { - ProcessReleaseType (state, jt.Type, AndroidTargetArch.None, appConfState, cache); - continue; - } - foreach (var kvp in jt.PerAbiTypes) { ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index a6e4fd465df..81ca3cf006b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -69,6 +69,7 @@ sealed class TypeMapModuleEntry [NativeAssembler (Ignore = true)] public TypeMapJava JavaTypeMapEntry; + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_map_index; } @@ -126,6 +127,8 @@ sealed class TypeMapJava public ulong JavaNameHash64; public uint module_index; + + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_name_index; } @@ -212,6 +215,7 @@ protected override void Construct (LlvmIrModule module) BeforeWriteCallbackCallerState = cs, GetArrayItemCommentCallback = GetJavaHashesItemComment, GetArrayItemCommentCallbackCallerState = cs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; map_java_hashes.WriteOptions &= ~LlvmIrVariableWriteOptions.ArrayWriteIndexComments; module.Add (map_java_hashes); @@ -258,7 +262,7 @@ uint GetJavaEntryIndex (TypeMapJava javaEntry) throw new InvalidOperationException ("Internal error: construction state expected but not found"); } - return $" {index}: 0x{value:x} => {cs.JavaMap[(int)index].Instance.JavaName}"; + return $" {index} => {cs.JavaMap[(int)index].Instance.JavaName}"; } void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 5f965ab46ff..4640251e521 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -14,15 +14,11 @@ class JavaType { public readonly TypeDefinition Type; public readonly IDictionary? PerAbiTypes; - public bool IsABiSpecific { get; } - public JavaType (TypeDefinition type, IDictionary? perAbiTypes) + public JavaType (TypeDefinition type, IDictionary perAbiTypes) { Type = type; - if (perAbiTypes != null) { - PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); - IsABiSpecific = perAbiTypes.Count > 1 || (perAbiTypes.Count == 1 && !perAbiTypes.ContainsKey (AndroidTargetArch.None)); - } + PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); } } @@ -33,8 +29,6 @@ sealed class TypeData public readonly TypeDefinition FirstType; public readonly Dictionary PerAbi; - public bool IsAbiSpecific => !PerAbi.ContainsKey (AndroidTargetArch.None); - public TypeData (TypeDefinition firstType) { FirstType = firstType; @@ -69,7 +63,7 @@ public List GetJavaTypes (ICollection inputAssemblies, XAAs var ret = new List (); foreach (var kvp in types) { - ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.IsAbiSpecific ? kvp.Value.PerAbi : null)); + ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.PerAbi)); } return ret; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets index fbd0fd842c7..fe34046bab6 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets @@ -268,35 +268,6 @@ - - - <_InputAssembliesThatExist Include="@(InputAssemblies)" Condition="Exists('%(Identity)')" /> - <_NetstandardPath Include="@(ReferencePath->'%(RootDir)%(Directory)')" Condition="'%(FileName)%(Extension)' == 'netstandard.dll'" /> - - - <_NetstandardDir>@(_NetstandardPath) - <_ILRepackArgs>/out:"$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" /internalize - <_ILRepackArgs>$(_ILRepackArgs) /keyfile:"$(XamarinAndroidSourcePath)product.snk" - <_ILRepackArgs>$(_ILRepackArgs) "$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" - <_ILRepackArgs>$(_ILRepackArgs) @(_InputAssembliesThatExist->'"%(Identity)"', ' ') - <_ILRepackArgs>$(_ILRepackArgs) /lib:"$(_NetstandardDir.TrimEnd('\'))" - - - - - - - - <_ExtraPackageSource Include="$(PkgXamarin_LibZipSharp)\lib\$(TargetFrameworkNETStandard)\libZipSharp.pdb" /> <_ExtraPackageTarget Include="$(OutputPath)\libZipSharp.pdb" /> From 7be2b3585601b1aa08c9b16f2087fea0c6e5c5c6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 3 Nov 2023 22:28:23 +0100 Subject: [PATCH 003/143] New assembly blob format implemented --- .../Tasks/BuildApk.cs | 77 ++++--- .../AssemblyStoreGeneratorNew.Classes.cs | 53 +++++ .../Utilities/AssemblyStoreGeneratorNew.cs | 198 +++++++++++++++++- .../Utilities/MonoAndroidHelper.cs | 13 +- .../Xamarin.Android.Common.targets | 2 + src/monodroid/jni/xamarin-app.hh | 12 +- 6 files changed, 303 insertions(+), 52 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index fbcaa1e0944..9e8c55bd88b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -33,6 +33,9 @@ public class BuildApk : AndroidTask [Required] public string ApkOutputPath { get; set; } + [Required] + public string AppSharedLibrariesDir { get; set; } + [Required] public ITaskItem[] ResolvedUserAssemblies { get; set; } @@ -212,10 +215,6 @@ void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOut } } - if (!String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath)) { - AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, $"{AssembliesPath}rc.bin", compressionMethod: UncompressedMethod); - } - foreach (var file in files) { var item = Path.Combine (file.archivePath.Replace (Path.DirectorySeparatorChar, '/')); existingEntries.Remove (item); @@ -367,29 +366,15 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> assemblyStorePaths = storeGenerator.Generate (Path.GetDirectoryName (ApkOutputPath)); - if (assemblyStorePaths == null) { + Dictionary assemblyStorePathsNew = storeGenerator.Generate (AppSharedLibrariesDir); + + if (assemblyStorePathsNew.Count == 0) { throw new InvalidOperationException ("Assembly store generator did not generate any stores"); } - if (!assemblyStorePaths.TryGetValue (assemblyStoreApkName, out List baseAssemblyStores) || baseAssemblyStores == null || baseAssemblyStores.Count == 0) { - throw new InvalidOperationException ("Assembly store generator didn't generate the required base stores"); + if (assemblyStorePathsNew.Count != SupportedAbis.Length) { + throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); } - string assemblyStorePrefix = $"{assemblyStoreApkName}_"; - foreach (string assemblyStorePath in baseAssemblyStores) { - string inArchiveName = Path.GetFileName (assemblyStorePath); - - if (inArchiveName.StartsWith (assemblyStorePrefix, StringComparison.Ordinal)) { - inArchiveName = inArchiveName.Substring (assemblyStorePrefix.Length); - } + // We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter + // our assembly store. Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99% + // of cases these days, since AAB enforces that split). `base.apk` contains only ABI-agnostic file, while one of the split config files contains only + // ABI-specific data+code. + bool addRuntimeConfig = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); + string libRootPath = "lib/"; + foreach (var kvp in assemblyStorePathsNew) { + string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); + string inArchivePath = GetArchLibPath (abi, Path.GetFileName (kvp.Value)); + AddFileToArchiveIfNewer (apk, kvp.Value, inArchivePath, UncompressedMethod); - CompressionMethod compressionMethod; - if (inArchiveName.EndsWith (".manifest", StringComparison.Ordinal)) { - compressionMethod = CompressionMethod.Default; - } else { - compressionMethod = UncompressedMethod; + if (!addRuntimeConfig) { + continue; } - AddFileToArchiveIfNewer (apk, assemblyStorePath, AssembliesPath + inArchiveName, compressionMethod); + // Prefix it with `a` because bundletool sorts entries alphabetically, and this will place it right next to `assemblies.*.blob.so`, which is what we + // like since we can finish scanning the zip central directory earlier at startup. + inArchivePath = GetArchLibPath (abi, "arc.bin.so"); + AddFileToArchiveIfNewer (apk, RuntimeConfigBinFilePath, inArchivePath, compressionMethod: UncompressedMethod); } + string GetArchLibPath (string abi, string fileName) => libRootPath + abi + "/" + fileName; + void AddAssembliesFromCollection (ITaskItem[] assemblies) { foreach (ITaskItem assembly in assemblies) { @@ -445,7 +436,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) // Add assembly var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); if (UseAssemblyStore) { - storeAssembly = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly.GetMetadata ("Abi")); + storeAssemblyInfo = new AssemblyStoreAssemblyInfoNew (sourcePath, assemblyPath, assembly); } else { AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); } @@ -453,7 +444,9 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) // Try to add config if exists var config = Path.ChangeExtension (assembly.ItemSpec, "dll.config"); if (UseAssemblyStore) { - storeAssembly.SetConfigPath (config); + if (File.Exists (config)) { + storeAssemblyInfo.ConfigFile = new FileInfo (config); + } } else { AddAssemblyConfigEntry (apk, assemblyPath, config); } @@ -469,7 +462,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) if (!String.IsNullOrEmpty (symbolsPath)) { if (UseAssemblyStore) { - storeAssembly.SetDebugInfoPath (symbolsPath); + storeAssemblyInfo.SymbolsFile = new FileInfo (symbolsPath); } else { AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); } @@ -477,7 +470,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) } if (UseAssemblyStore) { - storeGenerator.Add (assemblyStoreApkName, storeAssembly); + storeGenerator.Add (storeAssemblyInfo); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs new file mode 100644 index 00000000000..9739117ff04 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs @@ -0,0 +1,53 @@ +using System; +using System.IO; + +namespace Xamarin.Android.Tasks; + +partial class AssemblyStoreGeneratorNew +{ + sealed class AssemblyStoreHeader + { + public const uint NativeSize = 3 * sizeof (uint); + + public readonly uint magic = ASSEMBLY_STORE_MAGIC; + public readonly uint version; + public readonly uint entry_count = 0; + + public AssemblyStoreHeader (uint version, uint entry_count) + { + this.version = version; + this.entry_count = entry_count; + } + } + + sealed class AssemblyStoreIndexEntry + { + public const uint NativeSize32 = 2 * sizeof (uint); + public const uint NativeSize64 = sizeof (ulong) + sizeof (uint); + + public readonly ulong name_hash; + public readonly uint descriptor_index; + + public AssemblyStoreIndexEntry (ulong name_hash, uint descriptor_index) + { + this.name_hash = name_hash; + this.descriptor_index = descriptor_index; + } + } + + sealed class AssemblyStoreEntryDescriptor + { + public const uint NativeSize = 7 * sizeof (uint); + + public uint mapping_index; + + public uint data_offset; + public uint data_size; + + public uint debug_data_offset; + public uint debug_data_size; + + public uint config_data_offset; + public uint config_data_size; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs index f02ea69313a..30d249db46d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; +using System.IO; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Xamarin.Android.Tools; @@ -9,12 +11,12 @@ namespace Xamarin.Android.Tasks; class AssemblyStoreAssemblyInfoNew { - public AndroidTargetArch Arch { get; } - public string InArchivePath { get; } - public string SourceFilePath { get; } + public AndroidTargetArch Arch { get; } + public string InArchivePath { get; } + public FileInfo SourceFile { get; } - public string? SymbolsFilePath { get; set; } - public string? ConfigFilePath { get; set; } + public FileInfo? SymbolsFile { get; set; } + public FileInfo? ConfigFile { get; set; } public AssemblyStoreAssemblyInfoNew (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) { @@ -23,13 +25,50 @@ public AssemblyStoreAssemblyInfoNew (string sourceFilePath, string inArchiveAsse throw new InvalidOperationException ($"Internal error: assembly item '{assembly}' lacks ABI information metadata"); } - SourceFilePath = sourceFilePath; + SourceFile = new FileInfo (sourceFilePath); InArchivePath = inArchiveAssemblyPath; } } -class AssemblyStoreGeneratorNew +// +// Assembly store format +// +// Each target ABI/architecture has a single assembly store file, composed of the following parts: +// +// [HEADER] +// [INDEX] +// [ASSEMBLY_DESCRIPTORS] +// [ASSEMBLY DATA] +// +// Formats of the sections above are as follows: +// +// HEADER (fixed size) +// [MAGIC] uint; value: 0x41424158 +// [FORMAT] uint; store format version number +// [ENTRY_COUNT] uint; number of entries in the store +// +// INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) +// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name +// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array +// +// ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [MAPPING_INDEX] uint; index into a runtime array where assembly data pointers are stored +// [DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly data +// [DATA_SIZE] uint; size of the stored assembly data +// [DEBUG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly PDB data, 0 if absent +// [DEBUG_DATA_SIZE] uint; size of the stored assembly PDB data, 0 if absent +// [CONFIG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly .config contents, 0 if absent +// [CONFIG_DATA_SIZE] uint; size of the stored assembly .config contents, 0 if absent +// +partial class AssemblyStoreGeneratorNew { + // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh + const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant + + // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones + const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; + readonly TaskLoggingHelper log; readonly Dictionary> assemblies; @@ -48,4 +87,149 @@ public void Add (AssemblyStoreAssemblyInfoNew asmInfo) infos.Add (asmInfo); } + + public Dictionary Generate (string baseOutputDirectory) + { + var ret = new Dictionary (); + + foreach (var kvp in assemblies) { + string storePath = Generate (baseOutputDirectory, kvp.Key, kvp.Value); + ret.Add (kvp.Key, storePath); + } + + return ret; + } + + string Generate (string baseOutputDirectory, AndroidTargetArch arch, List infos) + { + bool is64Bit = arch switch { + AndroidTargetArch.Arm => false, + AndroidTargetArch.X86 => false, + AndroidTargetArch.Arm64 => true, + AndroidTargetArch.X86_64 => true, + _ => throw new NotSupportedException ($"Internal error: arch {arch} not supported") + }; + + string androidAbi = MonoAndroidHelper.ArchToAbi (arch); + uint infoCount = (uint)infos.Count; + string storePath = Path.Combine (baseOutputDirectory, androidAbi, $"assemblies.{androidAbi}.blob.so"); + var index = new List (); + var descriptors = new List (); + + ulong assemblyDataStart = (infoCount * IndexEntrySize () * 2) + (AssemblyStoreEntryDescriptor.NativeSize * infoCount) + AssemblyStoreHeader.NativeSize; + // We'll start writing to the stream after we seek to the position just after the header, index and descriptors data. + ulong curPos = assemblyDataStart; + + using var fs = File.Open (storePath, FileMode.Create, FileAccess.Write, FileShare.Read); + fs.Seek ((long)curPos, SeekOrigin.Begin); + + foreach (AssemblyStoreAssemblyInfoNew info in infos) { + (AssemblyStoreEntryDescriptor desc, curPos) = MakeDescriptor (info, curPos); + desc.mapping_index = (uint)descriptors.Count; + descriptors.Add (desc); + + if ((uint)fs.Position != desc.data_offset) { + throw new InvalidOperationException ($"Internal error: corrupted store '{storePath}' stream"); + } + + ulong name_with_ext_hash = LLVMIR.LlvmIrComposer.GetXxHash (Path.GetFileName (info.SourceFile.Name), is64Bit); + ulong name_no_ext_hash = LLVMIR.LlvmIrComposer.GetXxHash (Path.GetFileNameWithoutExtension (info.SourceFile.Name), is64Bit); + index.Add (new AssemblyStoreIndexEntry (name_with_ext_hash, desc.mapping_index)); + index.Add (new AssemblyStoreIndexEntry (name_no_ext_hash, desc.mapping_index)); + + CopyData (info.SourceFile, fs, storePath); + CopyData (info.SymbolsFile, fs, storePath); + CopyData (info.ConfigFile, fs, storePath); + } + fs.Flush (); + fs.Seek (0, SeekOrigin.Begin); + + uint storeVersion = is64Bit ? ASSEMBLY_STORE_FORMAT_VERSION_64BIT : ASSEMBLY_STORE_FORMAT_VERSION_32BIT; + var header = new AssemblyStoreHeader (storeVersion, infoCount); + using var writer = new BinaryWriter (fs); + writer.Write (header.magic); + writer.Write (header.version); + writer.Write (header.entry_count); + + index.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.name_hash.CompareTo (b.name_hash)); + foreach (AssemblyStoreIndexEntry entry in index) { + if (is64Bit) { + writer.Write (entry.name_hash); + } else { + writer.Write ((uint)entry.name_hash); + } + writer.Write (entry.descriptor_index); + } + writer.Flush (); + + Console.WriteLine ($"Number of descriptors: {descriptors.Count}; index entries: {index.Count}"); + Console.WriteLine ($"Header size: {AssemblyStoreHeader.NativeSize}; index entry size: {IndexEntrySize ()}; descriptor size: {AssemblyStoreEntryDescriptor.NativeSize}"); + + foreach (AssemblyStoreEntryDescriptor desc in descriptors) { + writer.Write (desc.mapping_index); + writer.Write (desc.data_offset); + writer.Write (desc.data_size); + writer.Write (desc.debug_data_offset); + writer.Write (desc.debug_data_size); + writer.Write (desc.config_data_offset); + writer.Write (desc.config_data_size); + } + writer.Flush (); + + if (fs.Position != (long)assemblyDataStart) { + Console.WriteLine ($"fs.Position == {fs.Position}; assemblyDataStart == {assemblyDataStart}"); + throw new InvalidOperationException ($"Internal error: store '{storePath}' position is different than metadata size after header write"); + } + + return storePath; + + uint IndexEntrySize () => is64Bit ? AssemblyStoreIndexEntry.NativeSize64 : AssemblyStoreIndexEntry.NativeSize32; + } + + void CopyData (FileInfo? src, Stream dest, string storePath) + { + if (src == null) { + return; + } + + log.LogDebugMessage ($"Adding file '{src.Name}' to assembly store '{storePath}'"); + using var fs = src.Open (FileMode.Open, FileAccess.Read, FileShare.Read); + fs.CopyTo (dest); + } + + static (AssemblyStoreEntryDescriptor desc, ulong newPos) MakeDescriptor (AssemblyStoreAssemblyInfoNew info, ulong curPos) + { + var ret = new AssemblyStoreEntryDescriptor { + data_offset = (uint)curPos, + data_size = GetDataLength (info.SourceFile), + }; + if (info.SymbolsFile != null) { + ret.debug_data_offset = ret.data_offset + ret.data_size; + ret.debug_data_size = GetDataLength (info.SymbolsFile); + } + + if (info.ConfigFile != null) { + ret.config_data_offset = ret.data_offset + ret.data_size + ret.debug_data_size; + ret.config_data_size = GetDataLength (info.ConfigFile); + } + + curPos += ret.data_size + ret.debug_data_size + ret.config_data_size; + if (curPos > UInt32.MaxValue) { + throw new NotSupportedException ("Assembly store size exceeds the maximum supported value"); + } + + return (ret, curPos); + + uint GetDataLength (FileInfo? info) { + if (info == null) { + return 0; + } + + if (info.Length > UInt32.MaxValue) { + throw new NotSupportedException ($"File '{info.Name}' exceeds the maximum supported size"); + } + + return (uint)info.Length; + } + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index d72b347d5e9..984e8190261 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -574,7 +574,18 @@ public static string ArchToRid (AndroidTargetArch arch) AndroidTargetArch.Arm => "android-arm", AndroidTargetArch.X86 => "android-x86", AndroidTargetArch.X86_64 => "android-x64", - _ => throw new InvalidOperationException ($"Internal error: unsupported ABI '{arch}'") + _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") + }; + } + + public static string ArchToAbi (AndroidTargetArch arch) + { + return arch switch { + AndroidTargetArch.Arm64 => "arm64-v8a", + AndroidTargetArch.Arm => "armeabi-v7a", + AndroidTargetArch.X86 => "x86", + AndroidTargetArch.X86_64 => "x86_64", + _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") }; } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 330b30f9a7e..4edb130ae9e 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -2111,6 +2111,7 @@ because xbuild doesn't support framework reference assemblies. AndroidNdkDirectory="$(_AndroidNdkDirectory)" ApkInputPath="$(_PackagedResources)" ApkOutputPath="$(ApkFileIntermediate)" + AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)" BundleNativeLibraries="$(_BundleResultNativeLibraries)" EmbedAssemblies="$(EmbedAssembliesIntoApk)" ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)" @@ -2146,6 +2147,7 @@ because xbuild doesn't support framework reference assemblies. AndroidNdkDirectory="$(_AndroidNdkDirectory)" ApkInputPath="$(_PackagedResources)" ApkOutputPath="$(_BaseZipIntermediate)" + AppSharedLibrariesDir="$(_AndroidApplicationSharedLibraryPath)" BundleNativeLibraries="$(_BundleResultNativeLibraries)" EmbedAssemblies="$(EmbedAssembliesIntoApk)" ResolvedUserAssemblies="@(_ShrunkUserAssemblies);@(_AndroidResolvedSatellitePaths)" diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index d50fc39a683..a125c5aaf42 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -13,8 +13,16 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the - // assembly store format + +#if INTPTR_MAX == INT64_MAX +static constexpr uint32_t ASSEMBLY_STORE_64BIT_FLAG = 0x80000000; +#else +static constexpr uint32_t ASSEMBLY_STORE_64BIT_FLAG = 0x00000000; +#endif + +// Increase whenever an incompatible change is made to the assembly store format +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 2 | ASSEMBLY_STORE_64BIT_FLAG; + static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian static constexpr uint8_t MODULE_FORMAT_VERSION = 2; // Keep in sync with the value in src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs From 972c8ddcb48468e516005e72fcf8343ebf60ee1f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 6 Nov 2023 23:01:38 +0100 Subject: [PATCH 004/143] Runtime almost works, binary search fails. Fix tmrw. --- .../Tasks/BuildApk.cs | 8 +- .../Tasks/GeneratePackageManagerJava.cs | 4 - .../Utilities/ApplicationConfig.cs | 1 - ...pplicationConfigNativeAssemblyGenerator.cs | 19 +- .../Utilities/ArchAssemblyStore.cs | 112 ------ .../Utilities/AssemblyStore.cs | 377 ------------------ .../Utilities/AssemblyStoreAssemblyInfo.cs | 54 +-- ...s.cs => AssemblyStoreGenerator.Classes.cs} | 9 +- .../Utilities/AssemblyStoreGenerator.cs | 290 +++++++++----- .../Utilities/AssemblyStoreGeneratorNew.cs | 235 ----------- .../Utilities/AssemblyStoreGlobalIndex.cs | 29 -- .../Utilities/AssemblyStoreIndexEntry.cs | 43 -- .../Utilities/CommonAssemblyStore.cs | 39 -- .../LlvmIrGenerator/LlvmIrGenerator.cs | 2 +- src/monodroid/jni/application_dso_stub.cc | 17 +- src/monodroid/jni/embedded-assemblies-zip.cc | 68 +--- src/monodroid/jni/embedded-assemblies.cc | 61 ++- src/monodroid/jni/embedded-assemblies.hh | 36 +- src/monodroid/jni/monodroid-glue.cc | 5 +- src/monodroid/jni/shared-constants.hh | 6 +- src/monodroid/jni/xamarin-app.hh | 33 +- 21 files changed, 323 insertions(+), 1125 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs rename src/Xamarin.Android.Build.Tasks/Utilities/{AssemblyStoreGeneratorNew.Classes.cs => AssemblyStoreGenerator.Classes.cs} (85%) delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 9e8c55bd88b..27eb621cc12 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -366,15 +366,15 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary>), "assembly_stores", LlvmIrVariableOptions.GlobalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = itemCount, + var storeRuntimeData = new AssemblyStoreRuntimeData { + data_start = 0, + assembly_count = 0, }; - module.Add (assembly_stores); + + var assembly_store = new LlvmIrGlobalVariable ( + new StructureInstance(assemblyStoreRuntimeDataStructureInfo, storeRuntimeData), + "assembly_store", + LlvmIrVariableOptions.GlobalWritable + ); + module.Add (assembly_store); } void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs deleted file mode 100644 index a5b5811b7de..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ArchAssemblyStore.cs +++ /dev/null @@ -1,112 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class ArchAssemblyStore : AssemblyStore - { - readonly Dictionary> assemblies; - HashSet seenArchAssemblyNames; - - public ArchAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new Dictionary> (StringComparer.OrdinalIgnoreCase); - } - - public override string WriteIndex (List globalIndex) - { - throw new InvalidOperationException ("Architecture-specific assembly blob cannot contain global assembly index"); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-agnostic assembly cannot be added to an architecture-specific blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - if (!assemblies.ContainsKey (blobAssembly.Abi)) { - assemblies.Add (blobAssembly.Abi, new List ()); - } - - List blobAssemblies = assemblies[blobAssembly.Abi]; - blobAssemblies.Add (blobAssembly); - - if (seenArchAssemblyNames == null) { - seenArchAssemblyNames = new HashSet (StringComparer.Ordinal); - } - - string assemblyName = GetAssemblyName (blobAssembly); - if (seenArchAssemblyNames.Contains (assemblyName)) { - return; - } - - seenArchAssemblyNames.Add (assemblyName); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - var assemblyNames = new Dictionary (); - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - // All the architecture blobs must have assemblies in exactly the same order - archAssemblies.Sort ((AssemblyStoreAssemblyInfo a, AssemblyStoreAssemblyInfo b) => Path.GetFileName (a.FilesystemAssemblyPath).CompareTo (Path.GetFileName (b.FilesystemAssemblyPath))); - if (assemblyNames.Count == 0) { - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - assemblyNames.Add (i, Path.GetFileName (info.FilesystemAssemblyPath)); - } - continue; - } - - if (archAssemblies.Count != assemblyNames.Count) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' has a different number of assemblies than other ABI lists (expected {assemblyNames.Count}, found {archAssemblies.Count}"); - } - - for (int i = 0; i < archAssemblies.Count; i++) { - AssemblyStoreAssemblyInfo info = archAssemblies[i]; - string fileName = Path.GetFileName (info.FilesystemAssemblyPath); - - if (assemblyNames[i] != fileName) { - throw new InvalidOperationException ($"Assembly list for ABI '{abi}' differs from other lists at index {i}. Expected '{assemblyNames[i]}', found '{fileName}'"); - } - } - } - - bool addToGlobalIndex = true; - foreach (var kvp in assemblies) { - string abi = kvp.Key; - List archAssemblies = kvp.Value; - - if (archAssemblies.Count == 0) { - continue; - } - - // Android uses underscores in place of dashes in ABI names, let's follow the convention - string androidAbi = abi.Replace ('-', '_'); - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}.{androidAbi}{BlobExtension}"), archAssemblies, globalIndex, blobPaths, addToGlobalIndex); - - // NOTE: not thread safe! The counter must grow monotonically but we also don't want to use different index values for the architecture-specific - // assemblies with the same names, that would only waste space in the generated `libxamarin-app.so`. To use the same index values for the same - // assemblies in different architectures we need to move the counter back here. - GlobalIndexCounter.Subtract ((uint)archAssemblies.Count); - - if (addToGlobalIndex) { - // We want the architecture-specific assemblies to be added to the global index only once - addToGlobalIndex = false; - } - } - - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs deleted file mode 100644 index 378a9ee8c46..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStore.cs +++ /dev/null @@ -1,377 +0,0 @@ -using System; -using System.Buffers; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - abstract class AssemblyStore - { - // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh - const uint BlobMagic = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - const uint BlobVersion = 1; // Must match the BUNDLED_ASSEMBLIES_BLOB_VERSION native constant - - // MUST be equal to the size of the BlobBundledAssembly struct in src/monodroid/jni/xamarin-app.hh - const uint BlobBundledAssemblyNativeStructSize = 6 * sizeof (uint); - - // MUST be equal to the size of the BlobHashEntry struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHashEntryNativeStructSize = sizeof (ulong) + (3 * sizeof (uint)); - - // MUST be equal to the size of the BundledAssemblyBlobHeader struct in src/monodroid/jni/xamarin-app.hh - const uint BlobHeaderNativeStructSize = sizeof (uint) * 5; - - protected const string BlobPrefix = "assemblies"; - protected const string BlobExtension = ".blob"; - - static readonly ArrayPool bytePool = ArrayPool.Shared; - - string archiveAssembliesPrefix; - string indexBlobPath; - - protected string ApkName { get; } - protected TaskLoggingHelper Log { get; } - protected AssemblyStoreGlobalIndex GlobalIndexCounter { get; } - - public uint ID { get; } - public bool IsIndexStore => ID == 0; - - protected AssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } - - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); - } - - GlobalIndexCounter = globalIndexCounter ?? throw new ArgumentNullException (nameof (globalIndexCounter)); - ID = id; - - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - ApkName = apkName; - Log = log; - } - - public abstract void Add (AssemblyStoreAssemblyInfo blobAssembly); - public abstract void Generate (string outputDirectory, List globalIndex, List blobPaths); - - public virtual string WriteIndex (List globalIndex) - { - if (!IsIndexStore) { - throw new InvalidOperationException ("Assembly index may be written only to blob with index 0"); - } - - if (String.IsNullOrEmpty (indexBlobPath)) { - throw new InvalidOperationException ("Index blob path not set, was Generate called properly?"); - } - - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - string indexBlobHeaderPath = $"{indexBlobPath}.hdr"; - string indexBlobManifestPath = Path.ChangeExtension (indexBlobPath, "manifest"); - - using (var hfs = File.Open (indexBlobHeaderPath, FileMode.Create, FileAccess.Write, FileShare.None)) { - using (var writer = new BinaryWriter (hfs, Encoding.UTF8, leaveOpen: true)) { - WriteIndex (writer, indexBlobManifestPath, globalIndex); - writer.Flush (); - } - - using (var ifs = File.Open (indexBlobPath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - ifs.CopyTo (hfs); - hfs.Flush (); - } - } - - File.Delete (indexBlobPath); - File.Move (indexBlobHeaderPath, indexBlobPath); - - return indexBlobManifestPath; - } - - void WriteIndex (BinaryWriter blobWriter, string manifestPath, List globalIndex) - { - using (var manifest = File.Open (manifestPath, FileMode.Create, FileAccess.Write)) { - using (var manifestWriter = new StreamWriter (manifest, new UTF8Encoding (false))) { - WriteIndex (blobWriter, manifestWriter, globalIndex); - manifestWriter.Flush (); - } - } - } - - void WriteIndex (BinaryWriter blobWriter, StreamWriter manifestWriter, List globalIndex) - { - uint localEntryCount = 0; - var localAssemblies = new List (); - - manifestWriter.WriteLine ("Hash 32 Hash 64 Blob ID Blob idx Name"); - - var seenHashes32 = new HashSet (); - var seenHashes64 = new HashSet (); - bool haveDuplicates = false; - foreach (AssemblyStoreIndexEntry assembly in globalIndex) { - if (assembly.StoreID == ID) { - localEntryCount++; - localAssemblies.Add (assembly); - } - - if (WarnAboutDuplicateHash ("32", assembly.Name, assembly.NameHash32, seenHashes32) || - WarnAboutDuplicateHash ("64", assembly.Name, assembly.NameHash64, seenHashes64)) { - haveDuplicates = true; - } - - manifestWriter.WriteLine ($"0x{assembly.NameHash32:x08} 0x{assembly.NameHash64:x016} {assembly.StoreID:d03} {assembly.LocalBlobIndex:d04} {assembly.Name}"); - } - - if (haveDuplicates) { - throw new InvalidOperationException ("Duplicate assemblies encountered"); - } - - uint globalAssemblyCount = (uint)globalIndex.Count; - - blobWriter.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (blobWriter, localEntryCount, globalAssemblyCount); - - // Header and two tables of the same size, each for 32 and 64-bit hashes - uint offsetFixup = BlobHeaderNativeStructSize + (BlobHashEntryNativeStructSize * globalAssemblyCount * 2); - - WriteAssemblyDescriptors (blobWriter, localAssemblies, CalculateOffsetFixup ((uint)localAssemblies.Count, offsetFixup)); - - var sortedIndex = new List (globalIndex); - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash32.CompareTo (b.NameHash32)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash32); - } - - sortedIndex.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.NameHash64.CompareTo (b.NameHash64)); - foreach (AssemblyStoreIndexEntry entry in sortedIndex) { - WriteHash (entry, entry.NameHash64); - } - - void WriteHash (AssemblyStoreIndexEntry entry, ulong hash) - { - blobWriter.Write (hash); - blobWriter.Write (entry.MappingIndex); - blobWriter.Write (entry.LocalBlobIndex); - blobWriter.Write (entry.StoreID); - } - - bool WarnAboutDuplicateHash (string bitness, string assemblyName, ulong hash, HashSet seenHashes) - { - if (seenHashes.Contains (hash)) { - Log.LogMessage (MessageImportance.High, $"Duplicate {bitness}-bit hash 0x{hash} encountered for assembly {assemblyName}"); - return true; - } - - seenHashes.Add (hash); - return false; - } - } - - protected string GetAssemblyName (AssemblyStoreAssemblyInfo assembly) - { - string assemblyName = Path.GetFileNameWithoutExtension (assembly.FilesystemAssemblyPath); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - return assemblyName; - } - - protected void Generate (string outputFilePath, List assemblies, List globalIndex, List blobPaths, bool addToGlobalIndex = true) - { - if (globalIndex == null) { - throw new ArgumentNullException (nameof (globalIndex)); - } - - if (blobPaths == null) { - throw new ArgumentNullException (nameof (blobPaths)); - } - - if (IsIndexStore) { - indexBlobPath = outputFilePath; - } - - blobPaths.Add (outputFilePath); - Log.LogMessage (MessageImportance.Low, $"AssemblyBlobGenerator: generating blob: {outputFilePath}"); - - using (var fs = File.Open (outputFilePath, FileMode.Create, FileAccess.Write, FileShare.Read)) { - using (var writer = new BinaryWriter (fs, Encoding.UTF8)) { - Generate (writer, assemblies, globalIndex, addToGlobalIndex); - writer.Flush (); - } - } - } - - void Generate (BinaryWriter writer, List assemblies, List globalIndex, bool addToGlobalIndex) - { - var localAssemblies = new List (); - - if (!IsIndexStore) { - // Index blob's header and data before the assemblies is handled in WriteIndex in a slightly different - // way. - uint nbytes = BlobHeaderNativeStructSize + (BlobBundledAssemblyNativeStructSize * (uint)assemblies.Count); - var zeros = bytePool.Rent ((int)nbytes); - writer.Write (zeros, 0, (int)nbytes); - bytePool.Return (zeros); - } - - foreach (AssemblyStoreAssemblyInfo assembly in assemblies) { - string assemblyName = GetAssemblyName (assembly); - string archivePath = assembly.ArchiveAssemblyPath; - if (archivePath.StartsWith (archiveAssembliesPrefix, StringComparison.OrdinalIgnoreCase)) { - archivePath = archivePath.Substring (archiveAssembliesPrefix.Length); - } - - if (!String.IsNullOrEmpty (assembly.Abi)) { - string abiPath = $"{assembly.Abi}/"; - if (archivePath.StartsWith (abiPath, StringComparison.Ordinal)) { - archivePath = archivePath.Substring (abiPath.Length); - } - } - - if (!String.IsNullOrEmpty (archivePath)) { - if (archivePath.EndsWith ("/", StringComparison.Ordinal)) { - assemblyName = $"{archivePath}{assemblyName}"; - } else { - assemblyName = $"{archivePath}/{assemblyName}"; - } - } - - AssemblyStoreIndexEntry entry = WriteAssembly (writer, assembly, assemblyName, (uint)localAssemblies.Count); - if (addToGlobalIndex) { - globalIndex.Add (entry); - } - localAssemblies.Add (entry); - } - - writer.Flush (); - - if (IsIndexStore) { - return; - } - - writer.Seek (0, SeekOrigin.Begin); - WriteBlobHeader (writer, (uint)localAssemblies.Count); - WriteAssemblyDescriptors (writer, localAssemblies); - } - - uint CalculateOffsetFixup (uint localAssemblyCount, uint extraOffset = 0) - { - return (BlobBundledAssemblyNativeStructSize * (uint)localAssemblyCount) + extraOffset; - } - - void WriteBlobHeader (BinaryWriter writer, uint localEntryCount, uint globalEntryCount = 0) - { - // Header, must be identical to the BundledAssemblyBlobHeader structure in src/monodroid/jni/xamarin-app.hh - writer.Write (BlobMagic); // magic - writer.Write (BlobVersion); // version - writer.Write (localEntryCount); // local_entry_count - writer.Write (globalEntryCount); // global_entry_count - writer.Write ((uint)ID); // blob_id - } - - void WriteAssemblyDescriptors (BinaryWriter writer, List assemblies, uint offsetFixup = 0) - { - // Each assembly must be identical to the BlobBundledAssembly structure in src/monodroid/jni/xamarin-app.hh - - foreach (AssemblyStoreIndexEntry assembly in assemblies) { - AdjustOffsets (assembly, offsetFixup); - - writer.Write (assembly.DataOffset); - writer.Write (assembly.DataSize); - - writer.Write (assembly.DebugDataOffset); - writer.Write (assembly.DebugDataSize); - - writer.Write (assembly.ConfigDataOffset); - writer.Write (assembly.ConfigDataSize); - } - } - - void AdjustOffsets (AssemblyStoreIndexEntry assembly, uint offsetFixup) - { - if (offsetFixup == 0) { - return; - } - - assembly.DataOffset += offsetFixup; - - if (assembly.DebugDataOffset > 0) { - assembly.DebugDataOffset += offsetFixup; - } - - if (assembly.ConfigDataOffset > 0) { - assembly.ConfigDataOffset += offsetFixup; - } - } - - AssemblyStoreIndexEntry WriteAssembly (BinaryWriter writer, AssemblyStoreAssemblyInfo assembly, string assemblyName, uint localBlobIndex) - { - uint offset; - uint size; - - (offset, size) = WriteFile (assembly.FilesystemAssemblyPath, true); - - // NOTE: globalAssemblIndex++ is not thread safe but it **must** increase monotonically (see also ArchAssemblyStore.Generate for a special case) - var ret = new AssemblyStoreIndexEntry (assemblyName, ID, GlobalIndexCounter.Increment (), localBlobIndex) { - DataOffset = offset, - DataSize = size, - }; - - (offset, size) = WriteFile (assembly.DebugInfoPath, required: false); - if (offset != 0 && size != 0) { - ret.DebugDataOffset = offset; - ret.DebugDataSize = size; - } - - // Config files must end with \0 (nul) - (offset, size) = WriteFile (assembly.ConfigPath, required: false, appendNul: true); - if (offset != 0 && size != 0) { - ret.ConfigDataOffset = offset; - ret.ConfigDataSize = size; - } - - return ret; - - (uint offset, uint size) WriteFile (string filePath, bool required, bool appendNul = false) - { - if (!File.Exists (filePath)) { - if (required) { - throw new InvalidOperationException ($"Required file '{filePath}' not found"); - } - - return (0, 0); - } - - var fi = new FileInfo (filePath); - if (fi.Length == 0) { - return (0, 0); - } - - if (fi.Length > UInt32.MaxValue || writer.BaseStream.Position + fi.Length > UInt32.MaxValue) { - throw new InvalidOperationException ($"Writing assembly '{filePath}' to assembly blob would exceed the maximum allowed data size."); - } - - uint offset = (uint)writer.BaseStream.Position; - using (var fs = File.Open (filePath, FileMode.Open, FileAccess.Read, FileShare.Read)) { - fs.CopyTo (writer.BaseStream); - } - - uint length = (uint)fi.Length; - if (appendNul) { - length++; - writer.Write ((byte)0); - } - - return (offset, length); - } - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index c5c166fb787..3f5a44a5840 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -1,48 +1,28 @@ using System; using System.IO; -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreAssemblyInfo - { - public string FilesystemAssemblyPath { get; } - public string ArchiveAssemblyPath { get; } - public string DebugInfoPath { get; private set; } - public string ConfigPath { get; private set; } - public string Abi { get; } +using Microsoft.Build.Framework; +using Xamarin.Android.Tools; - public AssemblyStoreAssemblyInfo (string filesystemAssemblyPath, string archiveAssemblyPath, string abi) - { - if (String.IsNullOrEmpty (filesystemAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (filesystemAssemblyPath)); - } +namespace Xamarin.Android.Tasks; - if (String.IsNullOrEmpty (archiveAssemblyPath)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssemblyPath)); - } +class AssemblyStoreAssemblyInfo +{ + public AndroidTargetArch Arch { get; } + public string InArchivePath { get; } + public FileInfo SourceFile { get; } - FilesystemAssemblyPath = filesystemAssemblyPath; - ArchiveAssemblyPath = archiveAssemblyPath; - Abi = abi; - } + public FileInfo? SymbolsFile { get; set; } + public FileInfo? ConfigFile { get; set; } - public void SetDebugInfoPath (string path) - { - DebugInfoPath = GetExistingPath (path); - } - - public void SetConfigPath (string path) - { - ConfigPath = GetExistingPath (path); + public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) + { + Arch = MonoAndroidHelper.GetTargetArch (assembly); + if (Arch == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: assembly item '{assembly}' lacks ABI information metadata"); } - string GetExistingPath (string path) - { - if (String.IsNullOrEmpty (path) || !File.Exists (path)) { - return String.Empty; - } - - return path; - } + SourceFile = new FileInfo (sourceFilePath); + InArchivePath = inArchiveAssemblyPath; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs similarity index 85% rename from src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs index 9739117ff04..cd513597383 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs @@ -1,9 +1,6 @@ -using System; -using System.IO; - namespace Xamarin.Android.Tasks; -partial class AssemblyStoreGeneratorNew +partial class AssemblyStoreGenerator { sealed class AssemblyStoreHeader { @@ -25,11 +22,13 @@ sealed class AssemblyStoreIndexEntry public const uint NativeSize32 = 2 * sizeof (uint); public const uint NativeSize64 = sizeof (ulong) + sizeof (uint); + public readonly string name; public readonly ulong name_hash; public readonly uint descriptor_index; - public AssemblyStoreIndexEntry (ulong name_hash, uint descriptor_index) + public AssemblyStoreIndexEntry (string name, ulong name_hash, uint descriptor_index) { + this.name = name; this.name_hash = name_hash; this.descriptor_index = descriptor_index; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index d60af903bc9..f70ac0f7c67 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -1,133 +1,233 @@ using System; using System.Collections.Generic; using System.IO; -using System.Text; -using Microsoft.Build.Framework; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +// +// Assembly store format +// +// Each target ABI/architecture has a single assembly store file, composed of the following parts: +// +// [HEADER] +// [INDEX] +// [ASSEMBLY_DESCRIPTORS] +// [ASSEMBLY DATA] +// +// Formats of the sections above are as follows: +// +// HEADER (fixed size) +// [MAGIC] uint; value: 0x41424158 +// [FORMAT] uint; store format version number +// [ENTRY_COUNT] uint; number of entries in the store +// +// INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) +// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name +// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array +// +// ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [MAPPING_INDEX] uint; index into a runtime array where assembly data pointers are stored +// [DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly data +// [DATA_SIZE] uint; size of the stored assembly data +// [DEBUG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly PDB data, 0 if absent +// [DEBUG_DATA_SIZE] uint; size of the stored assembly PDB data, 0 if absent +// [CONFIG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly .config contents, 0 if absent +// [CONFIG_DATA_SIZE] uint; size of the stored assembly .config contents, 0 if absent +// +partial class AssemblyStoreGenerator { - class AssemblyStoreGenerator - { - sealed class Store - { - public AssemblyStore Common; - public AssemblyStore Arch; - } + // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh + const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - readonly string archiveAssembliesPrefix; - readonly TaskLoggingHelper log; + // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones + const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; - // NOTE: when/if we have parallel BuildApk these should become concurrent collections - readonly Dictionary stores = new Dictionary (StringComparer.Ordinal); + readonly TaskLoggingHelper log; + readonly Dictionary> assemblies; - AssemblyStore indexStore; + public AssemblyStoreGenerator (TaskLoggingHelper log) + { + this.log = log; + assemblies = new Dictionary> (); + } - // IDs must be counted per AssemblyStoreGenerator instance because it's possible that a single build will create more than one instance of the class and each time - // the stores must be assigned IDs starting from 0, or there will be errors due to "missing" index store - readonly Dictionary apkIds = new Dictionary (StringComparer.Ordinal); + public void Add (AssemblyStoreAssemblyInfo asmInfo) + { + if (!assemblies.TryGetValue (asmInfo.Arch, out List infos)) { + infos = new List (); + assemblies.Add (asmInfo.Arch, infos); + } - // Global assembly index must be restarted from 0 for the same reasons as apkIds above and at the same time it must be unique for each assembly added to **any** - // assembly store, thus we need to keep the state here - AssemblyStoreGlobalIndex globalIndexCounter = new AssemblyStoreGlobalIndex (); + infos.Add (asmInfo); + } - public AssemblyStoreGenerator (string archiveAssembliesPrefix, TaskLoggingHelper log) - { - if (String.IsNullOrEmpty (archiveAssembliesPrefix)) { - throw new ArgumentException ("must not be null or empty", nameof (archiveAssembliesPrefix)); - } + public Dictionary Generate (string baseOutputDirectory) + { + var ret = new Dictionary (); - this.archiveAssembliesPrefix = archiveAssembliesPrefix; - this.log = log; + foreach (var kvp in assemblies) { + string storePath = Generate (baseOutputDirectory, kvp.Key, kvp.Value); + ret.Add (kvp.Key, storePath); } - public void Add (string apkName, AssemblyStoreAssemblyInfo storeAssembly) - { - if (String.IsNullOrEmpty (apkName)) { - throw new ArgumentException ("must not be null or empty", nameof (apkName)); + return ret; + } + + string Generate (string baseOutputDirectory, AndroidTargetArch arch, List infos) + { + bool is64Bit = arch switch { + AndroidTargetArch.Arm => false, + AndroidTargetArch.X86 => false, + AndroidTargetArch.Arm64 => true, + AndroidTargetArch.X86_64 => true, + _ => throw new NotSupportedException ($"Internal error: arch {arch} not supported") + }; + + string androidAbi = MonoAndroidHelper.ArchToAbi (arch); + uint infoCount = (uint)infos.Count; + string storePath = Path.Combine (baseOutputDirectory, androidAbi, $"assemblies.{androidAbi}.blob.so"); + var index = new List (); + var descriptors = new List (); + + ulong assemblyDataStart = (infoCount * IndexEntrySize () * 2) + (AssemblyStoreEntryDescriptor.NativeSize * infoCount) + AssemblyStoreHeader.NativeSize; + // We'll start writing to the stream after we seek to the position just after the header, index and descriptors data. + ulong curPos = assemblyDataStart; + + using var fs = File.Open (storePath, FileMode.Create, FileAccess.Write, FileShare.Read); + fs.Seek ((long)curPos, SeekOrigin.Begin); + + foreach (AssemblyStoreAssemblyInfo info in infos) { + (AssemblyStoreEntryDescriptor desc, curPos) = MakeDescriptor (info, curPos); + desc.mapping_index = (uint)descriptors.Count; + descriptors.Add (desc); + + if ((uint)fs.Position != desc.data_offset) { + throw new InvalidOperationException ($"Internal error: corrupted store '{storePath}' stream"); } - Store store; - if (!stores.ContainsKey (apkName)) { - store = new Store { - Common = new CommonAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter), - Arch = new ArchAssemblyStore (apkName, archiveAssembliesPrefix, log, GetNextStoreID (apkName), globalIndexCounter) - }; + string name_with_ext = Path.GetFileName (info.SourceFile.Name); + ulong name_with_ext_hash = LLVMIR.LlvmIrComposer.GetXxHash (name_with_ext, is64Bit); - stores.Add (apkName, store); - SetIndexStore (store.Common); - SetIndexStore (store.Arch); - } + string name_no_ext = Path.GetFileNameWithoutExtension (info.SourceFile.Name); + ulong name_no_ext_hash = LLVMIR.LlvmIrComposer.GetXxHash (name_no_ext, is64Bit); + index.Add (new AssemblyStoreIndexEntry (name_with_ext, name_with_ext_hash, desc.mapping_index)); + index.Add (new AssemblyStoreIndexEntry (name_no_ext, name_no_ext_hash, desc.mapping_index)); - store = stores[apkName]; - if (String.IsNullOrEmpty (storeAssembly.Abi)) { - store.Common.Add (storeAssembly); + CopyData (info.SourceFile, fs, storePath); + CopyData (info.SymbolsFile, fs, storePath); + CopyData (info.ConfigFile, fs, storePath); + } + fs.Flush (); + fs.Seek (0, SeekOrigin.Begin); + + uint storeVersion = is64Bit ? ASSEMBLY_STORE_FORMAT_VERSION_64BIT : ASSEMBLY_STORE_FORMAT_VERSION_32BIT; + var header = new AssemblyStoreHeader (storeVersion, infoCount); + using var writer = new BinaryWriter (fs); + writer.Write (header.magic); + writer.Write (header.version); + writer.Write (header.entry_count); + + index.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.name_hash.CompareTo (b.name_hash)); + + using var manifestFs = File.Open ($"{storePath}.manifest", FileMode.Create, FileAccess.Write, FileShare.Read); + using var mw = new StreamWriter (manifestFs, new System.Text.UTF8Encoding (false)); + foreach (AssemblyStoreIndexEntry entry in index) { + if (is64Bit) { + writer.Write (entry.name_hash); + mw.Write ($"0x{entry.name_hash:x}"); } else { - store.Arch.Add (storeAssembly); + writer.Write ((uint)entry.name_hash); + mw.Write ($"0x{(uint)entry.name_hash:x}"); } + writer.Write (entry.descriptor_index); + mw.Write ($" di:{entry.descriptor_index}"); + + AssemblyStoreEntryDescriptor desc = descriptors[(int)entry.descriptor_index]; + mw.Write ($" mi:{desc.mapping_index}"); + mw.Write ($" do:{desc.data_offset}"); + mw.Write ($" ds:{desc.data_size}"); + mw.Write ($" ddo:{desc.debug_data_offset}"); + mw.Write ($" dds:{desc.debug_data_size}"); + mw.Write ($" cdo:{desc.config_data_offset}"); + mw.Write ($" cds:{desc.config_data_size}"); + mw.WriteLine ($" {entry.name}"); + } + writer.Flush (); + mw.Flush (); + + Console.WriteLine ($"Number of descriptors: {descriptors.Count}; index entries: {index.Count}"); + Console.WriteLine ($"Header size: {AssemblyStoreHeader.NativeSize}; index entry size: {IndexEntrySize ()}; descriptor size: {AssemblyStoreEntryDescriptor.NativeSize}"); + + foreach (AssemblyStoreEntryDescriptor desc in descriptors) { + writer.Write (desc.mapping_index); + writer.Write (desc.data_offset); + writer.Write (desc.data_size); + writer.Write (desc.debug_data_offset); + writer.Write (desc.debug_data_size); + writer.Write (desc.config_data_offset); + writer.Write (desc.config_data_size); + } + writer.Flush (); - void SetIndexStore (AssemblyStore b) - { - if (!b.IsIndexStore) { - return; - } + if (fs.Position != (long)assemblyDataStart) { + Console.WriteLine ($"fs.Position == {fs.Position}; assemblyDataStart == {assemblyDataStart}"); + throw new InvalidOperationException ($"Internal error: store '{storePath}' position is different than metadata size after header write"); + } - if (indexStore != null) { - throw new InvalidOperationException ("Index store already set!"); - } + return storePath; - indexStore = b; - } - } + uint IndexEntrySize () => is64Bit ? AssemblyStoreIndexEntry.NativeSize64 : AssemblyStoreIndexEntry.NativeSize32; + } - uint GetNextStoreID (string apkName) - { - // NOTE: NOT thread safe, if we ever have parallel runs of BuildApk this operation must either be atomic or protected with a lock - if (!apkIds.ContainsKey (apkName)) { - apkIds.Add (apkName, 0); - } - return apkIds[apkName]++; + void CopyData (FileInfo? src, Stream dest, string storePath) + { + if (src == null) { + return; } - public Dictionary> Generate (string outputDirectory) - { - if (stores.Count == 0) { - return null; - } + log.LogDebugMessage ($"Adding file '{src.Name}' to assembly store '{storePath}'"); + using var fs = src.Open (FileMode.Open, FileAccess.Read, FileShare.Read); + fs.CopyTo (dest); + } - if (indexStore == null) { - throw new InvalidOperationException ("Index store not found"); - } + static (AssemblyStoreEntryDescriptor desc, ulong newPos) MakeDescriptor (AssemblyStoreAssemblyInfo info, ulong curPos) + { + var ret = new AssemblyStoreEntryDescriptor { + data_offset = (uint)curPos, + data_size = GetDataLength (info.SourceFile), + }; + if (info.SymbolsFile != null) { + ret.debug_data_offset = ret.data_offset + ret.data_size; + ret.debug_data_size = GetDataLength (info.SymbolsFile); + } - var globalIndex = new List (); - var ret = new Dictionary> (StringComparer.Ordinal); - string indexStoreApkName = null; - foreach (var kvp in stores) { - string apkName = kvp.Key; - Store store = kvp.Value; + if (info.ConfigFile != null) { + ret.config_data_offset = ret.data_offset + ret.data_size + ret.debug_data_size; + ret.config_data_size = GetDataLength (info.ConfigFile); + } - if (!ret.ContainsKey (apkName)) { - ret.Add (apkName, new List ()); - } + curPos += ret.data_size + ret.debug_data_size + ret.config_data_size; + if (curPos > UInt32.MaxValue) { + throw new NotSupportedException ("Assembly store size exceeds the maximum supported value"); + } - if (store.Common == indexStore || store.Arch == indexStore) { - indexStoreApkName = apkName; - } + return (ret, curPos); - GenerateStore (store.Common, apkName); - GenerateStore (store.Arch, apkName); + uint GetDataLength (FileInfo? info) { + if (info == null) { + return 0; } - string manifestPath = indexStore.WriteIndex (globalIndex); - ret[indexStoreApkName].Add (manifestPath); - - return ret; - - void GenerateStore (AssemblyStore store, string apkName) - { - store.Generate (outputDirectory, globalIndex, ret[apkName]); + if (info.Length > UInt32.MaxValue) { + throw new NotSupportedException ($"File '{info.Name}' exceeds the maximum supported size"); } + + return (uint)info.Length; } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs deleted file mode 100644 index 30d249db46d..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGeneratorNew.cs +++ /dev/null @@ -1,235 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks; - -class AssemblyStoreAssemblyInfoNew -{ - public AndroidTargetArch Arch { get; } - public string InArchivePath { get; } - public FileInfo SourceFile { get; } - - public FileInfo? SymbolsFile { get; set; } - public FileInfo? ConfigFile { get; set; } - - public AssemblyStoreAssemblyInfoNew (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) - { - Arch = MonoAndroidHelper.GetTargetArch (assembly); - if (Arch == AndroidTargetArch.None) { - throw new InvalidOperationException ($"Internal error: assembly item '{assembly}' lacks ABI information metadata"); - } - - SourceFile = new FileInfo (sourceFilePath); - InArchivePath = inArchiveAssemblyPath; - } -} - -// -// Assembly store format -// -// Each target ABI/architecture has a single assembly store file, composed of the following parts: -// -// [HEADER] -// [INDEX] -// [ASSEMBLY_DESCRIPTORS] -// [ASSEMBLY DATA] -// -// Formats of the sections above are as follows: -// -// HEADER (fixed size) -// [MAGIC] uint; value: 0x41424158 -// [FORMAT] uint; store format version number -// [ENTRY_COUNT] uint; number of entries in the store -// -// INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) -// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name -// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array -// -// ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: -// [MAPPING_INDEX] uint; index into a runtime array where assembly data pointers are stored -// [DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly data -// [DATA_SIZE] uint; size of the stored assembly data -// [DEBUG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly PDB data, 0 if absent -// [DEBUG_DATA_SIZE] uint; size of the stored assembly PDB data, 0 if absent -// [CONFIG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly .config contents, 0 if absent -// [CONFIG_DATA_SIZE] uint; size of the stored assembly .config contents, 0 if absent -// -partial class AssemblyStoreGeneratorNew -{ - // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh - const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - - // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones - const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant - const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; - - readonly TaskLoggingHelper log; - readonly Dictionary> assemblies; - - public AssemblyStoreGeneratorNew (TaskLoggingHelper log) - { - this.log = log; - assemblies = new Dictionary> (); - } - - public void Add (AssemblyStoreAssemblyInfoNew asmInfo) - { - if (!assemblies.TryGetValue (asmInfo.Arch, out List infos)) { - infos = new List (); - assemblies.Add (asmInfo.Arch, infos); - } - - infos.Add (asmInfo); - } - - public Dictionary Generate (string baseOutputDirectory) - { - var ret = new Dictionary (); - - foreach (var kvp in assemblies) { - string storePath = Generate (baseOutputDirectory, kvp.Key, kvp.Value); - ret.Add (kvp.Key, storePath); - } - - return ret; - } - - string Generate (string baseOutputDirectory, AndroidTargetArch arch, List infos) - { - bool is64Bit = arch switch { - AndroidTargetArch.Arm => false, - AndroidTargetArch.X86 => false, - AndroidTargetArch.Arm64 => true, - AndroidTargetArch.X86_64 => true, - _ => throw new NotSupportedException ($"Internal error: arch {arch} not supported") - }; - - string androidAbi = MonoAndroidHelper.ArchToAbi (arch); - uint infoCount = (uint)infos.Count; - string storePath = Path.Combine (baseOutputDirectory, androidAbi, $"assemblies.{androidAbi}.blob.so"); - var index = new List (); - var descriptors = new List (); - - ulong assemblyDataStart = (infoCount * IndexEntrySize () * 2) + (AssemblyStoreEntryDescriptor.NativeSize * infoCount) + AssemblyStoreHeader.NativeSize; - // We'll start writing to the stream after we seek to the position just after the header, index and descriptors data. - ulong curPos = assemblyDataStart; - - using var fs = File.Open (storePath, FileMode.Create, FileAccess.Write, FileShare.Read); - fs.Seek ((long)curPos, SeekOrigin.Begin); - - foreach (AssemblyStoreAssemblyInfoNew info in infos) { - (AssemblyStoreEntryDescriptor desc, curPos) = MakeDescriptor (info, curPos); - desc.mapping_index = (uint)descriptors.Count; - descriptors.Add (desc); - - if ((uint)fs.Position != desc.data_offset) { - throw new InvalidOperationException ($"Internal error: corrupted store '{storePath}' stream"); - } - - ulong name_with_ext_hash = LLVMIR.LlvmIrComposer.GetXxHash (Path.GetFileName (info.SourceFile.Name), is64Bit); - ulong name_no_ext_hash = LLVMIR.LlvmIrComposer.GetXxHash (Path.GetFileNameWithoutExtension (info.SourceFile.Name), is64Bit); - index.Add (new AssemblyStoreIndexEntry (name_with_ext_hash, desc.mapping_index)); - index.Add (new AssemblyStoreIndexEntry (name_no_ext_hash, desc.mapping_index)); - - CopyData (info.SourceFile, fs, storePath); - CopyData (info.SymbolsFile, fs, storePath); - CopyData (info.ConfigFile, fs, storePath); - } - fs.Flush (); - fs.Seek (0, SeekOrigin.Begin); - - uint storeVersion = is64Bit ? ASSEMBLY_STORE_FORMAT_VERSION_64BIT : ASSEMBLY_STORE_FORMAT_VERSION_32BIT; - var header = new AssemblyStoreHeader (storeVersion, infoCount); - using var writer = new BinaryWriter (fs); - writer.Write (header.magic); - writer.Write (header.version); - writer.Write (header.entry_count); - - index.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.name_hash.CompareTo (b.name_hash)); - foreach (AssemblyStoreIndexEntry entry in index) { - if (is64Bit) { - writer.Write (entry.name_hash); - } else { - writer.Write ((uint)entry.name_hash); - } - writer.Write (entry.descriptor_index); - } - writer.Flush (); - - Console.WriteLine ($"Number of descriptors: {descriptors.Count}; index entries: {index.Count}"); - Console.WriteLine ($"Header size: {AssemblyStoreHeader.NativeSize}; index entry size: {IndexEntrySize ()}; descriptor size: {AssemblyStoreEntryDescriptor.NativeSize}"); - - foreach (AssemblyStoreEntryDescriptor desc in descriptors) { - writer.Write (desc.mapping_index); - writer.Write (desc.data_offset); - writer.Write (desc.data_size); - writer.Write (desc.debug_data_offset); - writer.Write (desc.debug_data_size); - writer.Write (desc.config_data_offset); - writer.Write (desc.config_data_size); - } - writer.Flush (); - - if (fs.Position != (long)assemblyDataStart) { - Console.WriteLine ($"fs.Position == {fs.Position}; assemblyDataStart == {assemblyDataStart}"); - throw new InvalidOperationException ($"Internal error: store '{storePath}' position is different than metadata size after header write"); - } - - return storePath; - - uint IndexEntrySize () => is64Bit ? AssemblyStoreIndexEntry.NativeSize64 : AssemblyStoreIndexEntry.NativeSize32; - } - - void CopyData (FileInfo? src, Stream dest, string storePath) - { - if (src == null) { - return; - } - - log.LogDebugMessage ($"Adding file '{src.Name}' to assembly store '{storePath}'"); - using var fs = src.Open (FileMode.Open, FileAccess.Read, FileShare.Read); - fs.CopyTo (dest); - } - - static (AssemblyStoreEntryDescriptor desc, ulong newPos) MakeDescriptor (AssemblyStoreAssemblyInfoNew info, ulong curPos) - { - var ret = new AssemblyStoreEntryDescriptor { - data_offset = (uint)curPos, - data_size = GetDataLength (info.SourceFile), - }; - if (info.SymbolsFile != null) { - ret.debug_data_offset = ret.data_offset + ret.data_size; - ret.debug_data_size = GetDataLength (info.SymbolsFile); - } - - if (info.ConfigFile != null) { - ret.config_data_offset = ret.data_offset + ret.data_size + ret.debug_data_size; - ret.config_data_size = GetDataLength (info.ConfigFile); - } - - curPos += ret.data_size + ret.debug_data_size + ret.config_data_size; - if (curPos > UInt32.MaxValue) { - throw new NotSupportedException ("Assembly store size exceeds the maximum supported value"); - } - - return (ret, curPos); - - uint GetDataLength (FileInfo? info) { - if (info == null) { - return 0; - } - - if (info.Length > UInt32.MaxValue) { - throw new NotSupportedException ($"File '{info.Name}' exceeds the maximum supported size"); - } - - return (uint)info.Length; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs deleted file mode 100644 index 6ce93f11f9d..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGlobalIndex.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace Xamarin.Android.Tasks -{ - // This class may seem weird, but it's designed with the specific needs of AssemblyStore instances in mind and also prepared for thread-safe use in the future, should the - // need arise - sealed class AssemblyStoreGlobalIndex - { - uint value = 0; - - public uint Value => value; - - /// - /// Increments the counter and returns its previous value - /// - public uint Increment () - { - uint ret = value++; - return ret; - } - - public void Subtract (uint count) - { - if (value < count) { - return; - } - - value -= count; - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs deleted file mode 100644 index 51d2bd8ca77..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreIndexEntry.cs +++ /dev/null @@ -1,43 +0,0 @@ -using System; -using System.IO.Hashing; -using System.Text; - -namespace Xamarin.Android.Tasks -{ - class AssemblyStoreIndexEntry - { - public string Name { get; } - public uint StoreID { get; } - public uint MappingIndex { get; } - public uint LocalBlobIndex { get; } - - // Hash values must have the same type as they are inside a union in the native code - public ulong NameHash64 { get; } - public ulong NameHash32 { get; } - - public uint DataOffset { get; set; } - public uint DataSize { get; set; } - - public uint DebugDataOffset { get; set; } - public uint DebugDataSize { get; set; } - - public uint ConfigDataOffset { get; set; } - public uint ConfigDataSize { get; set; } - - public AssemblyStoreIndexEntry (string name, uint blobID, uint mappingIndex, uint localBlobIndex) - { - if (String.IsNullOrEmpty (name)) { - throw new ArgumentException ("must not be null or empty", nameof (name)); - } - - Name = name; - StoreID = blobID; - MappingIndex = mappingIndex; - LocalBlobIndex = localBlobIndex; - - byte[] nameBytes = Encoding.UTF8.GetBytes (name); - NameHash32 = XxHash32.HashToUInt32 (nameBytes); - NameHash64 = XxHash64.HashToUInt64 (nameBytes); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs deleted file mode 100644 index 9709a200e5a..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CommonAssemblyStore.cs +++ /dev/null @@ -1,39 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Text; - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -namespace Xamarin.Android.Tasks -{ - class CommonAssemblyStore : AssemblyStore - { - readonly List assemblies; - - public CommonAssemblyStore (string apkName, string archiveAssembliesPrefix, TaskLoggingHelper log, uint id, AssemblyStoreGlobalIndex globalIndexCounter) - : base (apkName, archiveAssembliesPrefix, log, id, globalIndexCounter) - { - assemblies = new List (); - } - - public override void Add (AssemblyStoreAssemblyInfo blobAssembly) - { - if (!String.IsNullOrEmpty (blobAssembly.Abi)) { - throw new InvalidOperationException ($"Architecture-specific assembly cannot be added to an architecture-agnostic blob ({blobAssembly.FilesystemAssemblyPath})"); - } - - assemblies.Add (blobAssembly); - } - - public override void Generate (string outputDirectory, List globalIndex, List blobPaths) - { - if (assemblies.Count == 0) { - return; - } - - Generate (Path.Combine (outputDirectory, $"{ApkName}_{BlobPrefix}{BlobExtension}"), assemblies, globalIndex, blobPaths); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index 5442c35fe77..be469065e3a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -406,7 +406,7 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv { if (IsStructureInstance (type)) { if (value == null) { - throw new ArgumentException ("must not be null for structure instances", nameof (value)); + throw new ArgumentException ($"must not be null for structure instances ({type})", nameof (value)); } WriteStructureType (context, (StructureInstance)value, out typeInfo); diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index 97f554e4c25..d8e5f9ea162 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -57,7 +57,6 @@ const ApplicationConfig application_config = { .system_property_count = 0, .number_of_assemblies_in_apk = 2, .bundled_assembly_name_width = 0, - .number_of_assembly_store_files = 2, .number_of_dso_cache_entries = 2, .android_runtime_jnienv_class_token = 1, .jnienv_initialize_method_token = 2, @@ -113,18 +112,10 @@ AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[] = { }, }; -AssemblyStoreRuntimeData assembly_stores[] = { - { - .data_start = nullptr, - .assembly_count = 0, - .assemblies = nullptr, - }, - - { - .data_start = nullptr, - .assembly_count = 0, - .assemblies = nullptr, - }, +AssemblyStoreRuntimeData assembly_store = { + .data_start = nullptr, + .assembly_count = 0, + .assemblies = nullptr, }; constexpr char fake_dso_name[] = "libaot-Some.Assembly.dll.so"; diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index f7be28a631e..ee8fa0fa438 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -67,7 +67,8 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector c force_inline void EmbeddedAssemblies::map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept { - if (number_of_mapped_assembly_stores >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", application_config.number_of_assembly_store_files); + if (number_of_mapped_assembly_stores > number_of_assembly_store_files) { + log_fatal (LOG_ASSEMBLY, "Too many assembly stores. Expected at most %u", number_of_assembly_store_files); Helpers::abort_application (); } @@ -190,45 +191,17 @@ EmbeddedAssemblies::map_assembly_store (dynamic_local_string Helpers::abort_application (); } - if (header->store_id >= application_config.number_of_assembly_store_files) { - log_fatal ( - LOG_ASSEMBLY, - "Assembly store '%s' index %u exceeds the number of stores known at application build time, %u", - entry_name.get (), - header->store_id, - application_config.number_of_assembly_store_files - ); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[header->store_id]; - if (rd.data_start != nullptr) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' has a duplicate ID (%u)", entry_name.get (), header->store_id); - Helpers::abort_application (); - } - constexpr size_t header_size = sizeof(AssemblyStoreHeader); + constexpr size_t index_assembly_info_size = sizeof(AssemblyStoreIndexEntry) * 2; // We have two records for each assembly + const size_t index_size = (index_assembly_info_size * header->entry_count); - rd.data_start = static_cast(assembly_store_map.area); - rd.assembly_count = header->local_entry_count; - rd.assemblies = reinterpret_cast(rd.data_start + header_size); - - number_of_found_assemblies += rd.assembly_count; - - if (header->store_id == 0) { - constexpr size_t bundled_assembly_size = sizeof(AssemblyStoreAssemblyDescriptor); - constexpr size_t hash_entry_size = sizeof(AssemblyStoreHashEntry); - - index_assembly_store_header = header; - - size_t bytes_before_hashes = header_size + (bundled_assembly_size * header->local_entry_count); - if constexpr (std::is_same_v) { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes + (hash_entry_size * header->global_entry_count)); - } else { - assembly_store_hashes = reinterpret_cast(rd.data_start + bytes_before_hashes); - } - } + log_debug (LOG_ASSEMBLY, "Assembly store: index size == %zu; header size == %zu; entry size == %zu", index_size, header_size, index_assembly_info_size / 2); + assembly_store.data_start = static_cast(assembly_store_map.area); + assembly_store.assembly_count = header->entry_count; + assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + index_size); + assembly_store_hashes = reinterpret_cast(assembly_store.data_start + header_size); + number_of_found_assemblies += assembly_store.assembly_count; number_of_mapped_assembly_stores++; have_and_want_debug_symbols = register_debug_symbols; } @@ -241,28 +214,25 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& } dynamic_local_string entry_name; - bool common_assembly_store_found = false; - bool arch_assembly_store_found = false; + bool assembly_store_found = false; - log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK (common: '%s'; arch-specific: '%s')", assembly_store_common_file_name.data (), assembly_store_arch_file_name.data ()); + log_debug (LOG_ASSEMBLY, "Looking for assembly stores in APK ('%s)", assembly_store_file_name.data ()); for (size_t i = 0; i < num_entries; i++) { if (all_required_zip_entries_found ()) { need_to_scan_more_apks = false; break; } + // log_debug (LOG_ASSEMBLY, "ZIP entry: %s", entry_name.get ()); bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); if (!interesting_entry) { continue; } - if (!common_assembly_store_found && utils.ends_with (entry_name, assembly_store_common_file_name)) { - common_assembly_store_found = true; - map_assembly_store (entry_name, state); - } - - if (!arch_assembly_store_found && utils.ends_with (entry_name, assembly_store_arch_file_name)) { - arch_assembly_store_found = true; + // log_debug (LOG_ASSEMBLY, "Interesting entry: %s", entry_name.get ()); + if (!assembly_store_found && utils.ends_with (entry_name, assembly_store_file_name)) { + assembly_store_found = true; + // log_debug (LOG_ASSEMBLY, "Loading assembly store"); map_assembly_store (entry_name, state); } } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 82b7607f437..967f93a16b7 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -319,21 +319,15 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin return nullptr; } -force_inline const AssemblyStoreHashEntry* -EmbeddedAssemblies::find_assembly_store_entry ([[maybe_unused]] hash_t hash, [[maybe_unused]] const AssemblyStoreHashEntry *entries, [[maybe_unused]] size_t entry_count) noexcept +force_inline const AssemblyStoreIndexEntry* +EmbeddedAssemblies::find_assembly_store_entry ([[maybe_unused]] hash_t hash, [[maybe_unused]] const AssemblyStoreIndexEntry *entries, [[maybe_unused]] size_t entry_count) noexcept { #if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) - hash_t entry_hash; - const AssemblyStoreHashEntry *ret = nullptr; + const AssemblyStoreIndexEntry *ret = nullptr; while (entry_count > 0) { ret = entries + (entry_count / 2); - if constexpr (std::is_same_v) { - entry_hash = ret->hash64; - } else { - entry_hash = ret->hash32; - } - auto result = hash <=> entry_hash; + auto result = hash <=> ret->name_hash; if (result < 0) { entry_count /= 2; @@ -363,39 +357,34 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_stringmapping_index >= application_config.number_of_assemblies_in_apk) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->mapping_index, application_config.number_of_assemblies_in_apk - 1); + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #1"); + if (hash_entry->descriptor_index >= assembly_store.assembly_count) { + log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->descriptor_index, assembly_store.assembly_count - 1); Helpers::abort_application (); } - - AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[hash_entry->mapping_index]; + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #2 (descriptor_index: %u)", hash_entry->descriptor_index); + AssemblyStoreEntryDescriptor *store_entry = &assembly_store.assemblies[hash_entry->descriptor_index]; + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #3 (mapping_index: %u)", store_entry->mapping_index); + AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[store_entry->mapping_index]; + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #4"); if (assembly_runtime_info.image_data == nullptr) { - if (hash_entry->store_id >= application_config.number_of_assembly_store_files) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly store ID %u, exceeds the maximum of %u", hash_entry->store_id, application_config.number_of_assembly_store_files - 1); - Helpers::abort_application (); - } - - AssemblyStoreRuntimeData &rd = assembly_stores[hash_entry->store_id]; - if (hash_entry->local_store_index >= rd.assembly_count) { - log_fatal (LOG_ASSEMBLY, "Invalid index %u into local store assembly descriptor array", hash_entry->local_store_index); - Helpers::abort_application (); - } - - AssemblyStoreAssemblyDescriptor *bba = &rd.assemblies[hash_entry->local_store_index]; - + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #5"); // The assignments here don't need to be atomic, the value will always be the same, so even if two threads // arrive here at the same time, nothing bad will happen. - assembly_runtime_info.image_data = rd.data_start + bba->data_offset; - assembly_runtime_info.descriptor = bba; - - if (bba->debug_data_offset != 0) { - assembly_runtime_info.debug_info_data = rd.data_start + bba->debug_data_offset; + assembly_runtime_info.image_data = assembly_store.data_start + store_entry->data_offset; + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #6"); + assembly_runtime_info.descriptor = store_entry; + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #7"); + if (store_entry->debug_data_offset != 0) { + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #8"); + assembly_runtime_info.debug_info_data = assembly_store.data_start + store_entry->debug_data_offset; + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #9"); } #if !defined (NET) if (bba->config_data_size != 0) { @@ -408,7 +397,7 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_stringconfig_data_size, name.get () ); + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #11"); } + log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #12"); uint8_t *assembly_data; uint32_t assembly_data_size; diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 4b4c3324dae..72e2afab908 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -97,10 +97,9 @@ namespace xamarin::android::internal { static constexpr char assemblies_prefix[] = "assemblies/"; static constexpr char zip_path_separator[] = "/"; - static constexpr char assembly_store_prefix[] = "assemblies"; - static constexpr char assembly_store_extension[] = ".blob"; - static constexpr auto assembly_store_common_file_name = concat_const ("/", assembly_store_prefix, assembly_store_extension); - static constexpr auto assembly_store_arch_file_name = concat_const ("/", assembly_store_prefix, ".", SharedConstants::android_abi, assembly_store_extension); + static constexpr uint32_t number_of_assembly_store_files = 1; + static constexpr char assembly_store_prefix[] = "lib/"; + static constexpr auto assembly_store_file_name = concat_const (assembly_store_prefix, SharedConstants::android_lib_abi, "/assemblies.", SharedConstants::android_lib_abi, ".blob.so"); #if defined (DEBUG) || !defined (ANDROID) @@ -174,7 +173,7 @@ namespace xamarin::android::internal { return; } - abort_unless (index_assembly_store_header != nullptr && assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); + abort_unless (assembly_store_hashes != nullptr, "Invalid or incomplete assembly store data"); } private: @@ -268,18 +267,34 @@ namespace xamarin::android::internal { const char* get_assemblies_prefix () const { - return assemblies_prefix_override != nullptr ? assemblies_prefix_override : assemblies_prefix; + if (assemblies_prefix_override != nullptr) { + return assemblies_prefix_override; + } + + if (application_config.have_assembly_store) { + return assembly_store_prefix; + } + + return assemblies_prefix; } uint32_t get_assemblies_prefix_length () const noexcept { - return assemblies_prefix_override != nullptr ? static_cast(strlen (assemblies_prefix_override)) : sizeof(assemblies_prefix) - 1; + if (assemblies_prefix_override != nullptr) { + return static_cast(strlen (assemblies_prefix_override)); + } + + if (application_config.have_assembly_store) { + return sizeof(assembly_store_prefix) - 1; + } + + return sizeof(assemblies_prefix) - 1; } bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_assembly_stores == application_config.number_of_assembly_store_files + number_of_mapped_assembly_stores == number_of_assembly_store_files #if defined (NET) && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) #endif // NET @@ -307,7 +322,7 @@ namespace xamarin::android::internal { void set_assembly_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void set_debug_entry_data (XamarinAndroidBundledAssembly &entry, int apk_fd, uint32_t data_offset, uint32_t data_size, uint32_t prefix_len, uint32_t max_name_size, dynamic_local_string const& entry_name) noexcept; void map_assembly_store (dynamic_local_string const& entry_name, ZipEntryLoadState &state) noexcept; - const AssemblyStoreHashEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreHashEntry *entries, size_t entry_count) noexcept; + const AssemblyStoreIndexEntry* find_assembly_store_entry (hash_t hash, const AssemblyStoreIndexEntry *entries, size_t entry_count) noexcept; private: std::vector *bundled_debug_data = nullptr; @@ -332,8 +347,7 @@ namespace xamarin::android::internal { uint32_t number_of_mapped_assembly_stores = 0; bool need_to_scan_more_apks = true; - AssemblyStoreHeader *index_assembly_store_header = nullptr; - AssemblyStoreHashEntry *assembly_store_hashes; + AssemblyStoreIndexEntry *assembly_store_hashes; std::mutex assembly_decompress_mutex; }; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index d5f8f5ae86d..379219880d7 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -367,7 +367,6 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, int64_t apk_count = static_cast(runtimeApks.get_length ()); size_t prev_num_assemblies = 0; bool got_split_config_abi_apk = false; - bool got_base_apk = false; for (int64_t i = 0; i < apk_count; i++) { jstring_wrapper &apk_file = runtimeApks [static_cast(i)]; @@ -375,10 +374,10 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, if (have_split_apks) { bool scan_apk = false; + // With split configs we need to scan only the abi apk, because both the assembly stores and the runtime + // configuration blob are in `lib/{ARCH}`, which in turn lives in the split config APK if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), SharedConstants::split_config_abi_apk_name)) { got_split_config_abi_apk = scan_apk = true; - } else if (!got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { - got_base_apk = scan_apk = true; } if (!scan_apk) { diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index a88dbcdf569..b0b2d2856c8 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -32,7 +32,7 @@ namespace xamarin::android::internal static constexpr char DLL_EXTENSION[] = ".dll"; #if defined (NET) - static constexpr char RUNTIME_CONFIG_BLOB_NAME[] = "rc.bin"; + static constexpr char RUNTIME_CONFIG_BLOB_NAME[] = "arc.bin.so"; #endif // def NET #if defined (ANDROID) || defined (__linux__) || defined (__linux) @@ -51,15 +51,19 @@ namespace xamarin::android::internal #if __arm__ static constexpr char android_abi[] = "armeabi_v7a"; + static constexpr char android_lib_abi[] = "armeabi-v7a"; static constexpr char runtime_identifier[] = "android-arm"; #elif __aarch64__ static constexpr char android_abi[] = "arm64_v8a"; + static constexpr char android_lib_abi[] = "arm64-v8a"; static constexpr char runtime_identifier[] = "android-arm64"; #elif __x86_64__ static constexpr char android_abi[] = "x86_64"; + static constexpr char android_lib_abi[] = "x86_64"; static constexpr char runtime_identifier[] = "android-x64"; #elif __i386__ static constexpr char android_abi[] = "x86"; + static constexpr char android_lib_abi[] = "x86"; static constexpr char runtime_identifier[] = "android-x86"; #endif diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index a125c5aaf42..92e3ac298f7 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -149,31 +149,19 @@ struct [[gnu::packed]] AssemblyStoreHeader final { uint32_t magic; uint32_t version; - uint32_t local_entry_count; - uint32_t global_entry_count; - uint32_t store_id; + uint32_t entry_count; }; -struct [[gnu::packed]] AssemblyStoreHashEntry final +struct [[gnu::packed]] AssemblyStoreIndexEntry final { - union { - uint64_t hash64; - uint32_t hash32; - }; - - // Index into the array with pointers to assembly data. - // It **must** be unique across all the stores from all the apks - uint32_t mapping_index; - - // Index into the array with assembly descriptors inside a store - uint32_t local_store_index; - - // Index into the array with assembly store mmap addresses - uint32_t store_id; + xamarin::android::hash_t name_hash; + uint32_t descriptor_index; }; -struct [[gnu::packed]] AssemblyStoreAssemblyDescriptor final +struct [[gnu::packed]] AssemblyStoreEntryDescriptor final { + uint32_t mapping_index; + uint32_t data_offset; uint32_t data_size; @@ -188,7 +176,7 @@ struct AssemblyStoreRuntimeData final { uint8_t *data_start; uint32_t assembly_count; - AssemblyStoreAssemblyDescriptor *assemblies; + AssemblyStoreEntryDescriptor *assemblies; }; struct AssemblyStoreSingleAssemblyRuntimeData final @@ -196,7 +184,7 @@ struct AssemblyStoreSingleAssemblyRuntimeData final uint8_t *image_data; uint8_t *debug_info_data; uint8_t *config_data; - AssemblyStoreAssemblyDescriptor *descriptor; + AssemblyStoreEntryDescriptor *descriptor; }; enum class MonoComponent : uint32_t @@ -225,7 +213,6 @@ struct ApplicationConfig uint32_t system_property_count; uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; - uint32_t number_of_assembly_store_files; uint32_t number_of_dso_cache_entries; uint32_t android_runtime_jnienv_class_token; uint32_t jnienv_initialize_method_token; @@ -304,7 +291,7 @@ MONO_API MONO_API_EXPORT const char* const mono_aot_mode_name; MONO_API MONO_API_EXPORT XamarinAndroidBundledAssembly bundled_assemblies[]; MONO_API MONO_API_EXPORT AssemblyStoreSingleAssemblyRuntimeData assembly_store_bundled_assemblies[]; -MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_stores[]; +MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_store; MONO_API MONO_API_EXPORT DSOCacheEntry dso_cache[]; From 6dd5b922b73530893485cc2abdee9b16ffa210ef Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 7 Nov 2023 17:59:34 +0100 Subject: [PATCH 005/143] Release+assembly stores works fine now --- ...pplicationConfigNativeAssemblyGenerator.cs | 16 +++-- .../AssemblyStoreGenerator.Classes.cs | 12 +++- .../Utilities/AssemblyStoreGenerator.cs | 6 +- src/monodroid/jni/embedded-assemblies-zip.cc | 13 ++-- src/monodroid/jni/embedded-assemblies.cc | 71 +++++-------------- src/monodroid/jni/embedded-assemblies.hh | 4 +- src/monodroid/jni/mono-image-loader.hh | 2 +- src/monodroid/jni/monodroid-glue.cc | 31 +++----- src/monodroid/jni/search.hh | 20 +++++- src/monodroid/jni/xamarin-app.hh | 3 + 10 files changed, 82 insertions(+), 96 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index fa511dd408a..bf153479cc2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -28,7 +28,7 @@ public override string GetComment (object data, string fieldName) { var dso_entry = EnsureType (data); if (String.Compare ("hash", fieldName, StringComparison.Ordinal) == 0) { - return $" hash 0x{dso_entry.hash:x}, from name: {dso_entry.HashedName}"; + return $" from name: {dso_entry.HashedName}"; } if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { @@ -48,7 +48,7 @@ sealed class DSOCacheEntry [NativeAssembler (Ignore = true)] public string HashedName; - [NativeAssembler (UsesDataProvider = true)] + [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public ulong hash; public bool ignore; @@ -92,9 +92,10 @@ sealed class AssemblyStoreSingleAssemblyRuntimeData // src/monodroid/jni/xamarin-app.hh AssemblyStoreRuntimeData structure sealed class AssemblyStoreRuntimeData { - [NativePointer] + [NativePointer (IsNull = true)] public byte data_start; public uint assembly_count; + public uint index_entry_count; [NativePointer (IsNull = true)] public AssemblyStoreAssemblyDescriptor assemblies; @@ -354,7 +355,14 @@ void AddNameMutations (string name) { nameMutations.Add (name); if (name.EndsWith (".dll.so", StringComparison.OrdinalIgnoreCase)) { - nameMutations.Add (Path.GetFileNameWithoutExtension (Path.GetFileNameWithoutExtension (name))!); + string nameNoExt = Path.GetFileNameWithoutExtension (Path.GetFileNameWithoutExtension (name))!; + nameMutations.Add (nameNoExt); + + // This helps us at runtime, because sometimes MonoVM will ask for "AssemblyName" and sometimes for "AssemblyName.dll". + // In the former case, the runtime would ask for the "libaot-AssemblyName.so" image, which doesn't exist - we have + // "libaot-AssemblyName.dll.so" instead and, thus, we are forced to check for and append the missing ".dll" extension when + // loading the assembly, unnecessarily wasting time. + nameMutations.Add ($"{nameNoExt}.so"); } else { nameMutations.Add (Path.GetFileNameWithoutExtension (name)!); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs index cd513597383..18877bc0a7a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs @@ -4,16 +4,22 @@ partial class AssemblyStoreGenerator { sealed class AssemblyStoreHeader { - public const uint NativeSize = 3 * sizeof (uint); + public const uint NativeSize = 5 * sizeof (uint); public readonly uint magic = ASSEMBLY_STORE_MAGIC; public readonly uint version; - public readonly uint entry_count = 0; + public readonly uint entry_count; + public readonly uint index_entry_count; - public AssemblyStoreHeader (uint version, uint entry_count) + // Index size in bytes + public readonly uint index_size; + + public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_count, uint index_size) { this.version = version; this.entry_count = entry_count; + this.index_entry_count = index_entry_count; + this.index_size = index_size; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index f70ac0f7c67..083161235d5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -24,6 +24,8 @@ namespace Xamarin.Android.Tasks; // [MAGIC] uint; value: 0x41424158 // [FORMAT] uint; store format version number // [ENTRY_COUNT] uint; number of entries in the store +// [INDEX_ENTRY_COUNT] uint; number of entries in the index +// [INDEX_SIZE] uint; index size in bytes // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) // [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name @@ -126,11 +128,13 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List a.name_hash.CompareTo (b.name_hash)); diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index ee8fa0fa438..4c94d2977dc 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -67,8 +67,7 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector } constexpr size_t header_size = sizeof(AssemblyStoreHeader); - constexpr size_t index_assembly_info_size = sizeof(AssemblyStoreIndexEntry) * 2; // We have two records for each assembly - const size_t index_size = (index_assembly_info_size * header->entry_count); - log_debug (LOG_ASSEMBLY, "Assembly store: index size == %zu; header size == %zu; entry size == %zu", index_size, header_size, index_assembly_info_size / 2); assembly_store.data_start = static_cast(assembly_store_map.area); assembly_store.assembly_count = header->entry_count; - assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + index_size); + assembly_store.index_entry_count = header->index_entry_count; + assembly_store.assemblies = reinterpret_cast(assembly_store.data_start + header_size + header->index_size); assembly_store_hashes = reinterpret_cast(assembly_store.data_start + header_size); number_of_found_assemblies += assembly_store.assembly_count; @@ -223,16 +220,14 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& break; } - // log_debug (LOG_ASSEMBLY, "ZIP entry: %s", entry_name.get ()); bool interesting_entry = zip_load_entry_common (i, buf, entry_name, state); if (!interesting_entry) { continue; } - // log_debug (LOG_ASSEMBLY, "Interesting entry: %s", entry_name.get ()); + log_debug (LOG_ASSEMBLY, "Interesting entry: %s", entry_name.get ()); if (!assembly_store_found && utils.ends_with (entry_name, assembly_store_file_name)) { assembly_store_found = true; - // log_debug (LOG_ASSEMBLY, "Loading assembly store"); map_assembly_store (entry_name, state); } } diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 967f93a16b7..45f67507f8b 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -320,25 +320,14 @@ EmbeddedAssemblies::individual_assemblies_open_from_bundles (dynamic_local_strin } force_inline const AssemblyStoreIndexEntry* -EmbeddedAssemblies::find_assembly_store_entry ([[maybe_unused]] hash_t hash, [[maybe_unused]] const AssemblyStoreIndexEntry *entries, [[maybe_unused]] size_t entry_count) noexcept +EmbeddedAssemblies::find_assembly_store_entry (hash_t hash, const AssemblyStoreIndexEntry *entries, size_t entry_count) noexcept { -#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) - const AssemblyStoreIndexEntry *ret = nullptr; - - while (entry_count > 0) { - ret = entries + (entry_count / 2); - auto result = hash <=> ret->name_hash; - - if (result < 0) { - entry_count /= 2; - } else if (result > 0) { - entries = ret + 1; - entry_count -= entry_count / 2 + 1; - } else { - return ret; - } + auto equal = [](AssemblyStoreIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash == key; }; + auto less_than = [](AssemblyStoreIndexEntry const& entry, hash_t key) -> bool { return entry.name_hash < key; }; + ssize_t idx = Search::binary_search (hash, entries, entry_count); + if (idx >= 0) { + return &entries[idx]; } -#endif // ndef __MINGW32__ || (def __MINGW32__ && __GNUC__ >= 10) return nullptr; } @@ -347,44 +336,30 @@ template force_inline MonoAssembly* EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string& name, TLoaderData loader_data, bool ref_only) noexcept { - size_t len = name.length (); - bool have_dll_ext = utils.ends_with (name, SharedConstants::DLL_EXTENSION); - - if (have_dll_ext) { - len -= sizeof(SharedConstants::DLL_EXTENSION) - 1; - } - - hash_t name_hash = xxhash::hash (name.get (), len); + hash_t name_hash = xxhash::hash (name.get (), name.length ()); log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: looking for bundled name: '%s' (hash 0x%zx)", name.get (), name_hash); - const AssemblyStoreIndexEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, application_config.number_of_assemblies_in_apk); - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #0"); + const AssemblyStoreIndexEntry *hash_entry = find_assembly_store_entry (name_hash, assembly_store_hashes, assembly_store.index_entry_count); if (hash_entry == nullptr) { log_warn (LOG_ASSEMBLY, "Assembly '%s' (hash 0x%zx) not found", name.get (), name_hash); return nullptr; } - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #1"); + if (hash_entry->descriptor_index >= assembly_store.assembly_count) { - log_fatal (LOG_ASSEMBLY, "Invalid assembly index %u, exceeds the maximum index of %u", hash_entry->descriptor_index, assembly_store.assembly_count - 1); + log_fatal (LOG_ASSEMBLY, "Invalid assembly descriptor index %u, exceeds the maximum value of %u", hash_entry->descriptor_index, assembly_store.assembly_count - 1); Helpers::abort_application (); } - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #2 (descriptor_index: %u)", hash_entry->descriptor_index); - AssemblyStoreEntryDescriptor *store_entry = &assembly_store.assemblies[hash_entry->descriptor_index]; - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #3 (mapping_index: %u)", store_entry->mapping_index); - AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[store_entry->mapping_index]; - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #4"); + + AssemblyStoreEntryDescriptor &store_entry = assembly_store.assemblies[hash_entry->descriptor_index]; + AssemblyStoreSingleAssemblyRuntimeData &assembly_runtime_info = assembly_store_bundled_assemblies[store_entry.mapping_index]; + if (assembly_runtime_info.image_data == nullptr) { - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #5"); // The assignments here don't need to be atomic, the value will always be the same, so even if two threads // arrive here at the same time, nothing bad will happen. - assembly_runtime_info.image_data = assembly_store.data_start + store_entry->data_offset; - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #6"); - assembly_runtime_info.descriptor = store_entry; - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #7"); - if (store_entry->debug_data_offset != 0) { - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #8"); - assembly_runtime_info.debug_info_data = assembly_store.data_start + store_entry->debug_data_offset; - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #9"); + assembly_runtime_info.image_data = assembly_store.data_start + store_entry.data_offset; + assembly_runtime_info.descriptor = &store_entry; + if (store_entry.debug_data_offset != 0) { + assembly_runtime_info.debug_info_data = assembly_store.data_start + store_entry.debug_data_offset; } #if !defined (NET) if (bba->config_data_size != 0) { @@ -397,7 +372,6 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_stringconfig_data_size, name.get () ); - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #11"); } - log_debug (LOG_ASSEMBLY, "assembly_store_open_from_bundles: #12"); uint8_t *assembly_data; uint32_t assembly_data_size; - if (!have_dll_ext) { - // AOT needs this since Mono will form the DSO name by appending the .so suffix to the assembly name passed to - // functions below and `monodroid_dlopen` uses the `.dll.so` extension to check whether we're being asked to load - // the AOTd code for an assembly. - name.append (SharedConstants::DLL_EXTENSION); - } - get_assembly_data (assembly_runtime_info, assembly_data, assembly_data_size); MonoImage *image = MonoImageLoader::load (name, loader_data, name_hash, assembly_data, assembly_data_size); if (image == nullptr) { diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 72e2afab908..3bab5e4a269 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -97,6 +97,8 @@ namespace xamarin::android::internal { static constexpr char assemblies_prefix[] = "assemblies/"; static constexpr char zip_path_separator[] = "/"; + // We have two records for each assembly, for names with and without the extension + static constexpr uint32_t assembly_store_index_entries_per_assembly = 2; static constexpr uint32_t number_of_assembly_store_files = 1; static constexpr char assembly_store_prefix[] = "lib/"; static constexpr auto assembly_store_file_name = concat_const (assembly_store_prefix, SharedConstants::android_lib_abi, "/assemblies.", SharedConstants::android_lib_abi, ".blob.so"); @@ -347,7 +349,7 @@ namespace xamarin::android::internal { uint32_t number_of_mapped_assembly_stores = 0; bool need_to_scan_more_apks = true; - AssemblyStoreIndexEntry *assembly_store_hashes; + AssemblyStoreIndexEntry *assembly_store_hashes; std::mutex assembly_decompress_mutex; }; } diff --git a/src/monodroid/jni/mono-image-loader.hh b/src/monodroid/jni/mono-image-loader.hh index 87bcbc2ed3a..66a73a1afed 100644 --- a/src/monodroid/jni/mono-image-loader.hh +++ b/src/monodroid/jni/mono-image-loader.hh @@ -134,7 +134,7 @@ namespace xamarin::android::internal { } #if defined (USE_CACHE) - static inline size_t number_of_cache_index_entries = application_config.number_of_assemblies_in_apk * number_of_assembly_name_forms_in_image_cache;; + static inline size_t number_of_cache_index_entries = application_config.number_of_assemblies_in_apk * number_of_assembly_name_forms_in_image_cache; #endif // def USE_CACHE }; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index 379219880d7..b81a056fa21 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -367,6 +367,7 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, int64_t apk_count = static_cast(runtimeApks.get_length ()); size_t prev_num_assemblies = 0; bool got_split_config_abi_apk = false; + bool got_base_apk = false; for (int64_t i = 0; i < apk_count; i++) { jstring_wrapper &apk_file = runtimeApks [static_cast(i)]; @@ -378,6 +379,8 @@ MonodroidRuntime::gather_bundled_assemblies (jstring_array_wrapper &runtimeApks, // configuration blob are in `lib/{ARCH}`, which in turn lives in the split config APK if (!got_split_config_abi_apk && utils.ends_with (apk_file.get_cstr (), SharedConstants::split_config_abi_apk_name)) { got_split_config_abi_apk = scan_apk = true; + } else if (!application_config.have_assembly_store && !got_base_apk && utils.ends_with (apk_file.get_cstr (), base_apk_name)) { + got_base_apk = scan_apk = true; } if (!scan_apk) { @@ -1262,29 +1265,15 @@ MonodroidRuntime::init_internal_api_dso (void *handle) #endif // ndef NET force_inline DSOCacheEntry* -MonodroidRuntime::find_dso_cache_entry ([[maybe_unused]] hash_t hash) noexcept +MonodroidRuntime::find_dso_cache_entry (hash_t hash) noexcept { -#if !defined (__MINGW32__) || (defined (__MINGW32__) && __GNUC__ >= 10) - hash_t entry_hash; - DSOCacheEntry *ret = nullptr; - size_t entry_count = application_config.number_of_dso_cache_entries; - DSOCacheEntry *entries = dso_cache; - - while (entry_count > 0) { - ret = entries + (entry_count / 2); - entry_hash = static_cast (ret->hash); - auto result = hash <=> entry_hash; - - if (result < 0) { - entry_count /= 2; - } else if (result > 0) { - entries = ret + 1; - entry_count -= entry_count / 2 + 1; - } else { - return ret; - } + auto equal = [](DSOCacheEntry const& entry, hash_t key) -> bool { return entry.hash == key; }; + auto less_than = [](DSOCacheEntry const& entry, hash_t key) -> bool { return entry.hash < key; }; + ssize_t idx = Search::binary_search (hash, dso_cache, application_config.number_of_dso_cache_entries); + if (idx >= 0) { + return &dso_cache[idx]; } -#endif // ndef MINGW32 || def MINGW32 && GNUC >= 10 + return nullptr; } diff --git a/src/monodroid/jni/search.hh b/src/monodroid/jni/search.hh index 554126d2bee..a933953056f 100644 --- a/src/monodroid/jni/search.hh +++ b/src/monodroid/jni/search.hh @@ -6,26 +6,40 @@ #include "platform-compat.hh" #include "xxhash.hh" +#include "logger.hh" namespace xamarin::android::internal { class Search final { public: - force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + template + force_inline static ssize_t binary_search (hash_t key, const T *arr, size_t n) noexcept { + static_assert (equal != nullptr, "equal is a required template parameter"); + static_assert (less_than != nullptr, "less_than is a required template parameter"); + ssize_t left = -1; ssize_t right = static_cast(n); while (right - left > 1) { ssize_t middle = (left + right) >> 1; - if (arr[middle] < key) { + if (less_than (arr[middle], key)) { left = middle; } else { right = middle; } } - return arr[right] == key ? right : -1; + return equal (arr[right], key) ? right : -1; + } + + force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept + { + log_debug (LOG_ASSEMBLY, "binary search over %zu entries for hash 0x%zx", n, key); + auto equal = [](hash_t const& entry, hash_t key) -> bool { return entry == key; }; + auto less_than = [](hash_t const& entry, hash_t key) -> bool { return entry < key; }; + + return binary_search (key, arr, n); } force_inline static ptrdiff_t binary_search_branchless (hash_t x, const hash_t *arr, uint32_t len) noexcept diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 92e3ac298f7..df989a63b83 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -150,6 +150,8 @@ struct [[gnu::packed]] AssemblyStoreHeader final uint32_t magic; uint32_t version; uint32_t entry_count; + uint32_t index_entry_count; + uint32_t index_size; // index size in bytes }; struct [[gnu::packed]] AssemblyStoreIndexEntry final @@ -176,6 +178,7 @@ struct AssemblyStoreRuntimeData final { uint8_t *data_start; uint32_t assembly_count; + uint32_t index_entry_count; AssemblyStoreEntryDescriptor *assemblies; }; From 41243e50c3b643f1c2c9f1bdcbb498afae3b8a62 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 7 Nov 2023 18:54:03 +0100 Subject: [PATCH 006/143] Fix handling of discrete dlls in Release builds --- .../Tasks/BuildApk.cs | 37 ++++++++++--------- .../Tasks/GeneratePackageManagerJava.cs | 21 ++--------- .../Utilities/MonoAndroidHelper.cs | 8 ++-- src/monodroid/jni/embedded-assemblies-zip.cc | 2 +- src/monodroid/jni/embedded-assemblies.hh | 6 +-- 5 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 27eb621cc12..a518e6bc02a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -382,6 +382,23 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary libRootPath + abi + "/" + fileName; @@ -570,8 +572,9 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi /// string GetAssemblyPath (ITaskItem assembly, bool frameworkAssembly) { - var assembliesPath = AssembliesPath; var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + string assembliesPath = AssembliesPath + "/" + abi + "/"; if (!string.IsNullOrEmpty (subDirectory)) { assembliesPath += subDirectory.Replace ('\\', '/'); if (!assembliesPath.EndsWith ("/", StringComparison.Ordinal)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 73f5a533426..3731bc46eae 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -269,25 +269,12 @@ void AddEnvironment () uniqueAssemblyNames.Add (assemblyName); } - if (!UseAssemblyStore) { - assemblyCount++; - return; - } - - if (Boolean.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - return; - } + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); - string abi = assembly.GetMetadata ("Abi"); - if (String.IsNullOrEmpty (abi)) { + if (!archAssemblyNames.Contains (assemblyName)) { assemblyCount++; - } else { - archAssemblyNames ??= new HashSet (StringComparer.OrdinalIgnoreCase); - - if (!archAssemblyNames.Contains (assemblyName)) { - assemblyCount++; - archAssemblyNames.Add (assemblyName); - } + archAssemblyNames.Add (assemblyName); } }; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 984e8190261..94bee97d49d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -615,16 +615,18 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) return apiLevel; } - public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) + public static string GetAssemblyAbi (ITaskItem asmItem) { string? abi = asmItem.GetMetadata ("Abi"); if (String.IsNullOrEmpty (abi)) { - return AndroidTargetArch.None; + throw new InvalidOperationException ($"Internal error: assembly '{asmItem}' lacks ABI metadata"); } - return AbiToTargetArch (abi); + return abi; } + public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) => AbiToTargetArch (GetAssemblyAbi (asmItem)); + static string GetToolsRootDirectoryRelativePath (string androidBinUtilsDirectory) { // We need to link against libc and libm, but since NDK is not in use, the linker won't be able to find the actual Android libraries. diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 4c94d2977dc..0f780c8f642 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -147,7 +147,7 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c if (XA_UNLIKELY (bundled_assembly_index >= application_config.number_of_assemblies_in_apk || bundled_assemblies_slow_path)) { if (!bundled_assemblies_slow_path && bundled_assembly_index == application_config.number_of_assemblies_in_apk) { - log_warn (LOG_ASSEMBLY, "Number of assemblies stored at build time (%u) was incorrect, switching to slow bundling path."); + log_warn (LOG_ASSEMBLY, "Number of assemblies stored at build time (%u) was incorrect, switching to slow bundling path.", application_config.number_of_assemblies_in_apk); } if (extra_bundled_assemblies == nullptr) { diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 3bab5e4a269..9d3a6646f8b 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -94,7 +94,7 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_EOCD_LEN = 22; static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; - static constexpr char assemblies_prefix[] = "assemblies/"; + static constexpr auto assemblies_prefix = concat_const ("assemblies/", SharedConstants::android_lib_abi, "/"); static constexpr char zip_path_separator[] = "/"; // We have two records for each assembly, for names with and without the extension @@ -277,7 +277,7 @@ namespace xamarin::android::internal { return assembly_store_prefix; } - return assemblies_prefix; + return assemblies_prefix.data (); } uint32_t get_assemblies_prefix_length () const noexcept @@ -290,7 +290,7 @@ namespace xamarin::android::internal { return sizeof(assembly_store_prefix) - 1; } - return sizeof(assemblies_prefix) - 1; + return assemblies_prefix.size () - 1; } bool all_required_zip_entries_found () const noexcept From e46460cbdc4c321014e52193b910c4b56350e4ac Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 7 Nov 2023 21:34:34 +0100 Subject: [PATCH 007/143] Prepare for experimental dlopen code --- .../Utilities/ApplicationConfig.cs | 1 + ...pplicationConfigNativeAssemblyGenerator.cs | 1 + src/monodroid/jni/embedded-assemblies-zip.cc | 27 +++++++++++++--- src/monodroid/jni/embedded-assemblies.hh | 32 +++++++------------ src/monodroid/jni/xamarin-app.hh | 6 ++++ 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 7dc580dcee7..8ff364520fa 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -41,6 +41,7 @@ sealed class ApplicationConfig public uint number_of_assemblies_in_apk; public uint bundled_assembly_name_width; public uint number_of_dso_cache_entries; + public uint number_of_shared_libraries; public uint android_runtime_jnienv_class_token; public uint jnienv_initialize_method_token; public uint jnienv_registerjninatives_method_token; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index bf153479cc2..f18784eb565 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -218,6 +218,7 @@ protected override void Construct (LlvmIrModule module) environment_variable_count = (uint)(environmentVariables == null ? 0 : environmentVariables.Count * 2), system_property_count = (uint)(systemProperties == null ? 0 : systemProperties.Count * 2), number_of_assemblies_in_apk = (uint)NumberOfAssembliesInApk, + number_of_shared_libraries = (uint)NativeLibraries.Count, bundled_assembly_name_width = (uint)BundledAssemblyNameWidth, number_of_dso_cache_entries = (uint)dsoCache.Count, android_runtime_jnienv_class_token = (uint)AndroidRuntimeJNIEnvToken, diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index 0f780c8f642..d523ae615ee 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -61,8 +61,14 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& if (!assembly_store_found && utils.ends_with (entry_name, assembly_store_file_name)) { assembly_store_found = true; map_assembly_store (entry_name, state); + continue; + } + + if (number_of_zip_dso_entries >= application_config.number_of_shared_libraries) { + continue; + } + + // Since it's not an assembly store, it's a shared library most likely and it is long enough for us not to have + // to check the length + if (utils.ends_with (entry_name, dso_suffix)) { + log_debug (LOG_ASSEMBLY, "Found a shared library: %s", entry_name.get ()); + number_of_zip_dso_entries++; } } } @@ -256,11 +274,12 @@ EmbeddedAssemblies::zip_load_entries (int fd, const char *apk_name, [[maybe_unus } std::vector buf (cd_size); + const auto [prefix, prefix_len] = get_assemblies_prefix_and_length (); ZipEntryLoadState state { .apk_fd = fd, .apk_name = apk_name, - .prefix = get_assemblies_prefix (), - .prefix_len = get_assemblies_prefix_length (), + .prefix = prefix, + .prefix_len = prefix_len, .buf_offset = 0, .compression_method = 0, .local_header_offset = 0, diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 9d3a6646f8b..d39fef2e063 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -94,14 +95,15 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_EOCD_LEN = 22; static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; - static constexpr auto assemblies_prefix = concat_const ("assemblies/", SharedConstants::android_lib_abi, "/"); static constexpr char zip_path_separator[] = "/"; + static constexpr auto assemblies_prefix = concat_const ("assemblies", zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); // We have two records for each assembly, for names with and without the extension static constexpr uint32_t assembly_store_index_entries_per_assembly = 2; static constexpr uint32_t number_of_assembly_store_files = 1; - static constexpr char assembly_store_prefix[] = "lib/"; - static constexpr auto assembly_store_file_name = concat_const (assembly_store_prefix, SharedConstants::android_lib_abi, "/assemblies.", SharedConstants::android_lib_abi, ".blob.so"); + static constexpr char dso_suffix[] = ".so"; + static constexpr char apk_lib_prefix[] = "lib/"; + static constexpr auto assembly_store_file_name = concat_const (apk_lib_prefix, SharedConstants::android_lib_abi, zip_path_separator, "assemblies.", SharedConstants::android_lib_abi, ".blob.so"); #if defined (DEBUG) || !defined (ANDROID) @@ -267,36 +269,23 @@ namespace xamarin::android::internal { bool zip_read_entry_info (std::vector const& buf, dynamic_local_string& file_name, ZipEntryLoadState &state); - const char* get_assemblies_prefix () const + std::tuple get_assemblies_prefix_and_length () const noexcept { if (assemblies_prefix_override != nullptr) { - return assemblies_prefix_override; + return { assemblies_prefix_override, static_cast(strlen (assemblies_prefix_override)) }; } if (application_config.have_assembly_store) { - return assembly_store_prefix; + return { apk_lib_prefix, sizeof(apk_lib_prefix) - 1 }; } - return assemblies_prefix.data (); - } - - uint32_t get_assemblies_prefix_length () const noexcept - { - if (assemblies_prefix_override != nullptr) { - return static_cast(strlen (assemblies_prefix_override)); - } - - if (application_config.have_assembly_store) { - return sizeof(assembly_store_prefix) - 1; - } - - return assemblies_prefix.size () - 1; + return {assemblies_prefix.data (), assemblies_prefix.size () - 1}; } bool all_required_zip_entries_found () const noexcept { return - number_of_mapped_assembly_stores == number_of_assembly_store_files + number_of_mapped_assembly_stores == number_of_assembly_store_files && number_of_zip_dso_entries >= application_config.number_of_shared_libraries #if defined (NET) && ((application_config.have_runtime_config_blob && runtime_config_blob_found) || !application_config.have_runtime_config_blob) #endif // NET @@ -347,6 +336,7 @@ namespace xamarin::android::internal { bool runtime_config_blob_found = false; #endif // def NET uint32_t number_of_mapped_assembly_stores = 0; + uint32_t number_of_zip_dso_entries = 0; bool need_to_scan_more_apks = true; AssemblyStoreIndexEntry *assembly_store_hashes; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index df989a63b83..f212f040db1 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -217,6 +217,7 @@ struct ApplicationConfig uint32_t number_of_assemblies_in_apk; uint32_t bundled_assembly_name_width; uint32_t number_of_dso_cache_entries; + uint32_t number_of_shared_libraries; uint32_t android_runtime_jnienv_class_token; uint32_t jnienv_initialize_method_token; uint32_t jnienv_registerjninatives_method_token; @@ -226,6 +227,11 @@ struct ApplicationConfig const char *android_package_name; }; +struct DSOApkEntry +{ + uint32_t offset; // offset into the APK +}; + struct DSOCacheEntry { uint64_t hash; From 9c1b501f085197f412b0a325b08ac97dbc4b5d9e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 8 Nov 2023 22:17:06 +0100 Subject: [PATCH 008/143] New dlopen code, fixes for handling of compressed assemblies --- .../Tasks/BuildApk.cs | 10 +- ...teCompressedAssembliesNativeSourceFiles.cs | 17 +-- ...pplicationConfigNativeAssemblyGenerator.cs | 22 +++- .../Utilities/AssemblyCompression.cs | 1 + .../Utilities/AssemblyStoreGenerator.cs | 13 ++- ...ressedAssembliesNativeAssemblyGenerator.cs | 109 +++++++++++++++--- .../Utilities/CompressedAssemblyInfo.cs | 17 ++- .../LlvmIrGenerator/LlvmIrVariable.cs | 7 +- .../MarshalMethodsNativeAssemblyGenerator.cs | 4 +- ...peMappingReleaseNativeAssemblyGenerator.cs | 2 +- src/monodroid/jni/application_dso_stub.cc | 4 + src/monodroid/jni/embedded-assemblies-zip.cc | 17 ++- src/monodroid/jni/embedded-assemblies.hh | 7 +- src/monodroid/jni/monodroid-glue.cc | 35 +++++- src/monodroid/jni/search.hh | 1 - src/monodroid/jni/xamarin-app.hh | 4 + 16 files changed, 218 insertions(+), 52 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index a518e6bc02a..ae398914e6d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -501,10 +501,12 @@ string CompressAssembly (ITaskItem assembly) EnsureCompressedAssemblyData (assembly.ItemSpec, info.DescriptorIndex); string assemblyOutputDir; string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - if (!String.IsNullOrEmpty (subDirectory)) - assemblyOutputDir = Path.Combine (compressedOutputDir, subDirectory); - else - assemblyOutputDir = compressedOutputDir; + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + if (!String.IsNullOrEmpty (subDirectory)) { + assemblyOutputDir = Path.Combine (compressedOutputDir, abi, subDirectory); + } else { + assemblyOutputDir = Path.Combine (compressedOutputDir, abi); + } AssemblyCompression.CompressionResult result = AssemblyCompression.Compress (compressedAssembly, assemblyOutputDir); if (result != AssemblyCompression.CompressionResult.Success) { switch (result) { diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs index 48f849596e0..ada3d4a0f8a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs @@ -4,6 +4,8 @@ using Microsoft.Build.Framework; using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; + namespace Xamarin.Android.Tasks { public class GenerateCompressedAssembliesNativeSourceFiles : AndroidTask @@ -41,7 +43,8 @@ void GenerateCompressedAssemblySources () return; } - var assemblies = new SortedDictionary (StringComparer.Ordinal); + var assemblies = new Dictionary (StringComparer.Ordinal); + var counters = new Dictionary (); foreach (ITaskItem assembly in ResolvedAssemblies) { if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { continue; @@ -59,12 +62,12 @@ void GenerateCompressedAssemblySources () continue; } - assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length))); - } - - uint index = 0; - foreach (var kvp in assemblies) { - kvp.Value.DescriptorIndex = index++; + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!counters.TryGetValue (arch, out uint counter)) { + counter = 0; + } + assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length), counter++, arch, Path.GetFileNameWithoutExtension (assembly.ItemSpec))); + counters[arch] = counter; } string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index f18784eb565..c65eced6f5e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -50,6 +50,9 @@ sealed class DSOCacheEntry [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public ulong hash; + + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] + public ulong real_name_hash; public bool ignore; [NativeAssembler (UsesDataProvider = true)] @@ -57,6 +60,13 @@ sealed class DSOCacheEntry public IntPtr handle = IntPtr.Zero; } + sealed class DSOApkEntry + { + public ulong name_hash; + public uint offset; // offset into the APK + public int fd; // apk file descriptor + }; + // Order of fields and their type must correspond *exactly* to that in // src/monodroid/jni/xamarin-app.hh AssemblyStoreAssemblyDescriptor structure sealed class AssemblyStoreAssemblyDescriptor @@ -143,6 +153,7 @@ sealed class XamarinAndroidBundledAssembly StructureInfo? applicationConfigStructureInfo; StructureInfo? dsoCacheEntryStructureInfo; + StructureInfo? dsoApkEntryStructureInfo; StructureInfo? xamarinAndroidBundledAssemblyStructureInfo; StructureInfo? assemblyStoreSingleAssemblyRuntimeDataStructureinfo; StructureInfo? assemblyStoreRuntimeDataStructureInfo; @@ -238,6 +249,13 @@ protected override void Construct (LlvmIrModule module) }; module.Add (dso_cache); + var dso_apk_entries = new LlvmIrGlobalVariable (typeof(List>), "dso_apk_entries") { + ArrayItemCount = (ulong)NativeLibraries.Count, + Options = LlvmIrVariableOptions.GlobalWritable, + ZeroInitializeArray = true, + }; + module.Add (dso_apk_entries); + if (!HaveAssemblyStore) { xamarinAndroidBundledAssemblies = new List> (NumberOfAssembliesInApk); @@ -291,7 +309,7 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob { var cache = variable.Value as List>; if (cache == null) { - throw new InvalidOperationException ($"Internal error: DSO cache must no be empty"); + throw new InvalidOperationException ($"Internal error: DSO cache must not be empty"); } bool is64Bit = target.Is64Bit; @@ -306,6 +324,7 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob } entry.hash = GetXxHash (entry.HashedName, is64Bit); + entry.real_name_hash = GetXxHash (entry.name, is64Bit); } cache.Sort ((StructureInstance a, StructureInstance b) => a.Instance.hash.CompareTo (b.Instance.hash)); @@ -388,6 +407,7 @@ void MapStructures (LlvmIrModule module) assemblyStoreRuntimeDataStructureInfo = module.MapStructure (); xamarinAndroidBundledAssemblyStructureInfo = module.MapStructure (); dsoCacheEntryStructureInfo = module.MapStructure (); + dsoApkEntryStructureInfo = module.MapStructure (); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index 1972dba0d85..57c40136717 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -69,6 +69,7 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect data.DestinationPath = Path.Combine (outputDirectory, $"{Path.GetFileName (data.SourcePath)}.lz4"); data.SourceSize = (uint)fi.Length; + Console.WriteLine ($"Compressing: {data.SourcePath} => {data.DestinationPath}; Index: {data.DescriptorIndex}"); byte[] sourceBytes = null; byte[] destBytes = null; try { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index 083161235d5..fab2ea06bb4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -112,10 +112,19 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List assemblies; StructureInfo compressedAssemblyDescriptorStructureInfo; StructureInfo compressedAssembliesStructureInfo; + Dictionary>> archData = new Dictionary>> (); public CompressedAssembliesNativeAssemblyGenerator (IDictionary assemblies) { this.assemblies = assemblies; } - void InitCompressedAssemblies (out List>? compressedAssemblyDescriptors, - out StructureInstance? compressedAssemblies, + void InitCompressedAssemblies (out StructureInstance? compressedAssemblies, out List? buffers) { if (assemblies == null || assemblies.Count == 0) { - compressedAssemblyDescriptors = null; compressedAssemblies = null; buffers = null; return; } - ulong counter = 0; - compressedAssemblyDescriptors = new List> (assemblies.Count); - buffers = new List (assemblies.Count); + AndroidTargetArch? firstArch = null; foreach (var kvp in assemblies) { - string assemblyName = kvp.Key; CompressedAssemblyInfo info = kvp.Value; - string bufferName = $"__compressedAssemblyData_{counter++}"; + if (firstArch == null) { + firstArch = info.TargetArch; + } + + if (!archData.TryGetValue (info.TargetArch, out List> descriptors)) { + descriptors = new List> (); + archData.Add (info.TargetArch, descriptors); + } + + string bufferName = $"__compressedAssemblyData_{info.DescriptorIndex}"; var descriptor = new CompressedAssemblyDescriptor { + Index = info.DescriptorIndex, BufferSymbolName = bufferName, + AssemblyName = info.AssemblyName, uncompressed_file_size = info.FileSize, loaded = false, data = 0 }; - var bufferVar = new LlvmIrGlobalVariable (typeof(List), bufferName, LlvmIrVariableOptions.LocalWritable) { - ZeroInitializeArray = true, - ArrayItemCount = descriptor.uncompressed_file_size, - }; - buffers.Add (bufferVar); + descriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); + } + + buffers = new List (); + uint assembliesCount = 0; + foreach (var kvp in archData) { + List> descriptors = kvp.Value; + descriptors.Sort ((StructureInstance a, StructureInstance b) => a.Instance.Index.CompareTo (b.Instance.Index)); + if (buffers.Count > 0) { + continue; + } - compressedAssemblyDescriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); + assembliesCount = (uint)descriptors.Count; + foreach (StructureInstance desc in descriptors) { + var buffer = new LlvmIrGlobalVariable (typeof(List), desc.Instance.BufferSymbolName, LlvmIrVariableOptions.LocalWritable) { + BeforeWriteCallback = BufferBeforeWrite, + BeforeWriteCallbackCallerState = desc.Instance.Index, + ZeroInitializeArray = true, + }; + buffers.Add (buffer); + } } - compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)assemblies.Count }); + compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = assembliesCount }); } protected override void Construct (LlvmIrModule module) { MapStructures (module); - List>? compressedAssemblyDescriptors; StructureInstance? compressedAssemblies; List? buffers; - InitCompressedAssemblies (out compressedAssemblyDescriptors, out compressedAssemblies, out buffers); + InitCompressedAssemblies (out compressedAssemblies, out buffers); - if (compressedAssemblyDescriptors == null) { + if (archData.Count == 0) { module.AddGlobalVariable ( typeof(StructureInstance), CompressedAssembliesSymbolName, @@ -130,13 +157,57 @@ protected override void Construct (LlvmIrModule module) } module.AddGlobalVariable (CompressedAssembliesSymbolName, compressedAssemblies, LlvmIrVariableOptions.GlobalWritable); - module.AddGlobalVariable (DescriptorsArraySymbolName, compressedAssemblyDescriptors, LlvmIrVariableOptions.LocalWritable); + + var compressedAssemblyDescriptors = new LlvmIrGlobalVariable (typeof(List>), DescriptorsArraySymbolName) { + BeforeWriteCallback = CompressedAssemblyDescriptorsBeforeWrite, + GetArrayItemCommentCallback = GetCompressedAssemblyDescriptorsItemComment, + Options = LlvmIrVariableOptions.LocalWritable, + }; + module.Add (compressedAssemblyDescriptors); module.Add (new LlvmIrGroupDelimiterVariable ()); module.Add (buffers); module.Add (new LlvmIrGroupDelimiterVariable ()); } + string? GetCompressedAssemblyDescriptorsItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) + { + List> descriptors = GetArchDescriptors (target); + if ((int)index >= descriptors.Count) { + throw new InvalidOperationException ($"Internal error: index {index} is too big for variable '{v.Name}'"); + } + StructureInstance desc = descriptors[(int)index]; + + return $" {index}: {desc.Instance.AssemblyName}"; + } + + void CompressedAssemblyDescriptorsBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + List> descriptors = GetArchDescriptors (target); + var gv = (LlvmIrGlobalVariable)variable; + gv.OverrideTypeAndValue (gv.Type, descriptors); + } + + void BufferBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) + { + List> descriptors = GetArchDescriptors (target); + uint descriptorIndex = (uint)state; + StructureInstance desc = descriptors[(int)descriptorIndex]; + var gv = (LlvmIrGlobalVariable)variable; + + gv.ArrayItemCount = desc.Instance.uncompressed_file_size; + gv.OverrideName (desc.Instance.BufferSymbolName); + } + + List> GetArchDescriptors (LlvmIrModuleTarget target) + { + if (!archData.TryGetValue (target.TargetArch, out List> descriptors)) { + throw new InvalidOperationException ($"Internal error: missing compressed descriptors data for architecture '{target.TargetArch}'"); + } + + return descriptors; + } + void MapStructures (LlvmIrModule module) { compressedAssemblyDescriptorStructureInfo = module.MapStructure (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs index b3f775a96b7..56cf36f92e5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs @@ -2,6 +2,8 @@ using System.IO; using Microsoft.Build.Framework; +using Xamarin.Android.Tools; + namespace Xamarin.Android.Tasks { class CompressedAssemblyInfo @@ -9,12 +11,16 @@ class CompressedAssemblyInfo const string CompressedAssembliesInfoKey = "__CompressedAssembliesInfo"; public uint FileSize { get; } - public uint DescriptorIndex { get; set; } + public uint DescriptorIndex { get; } + public AndroidTargetArch TargetArch { get; } + public string AssemblyName { get; } - public CompressedAssemblyInfo (uint fileSize) + public CompressedAssemblyInfo (uint fileSize, uint descriptorIndex, AndroidTargetArch targetArch, string assemblyName) { FileSize = fileSize; - DescriptorIndex = 0; + DescriptorIndex = descriptorIndex; + TargetArch = targetArch; + AssemblyName = assemblyName; } public static string GetKey (string projectFullPath) @@ -27,14 +33,15 @@ public static string GetKey (string projectFullPath) public static string GetDictionaryKey (ITaskItem assembly) { + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); // Prefer %(DestinationSubPath) if set var path = assembly.GetMetadata ("DestinationSubPath"); if (!string.IsNullOrEmpty (path)) { - return path; + return Path.Combine (abi, path); } // MSBuild sometimes only sets %(DestinationSubDirectory) var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - return Path.Combine (subDirectory, Path.GetFileName (assembly.ItemSpec)); + return Path.Combine (abi, subDirectory, Path.GetFileName (assembly.ItemSpec)); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 49524f0bf6c..cfc427ef47d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -274,11 +274,16 @@ public LlvmIrGlobalVariable (object value, string name, LlvmIrVariableOptions? o /// generating output for a specific target (e.g. 32-bit vs 64-bit integer variables). If the variable requires such /// type changes, this should be done at generation time from within the method. /// - public void OverrideValueAndType (Type newType, object? newValue) + public void OverrideTypeAndValue (Type newType, object? newValue) { Type = newType; Value = newValue; } + + public void OverrideName (string newName) + { + Name = newName; + } } class LlvmIrStringVariable : LlvmIrGlobalVariable diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index f87cf118971..92e557c50d0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -1047,7 +1047,7 @@ void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget } LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); - gv.OverrideValueAndType (type, value); + gv.OverrideTypeAndValue (type, value); } string? GetAssemblyImageCacheItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) @@ -1081,7 +1081,7 @@ void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarge } LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); - gv.OverrideValueAndType (variable.Type, value); + gv.OverrideTypeAndValue (variable.Type, value); } AssemblyCacheState EnsureAssemblyCacheState (object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index 81ca3cf006b..81a8930e1ec 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -291,7 +291,7 @@ void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget targ hashes = list; } - gv.OverrideValueAndType (listType, hashes); + gv.OverrideTypeAndValue (listType, hashes); } ConstructionState EnsureConstructionState (object? callerState) diff --git a/src/monodroid/jni/application_dso_stub.cc b/src/monodroid/jni/application_dso_stub.cc index d8e5f9ea162..db2d1a429e7 100644 --- a/src/monodroid/jni/application_dso_stub.cc +++ b/src/monodroid/jni/application_dso_stub.cc @@ -124,6 +124,7 @@ constexpr char fake_dso_name2[] = "libaot-Another.Assembly.dll.so"; DSOCacheEntry dso_cache[] = { { .hash = xamarin::android::xxhash::hash (fake_dso_name, sizeof(fake_dso_name) - 1), + .real_name_hash = xamarin::android::xxhash::hash (fake_dso_name, sizeof(fake_dso_name) - 1), .ignore = true, .name = fake_dso_name, .handle = nullptr, @@ -131,12 +132,15 @@ DSOCacheEntry dso_cache[] = { { .hash = xamarin::android::xxhash::hash (fake_dso_name2, sizeof(fake_dso_name2) - 1), + .real_name_hash = xamarin::android::xxhash::hash (fake_dso_name2, sizeof(fake_dso_name2) - 1), .ignore = true, .name = fake_dso_name2, .handle = nullptr, }, }; +DSOApkEntry dso_apk_entries[2] {}; + // // Support for marshal methods // diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index d523ae615ee..0ff1bdedd95 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -11,6 +11,7 @@ #include "cpp-util.hh" #include "globals.hh" #include "xamarin-app.hh" +#include "xxhash.hh" using namespace xamarin::android::internal; @@ -62,11 +63,11 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& continue; } - log_debug (LOG_ASSEMBLY, "Interesting entry: %s", entry_name.get ()); if (!assembly_store_found && utils.ends_with (entry_name, assembly_store_file_name)) { assembly_store_found = true; map_assembly_store (entry_name, state); @@ -245,7 +245,16 @@ EmbeddedAssemblies::zip_load_assembly_store_entries (std::vector const& // Since it's not an assembly store, it's a shared library most likely and it is long enough for us not to have // to check the length if (utils.ends_with (entry_name, dso_suffix)) { - log_debug (LOG_ASSEMBLY, "Found a shared library: %s", entry_name.get ()); + constexpr size_t apk_lib_prefix_len = apk_lib_prefix.size () - 1; + + const char *const name = entry_name.get () + apk_lib_prefix_len; + DSOApkEntry *apk_entry = reinterpret_cast(reinterpret_cast(dso_apk_entries) + (sizeof(DSOApkEntry) * number_of_zip_dso_entries)); + + apk_entry->name_hash = xxhash::hash (name, entry_name.length () - apk_lib_prefix_len); + apk_entry->offset = state.data_offset; + apk_entry->fd = state.apk_fd; + + log_debug (LOG_ASSEMBLY, "Found a shared library entry %s (index: %u; name: %s; hash: 0x%zx; apk offset: %u)", entry_name.get (), number_of_zip_dso_entries, name, apk_entry->name_hash, apk_entry->offset); number_of_zip_dso_entries++; } } diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index d39fef2e063..dea1847025f 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -102,8 +102,9 @@ namespace xamarin::android::internal { static constexpr uint32_t assembly_store_index_entries_per_assembly = 2; static constexpr uint32_t number_of_assembly_store_files = 1; static constexpr char dso_suffix[] = ".so"; - static constexpr char apk_lib_prefix[] = "lib/"; - static constexpr auto assembly_store_file_name = concat_const (apk_lib_prefix, SharedConstants::android_lib_abi, zip_path_separator, "assemblies.", SharedConstants::android_lib_abi, ".blob.so"); + static constexpr char apk_lib_dir_name[] = "lib"; + static constexpr auto apk_lib_prefix = concat_const (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); + static constexpr auto assembly_store_file_name = concat_const (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator, "assemblies.", SharedConstants::android_lib_abi, ".blob.so"); #if defined (DEBUG) || !defined (ANDROID) @@ -276,7 +277,7 @@ namespace xamarin::android::internal { } if (application_config.have_assembly_store) { - return { apk_lib_prefix, sizeof(apk_lib_prefix) - 1 }; + return { apk_lib_prefix.data (), apk_lib_prefix.size () - 1 }; } return {assemblies_prefix.data (), assemblies_prefix.size () - 1}; diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index b81a056fa21..f3006ca5bb3 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -16,10 +16,12 @@ #include #endif // def APPLE_OX_X +#include #include #include #include +#include #include #include @@ -1281,7 +1283,11 @@ force_inline void* MonodroidRuntime::monodroid_dlopen_log_and_return (void *handle, char **err, const char *full_name, bool free_memory, [[maybe_unused]] bool need_api_init) { if (handle == nullptr && err != nullptr) { - *err = utils.monodroid_strdup_printf ("Could not load library: Library '%s' not found.", full_name); + const char *load_error = dlerror (); + if (load_error == nullptr) { + load_error = "Unknown error"; + } + *err = utils.monodroid_strdup_printf ("Could not load library '%s'. %s", full_name, load_error); } if (free_memory) { @@ -1294,6 +1300,7 @@ MonodroidRuntime::monodroid_dlopen_log_and_return (void *handle, char **err, con init_internal_api_dso (handle); #endif // ndef NET + log_debug (LOG_ASSEMBLY, "DSO '%s' loaded", full_name); return handle; } @@ -1363,9 +1370,33 @@ MonodroidRuntime::monodroid_dlopen (const char *name, int flags, char **err) noe } StartupAwareLock lock (dso_handle_write_lock); - unsigned int dl_flags = monodroidRuntime.convert_dl_flags (flags); +#if defined (RELEASE) + if (androidSystem.is_embedded_dso_mode_enabled ()) { + DSOApkEntry *apk_entry = dso_apk_entries; + for (size_t i = 0; i < application_config.number_of_shared_libraries; i++) { + if (apk_entry->name_hash != dso->real_name_hash) { + apk_entry++; + continue; + } + + log_debug (LOG_ASSEMBLY, "Loading DSO '%s' from apk: offset == %u; fd == %d", dso->name, apk_entry->offset, apk_entry->fd); + + android_dlextinfo dli; + dli.flags = ANDROID_DLEXT_USE_LIBRARY_FD | ANDROID_DLEXT_USE_LIBRARY_FD_OFFSET; + dli.library_fd = apk_entry->fd; + dli.library_fd_offset = apk_entry->offset; + dso->handle = android_dlopen_ext (dso->name, flags, &dli); + if (dso->handle != nullptr) { + return monodroid_dlopen_log_and_return (dso->handle, err, dso->name, false /* name_needs_free */); + } + break; + } + } +#endif + unsigned int dl_flags = monodroidRuntime.convert_dl_flags (flags); dso->handle = androidSystem.load_dso_from_any_directories (dso->name, dl_flags); + if (dso->handle != nullptr) { return monodroid_dlopen_log_and_return (dso->handle, err, dso->name, false /* name_needs_free */); } diff --git a/src/monodroid/jni/search.hh b/src/monodroid/jni/search.hh index a933953056f..be6a24437e4 100644 --- a/src/monodroid/jni/search.hh +++ b/src/monodroid/jni/search.hh @@ -35,7 +35,6 @@ namespace xamarin::android::internal { force_inline static ssize_t binary_search (hash_t key, const hash_t *arr, size_t n) noexcept { - log_debug (LOG_ASSEMBLY, "binary search over %zu entries for hash 0x%zx", n, key); auto equal = [](hash_t const& entry, hash_t key) -> bool { return entry == key; }; auto less_than = [](hash_t const& entry, hash_t key) -> bool { return entry < key; }; diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index f212f040db1..8ead25f41a0 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -229,12 +229,15 @@ struct ApplicationConfig struct DSOApkEntry { + uint64_t name_hash; uint32_t offset; // offset into the APK + int32_t fd; // apk file descriptor }; struct DSOCacheEntry { uint64_t hash; + uint64_t real_name_hash; bool ignore; const char *name; void *handle; @@ -303,6 +306,7 @@ MONO_API MONO_API_EXPORT AssemblyStoreSingleAssemblyRuntimeData assembly_store_b MONO_API MONO_API_EXPORT AssemblyStoreRuntimeData assembly_store; MONO_API MONO_API_EXPORT DSOCacheEntry dso_cache[]; +MONO_API MONO_API_EXPORT DSOApkEntry dso_apk_entries[]; // // Support for marshal methods From eef6eced9c2674d5971aecd626868de4a0bece9f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 9 Nov 2023 19:36:21 +0100 Subject: [PATCH 009/143] Trying to make no-linking Release scenario work --- ...oft.Android.Sdk.AssemblyResolution.targets | 6 +- .../Tasks/BuildApk.cs | 74 +++++----- ...teCompressedAssembliesNativeSourceFiles.cs | 18 ++- ...ressedAssembliesNativeAssemblyGenerator.cs | 126 ++++++++---------- .../Utilities/CompressedAssemblyInfo.cs | 5 +- .../LlvmIrGenerator/LlvmIrGenerator.cs | 4 + .../LlvmIrGenerator/LlvmIrVariable.cs | 9 ++ src/monodroid/jni/embedded-assemblies.cc | 12 +- src/monodroid/jni/monodroid-glue.cc | 3 + 9 files changed, 136 insertions(+), 121 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index ae35d401aee..b1cebb26584 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -120,11 +120,13 @@ _ResolveAssemblies MSBuild target. <_ResolvedSymbolFiles Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.pdb' " /> <_ResolvedJavaLibraries Include="@(ResolvedFileToPublish)" Condition=" '%(ResolvedFileToPublish.Extension)' == '.jar' " /> + + diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index ae398914e6d..2ad3de7c7ae 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -133,7 +133,7 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { } List excludePatterns = new List (); - void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void ExecuteWithAbi (string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { ArchiveFileList files = new ArchiveFileList (); bool refresh = true; @@ -312,12 +312,12 @@ public override bool RunTask () bool debug = _Debug; bool compress = !debug && EnableCompression; - IDictionary compressedAssembliesInfo = null; + IDictionary> compressedAssembliesInfo = null; if (compress) { string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); Log.LogDebugMessage ($"Retrieving assembly compression info with key '{key}'"); - compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal> (key, RegisteredTaskObjectLifetime.Build); + compressedAssembliesInfo = BuildEngine4.UnregisterTaskObjectAssemblyLocal>> (key, RegisteredTaskObjectLifetime.Build); if (compressedAssembliesInfo == null) throw new InvalidOperationException ($"Assembly compression info not found for key '{key}'. Compression will not be performed."); } @@ -361,7 +361,7 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options) return new Regex (sb.ToString (), options); } - void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary compressedAssembliesInfo, string assemblyStoreApkName) + void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> compressedAssembliesInfo, string assemblyStoreApkName) { string sourcePath; AssemblyCompression.AssemblyData compressedAssembly = null; @@ -496,40 +496,44 @@ string CompressAssembly (ITaskItem assembly) return assembly.ItemSpec; } - var key = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (compressedAssembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) && info != null) { - EnsureCompressedAssemblyData (assembly.ItemSpec, info.DescriptorIndex); - string assemblyOutputDir; - string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); - if (!String.IsNullOrEmpty (subDirectory)) { - assemblyOutputDir = Path.Combine (compressedOutputDir, abi, subDirectory); - } else { - assemblyOutputDir = Path.Combine (compressedOutputDir, abi); - } - AssemblyCompression.CompressionResult result = AssemblyCompression.Compress (compressedAssembly, assemblyOutputDir); - if (result != AssemblyCompression.CompressionResult.Success) { - switch (result) { - case AssemblyCompression.CompressionResult.EncodingFailed: - Log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); - break; - - case AssemblyCompression.CompressionResult.InputTooBig: - Log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); - break; - - default: - Log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); - break; - } - return assembly.ItemSpec; - } - return compressedAssembly.DestinationPath; - } else { + string key = CompressedAssemblyInfo.GetDictionaryKey (assembly); + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!compressedAssembliesInfo.TryGetValue (arch, out Dictionary assembliesInfo)) { + throw new InvalidOperationException ($"Internal error: compression assembly info for architecture {arch} not available"); + } + + if (!assembliesInfo.TryGetValue (key, out CompressedAssemblyInfo info) || info == null) { Log.LogDebugMessage ($"Assembly missing from {nameof (CompressedAssemblyInfo)}: {key}"); + return assembly.ItemSpec; } - return assembly.ItemSpec; + EnsureCompressedAssemblyData (assembly.ItemSpec, info.DescriptorIndex); + string assemblyOutputDir; + string subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); + string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); + if (!String.IsNullOrEmpty (subDirectory)) { + assemblyOutputDir = Path.Combine (compressedOutputDir, abi, subDirectory); + } else { + assemblyOutputDir = Path.Combine (compressedOutputDir, abi); + } + AssemblyCompression.CompressionResult result = AssemblyCompression.Compress (compressedAssembly, assemblyOutputDir); + if (result != AssemblyCompression.CompressionResult.Success) { + switch (result) { + case AssemblyCompression.CompressionResult.EncodingFailed: + Log.LogMessage ($"Failed to compress {assembly.ItemSpec}"); + break; + + case AssemblyCompression.CompressionResult.InputTooBig: + Log.LogMessage ($"Input assembly {assembly.ItemSpec} exceeds maximum input size"); + break; + + default: + Log.LogMessage ($"Unknown error compressing {assembly.ItemSpec}"); + break; + } + return assembly.ItemSpec; + } + return compressedAssembly.DestinationPath; } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs index ada3d4a0f8a..014cde1697b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs @@ -43,16 +43,22 @@ void GenerateCompressedAssemblySources () return; } - var assemblies = new Dictionary (StringComparer.Ordinal); + var archAssemblies = new Dictionary> (); var counters = new Dictionary (); foreach (ITaskItem assembly in ResolvedAssemblies) { if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { continue; } + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!archAssemblies.TryGetValue (arch, out Dictionary assemblies)) { + assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + archAssemblies.Add (arch, assemblies); + } + var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); if (assemblies.ContainsKey (assemblyKey)) { - Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec}"); + Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetAssemblyAbi(assembly)})"); continue; } @@ -62,7 +68,7 @@ void GenerateCompressedAssemblySources () continue; } - AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!counters.TryGetValue (arch, out uint counter)) { counter = 0; } @@ -72,10 +78,10 @@ void GenerateCompressedAssemblySources () string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); Log.LogDebugMessage ($"Storing compression assemblies info with key '{key}'"); - BuildEngine4.RegisterTaskObjectAssemblyLocal (key, assemblies, RegisteredTaskObjectLifetime.Build); - Generate (assemblies); + BuildEngine4.RegisterTaskObjectAssemblyLocal (key, archAssemblies, RegisteredTaskObjectLifetime.Build); + Generate (archAssemblies); - void Generate (IDictionary dict) + void Generate (Dictionary> dict) { var composer = new CompressedAssembliesNativeAssemblyGenerator (dict); LLVMIR.LlvmIrModule compressedAssemblies = composer.Construct (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs index 4be23f66693..e811760aa04 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssembliesNativeAssemblyGenerator.cs @@ -69,82 +69,90 @@ sealed class CompressedAssemblies public CompressedAssemblyDescriptor descriptors; }; - IDictionary assemblies; + IDictionary>? archAssemblies; StructureInfo compressedAssemblyDescriptorStructureInfo; StructureInfo compressedAssembliesStructureInfo; Dictionary>> archData = new Dictionary>> (); - public CompressedAssembliesNativeAssemblyGenerator (IDictionary assemblies) + public CompressedAssembliesNativeAssemblyGenerator (IDictionary>? archAssemblies) { - this.assemblies = assemblies; + this.archAssemblies = archAssemblies; } - void InitCompressedAssemblies (out StructureInstance? compressedAssemblies, + void InitCompressedAssemblies (out List? compressedAssemblies, + out List? compressedAssemblyDescriptors, out List? buffers) { - if (assemblies == null || assemblies.Count == 0) { + if (archAssemblies == null || archAssemblies.Count == 0) { compressedAssemblies = null; + compressedAssemblyDescriptors = null; buffers = null; return; } - AndroidTargetArch? firstArch = null; - foreach (var kvp in assemblies) { - CompressedAssemblyInfo info = kvp.Value; + buffers = new List (); + foreach (var kvpArch in archAssemblies) { + foreach (var kvp in kvpArch.Value) { + CompressedAssemblyInfo info = kvp.Value; + + if (!archData.TryGetValue (info.TargetArch, out List> descriptors)) { + descriptors = new List> (); + archData.Add (info.TargetArch, descriptors); + } + + string bufferName = $"__compressedAssemblyData_{info.DescriptorIndex}"; + var descriptor = new CompressedAssemblyDescriptor { + Index = info.DescriptorIndex, + BufferSymbolName = bufferName, + AssemblyName = info.AssemblyName, + uncompressed_file_size = info.FileSize, + loaded = false, + data = 0 + }; - if (firstArch == null) { - firstArch = info.TargetArch; - } + descriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); - if (!archData.TryGetValue (info.TargetArch, out List> descriptors)) { - descriptors = new List> (); - archData.Add (info.TargetArch, descriptors); + var buffer = new LlvmIrGlobalVariable (typeof(List), descriptor.BufferSymbolName, LlvmIrVariableOptions.LocalWritable) { + ArrayItemCount = descriptor.uncompressed_file_size, + TargetArch = info.TargetArch, + ZeroInitializeArray = true, + }; + buffers.Add (buffer); } - - string bufferName = $"__compressedAssemblyData_{info.DescriptorIndex}"; - var descriptor = new CompressedAssemblyDescriptor { - Index = info.DescriptorIndex, - BufferSymbolName = bufferName, - AssemblyName = info.AssemblyName, - uncompressed_file_size = info.FileSize, - loaded = false, - data = 0 - }; - - descriptors.Add (new StructureInstance (compressedAssemblyDescriptorStructureInfo, descriptor)); } - buffers = new List (); - uint assembliesCount = 0; + compressedAssemblies = new List (); + compressedAssemblyDescriptors = new List (); foreach (var kvp in archData) { List> descriptors = kvp.Value; descriptors.Sort ((StructureInstance a, StructureInstance b) => a.Instance.Index.CompareTo (b.Instance.Index)); - if (buffers.Count > 0) { - continue; - } - assembliesCount = (uint)descriptors.Count; - foreach (StructureInstance desc in descriptors) { - var buffer = new LlvmIrGlobalVariable (typeof(List), desc.Instance.BufferSymbolName, LlvmIrVariableOptions.LocalWritable) { - BeforeWriteCallback = BufferBeforeWrite, - BeforeWriteCallbackCallerState = desc.Instance.Index, - ZeroInitializeArray = true, - }; - buffers.Add (buffer); - } - } + var variable = new LlvmIrGlobalVariable (typeof(StructureInstance), CompressedAssembliesSymbolName) { + Options = LlvmIrVariableOptions.GlobalWritable, + TargetArch = kvp.Key, + Value = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = (uint)descriptors.Count, }), + }; + compressedAssemblies.Add (variable); - compressedAssemblies = new StructureInstance (compressedAssembliesStructureInfo, new CompressedAssemblies { count = assembliesCount }); + variable = new LlvmIrGlobalVariable (typeof(List>), DescriptorsArraySymbolName) { + GetArrayItemCommentCallback = GetCompressedAssemblyDescriptorsItemComment, + Options = LlvmIrVariableOptions.LocalWritable, + TargetArch = kvp.Key, + Value = descriptors, + }; + compressedAssemblyDescriptors.Add (variable); + } } protected override void Construct (LlvmIrModule module) { MapStructures (module); - StructureInstance? compressedAssemblies; - List? buffers; - - InitCompressedAssemblies (out compressedAssemblies, out buffers); + InitCompressedAssemblies ( + out List? compressedAssemblies, + out List? compressedAssemblyDescriptors, + out List? buffers + ); if (archData.Count == 0) { module.AddGlobalVariable ( @@ -156,13 +164,7 @@ protected override void Construct (LlvmIrModule module) return; } - module.AddGlobalVariable (CompressedAssembliesSymbolName, compressedAssemblies, LlvmIrVariableOptions.GlobalWritable); - - var compressedAssemblyDescriptors = new LlvmIrGlobalVariable (typeof(List>), DescriptorsArraySymbolName) { - BeforeWriteCallback = CompressedAssemblyDescriptorsBeforeWrite, - GetArrayItemCommentCallback = GetCompressedAssemblyDescriptorsItemComment, - Options = LlvmIrVariableOptions.LocalWritable, - }; + module.Add (compressedAssemblies); module.Add (compressedAssemblyDescriptors); module.Add (new LlvmIrGroupDelimiterVariable ()); @@ -181,24 +183,6 @@ protected override void Construct (LlvmIrModule module) return $" {index}: {desc.Instance.AssemblyName}"; } - void CompressedAssemblyDescriptorsBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) - { - List> descriptors = GetArchDescriptors (target); - var gv = (LlvmIrGlobalVariable)variable; - gv.OverrideTypeAndValue (gv.Type, descriptors); - } - - void BufferBeforeWrite (LlvmIrVariable variable, LlvmIrModuleTarget target, object? state) - { - List> descriptors = GetArchDescriptors (target); - uint descriptorIndex = (uint)state; - StructureInstance desc = descriptors[(int)descriptorIndex]; - var gv = (LlvmIrGlobalVariable)variable; - - gv.ArrayItemCount = desc.Instance.uncompressed_file_size; - gv.OverrideName (desc.Instance.BufferSymbolName); - } - List> GetArchDescriptors (LlvmIrModuleTarget target) { if (!archData.TryGetValue (target.TargetArch, out List> descriptors)) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs index 56cf36f92e5..c655b956c14 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/CompressedAssemblyInfo.cs @@ -33,15 +33,14 @@ public static string GetKey (string projectFullPath) public static string GetDictionaryKey (ITaskItem assembly) { - string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); // Prefer %(DestinationSubPath) if set var path = assembly.GetMetadata ("DestinationSubPath"); if (!string.IsNullOrEmpty (path)) { - return Path.Combine (abi, path); + return path; } // MSBuild sometimes only sets %(DestinationSubDirectory) var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - return Path.Combine (abi, subDirectory, Path.GetFileName (assembly.ItemSpec)); + return Path.Combine (subDirectory, Path.GetFileName (assembly.ItemSpec)); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index be469065e3a..086c4bdcebc 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -233,6 +233,10 @@ void WriteGlobalVariables (GeneratorWriteContext context) continue; } + if (gv.TargetArch.HasValue && gv.TargetArch.Value != target.TargetArch) { + continue; + } + if (gv.BeforeWriteCallback != null) { gv.BeforeWriteCallback (gv, target, gv.BeforeWriteCallbackCallerState); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index cfc427ef47d..d66bda3d7c9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -2,6 +2,8 @@ using System.Collections; using System.Globalization; +using Xamarin.Android.Tools; + namespace Xamarin.Android.Tasks.LLVMIR; [Flags] @@ -216,6 +218,13 @@ class LlvmIrGlobalVariable : LlvmIrVariable /// public virtual LlvmIrVariableOptions? Options { get; set; } + /// + /// There are situations when a variable differs enough between architectures, that the difference cannot be + /// handled with . In such situations one can create a separate variable + /// for each architecture and set this property. + /// + public AndroidTargetArch? TargetArch { get; set; } + /// /// If set to `true`, initialize the array with a shortcut zero-initializer statement. Useful when pre-allocating /// space for runtime use that won't be filled in with any data at the build time. diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 45f67507f8b..42fe5e41f44 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -385,24 +385,28 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); } - + log_debug (LOG_ASSEMBLY, "here #1.5"); MonoImageOpenStatus status; MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, ref_only); + log_debug (LOG_ASSEMBLY, "here #1.6"); if (a == nullptr || status != MonoImageOpenStatus::MONO_IMAGE_OK) { log_warn (LOG_ASSEMBLY, "Failed to load managed assembly '%s'. %s", name.get (), mono_image_strerror (status)); return nullptr; @@ -437,11 +441,11 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, TLoaderData load } else { a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); } - + log_debug (LOG_ASSEMBLY, "here #1.7"); if (a == nullptr) { log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); } - + log_debug (LOG_ASSEMBLY, "here #1.8"); return a; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index f3006ca5bb3..f80915e2f42 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -1372,9 +1372,12 @@ MonodroidRuntime::monodroid_dlopen (const char *name, int flags, char **err) noe StartupAwareLock lock (dso_handle_write_lock); #if defined (RELEASE) if (androidSystem.is_embedded_dso_mode_enabled ()) { + log_debug (LOG_ASSEMBLY, "here #2.0"); DSOApkEntry *apk_entry = dso_apk_entries; for (size_t i = 0; i < application_config.number_of_shared_libraries; i++) { + log_debug (LOG_ASSEMBLY, "here #2.1"); if (apk_entry->name_hash != dso->real_name_hash) { + log_debug (LOG_ASSEMBLY, "here #2.2"); apk_entry++; continue; } From 3fc39cb8407ab602294e73af5e8776b6c4d35662 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 13 Nov 2023 22:54:49 +0100 Subject: [PATCH 010/143] Fix build and run without linking --- ...oft.Android.Sdk.AssemblyResolution.targets | 8 +-- .../Tasks/BuildApk.cs | 71 +++++++++++-------- .../Tasks/BuildBaseAppBundle.cs | 2 +- .../Utilities/MonoAndroidHelper.cs | 29 ++++++++ .../Xamarin.Android.Common.targets | 4 +- src/monodroid/jni/embedded-assemblies-zip.cc | 1 + 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index b1cebb26584..5f0f4a8d232 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -180,10 +180,10 @@ _ResolveAssemblies MSBuild target. - <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> - <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> - <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> - <_ResolvedSymbols Include="@(ResolvedSymbols->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedSymbols Include="@(ResolvedSymbols->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ShrunkAssemblies Include="@(_ResolvedAssemblies)" /> <_ShrunkUserAssemblies Include="@(_ResolvedUserAssemblies)" /> <_ShrunkFrameworkAssemblies Include="@(_ResolvedFrameworkAssemblies)" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 2ad3de7c7ae..8dbeb1fc424 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -23,6 +23,9 @@ namespace Xamarin.Android.Tasks { public class BuildApk : AndroidTask { + const string ArchiveAssembliesPath = "assemblies"; + const string ArchiveLibPath = "lib"; + public override string TaskPrefix => "BLD"; public string AndroidNdkDirectory { get; set; } @@ -117,12 +120,11 @@ bool _Debug { SequencePointsMode sequencePointsMode = SequencePointsMode.None; public ITaskItem[] LibraryProjectJars { get; set; } - string [] uncompressedFileExtensions; + HashSet uncompressedFileExtensions; + // Do not use trailing / in the path protected virtual string RootPath => ""; - protected virtual string AssembliesPath => RootPath + "assemblies/"; - protected virtual string DalvikPath => ""; protected virtual CompressionMethod UncompressedMethod => CompressionMethod.Store; @@ -302,7 +304,18 @@ public override bool RunTask () Aot.TryGetSequencePointsMode (AndroidSequencePointsMode, out sequencePointsMode); var outputFiles = new List (); - uncompressedFileExtensions = UncompressedFileExtensions?.Split (new char [] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty (); + uncompressedFileExtensions = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (string? e in UncompressedFileExtensions?.Split (new char [] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries) ?? Array.Empty ()) { + string? ext = e?.Trim (); + if (String.IsNullOrEmpty (ext)) { + continue; + } + + if (ext[0] != '.') { + ext = $".{ext}"; + } + uncompressedFileExtensions.Add (ext); + } existingEntries.Clear (); @@ -382,7 +395,6 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary assemblyStorePathsNew = storeGenerator.Generate (AppSharedLibrariesDir); + Dictionary assemblyStorePaths = storeGenerator.Generate (AppSharedLibrariesDir); - if (assemblyStorePathsNew.Count == 0) { + if (assemblyStorePaths.Count == 0) { throw new InvalidOperationException ("Assembly store generator did not generate any stores"); } - if (assemblyStorePathsNew.Count != SupportedAbis.Length) { + if (assemblyStorePaths.Count != SupportedAbis.Length) { throw new InvalidOperationException ("Internal error: assembly store did not generate store for each supported ABI"); } - foreach (var kvp in assemblyStorePathsNew) { + foreach (var kvp in assemblyStorePaths) { string abi = MonoAndroidHelper.ArchToAbi (kvp.Key); - inArchivePath = GetArchLibPath (abi, Path.GetFileName (kvp.Value)); + inArchivePath = MakeArchiveLibPath (abi, Path.GetFileName (kvp.Value)); AddFileToArchiveIfNewer (apk, kvp.Value, inArchivePath, UncompressedMethod); } - string GetArchLibPath (string abi, string fileName) => libRootPath + abi + "/" + fileName; - void AddAssembliesFromCollection (ITaskItem[] assemblies) { foreach (ITaskItem assembly in assemblies) { @@ -436,7 +446,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) sourcePath = CompressAssembly (assembly); // Add assembly - var assemblyPath = GetAssemblyPath (assembly, frameworkAssembly: false); + var assemblyPath = GetInArchiveAssemblyPath (assembly, frameworkAssembly: false); if (UseAssemblyStore) { storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly); } else { @@ -576,20 +586,23 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi /// /// Returns the in-archive path for an assembly /// - string GetAssemblyPath (ITaskItem assembly, bool frameworkAssembly) + string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) { - var subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - string abi = MonoAndroidHelper.GetAssemblyAbi (assembly); - string assembliesPath = AssembliesPath + "/" + abi + "/"; + var parts = new List (); + bool haveRootPath = !String.IsNullOrEmpty (RootPath); + + if (haveRootPath) { + parts.Add (ArchiveAssembliesPath); + } + parts.Add (MonoAndroidHelper.GetAssemblyAbi (assembly)); + + string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); if (!string.IsNullOrEmpty (subDirectory)) { - assembliesPath += subDirectory.Replace ('\\', '/'); - if (!assembliesPath.EndsWith ("/", StringComparison.Ordinal)) { - assembliesPath += "/"; - } + parts.Add (subDirectory.Replace ('\\', '/')); } else if (!frameworkAssembly && SatelliteAssembly.TryGetSatelliteCultureAndFileName (assembly.ItemSpec, out var culture, out _)) { - assembliesPath += culture + "/"; + parts.Add (culture); } - return assembliesPath; + return MonoAndroidHelper.MakeZipArchivePath (haveRootPath ? RootPath : ArchiveAssembliesPath, parts) + "/"; } sealed class LibInfo @@ -602,14 +615,12 @@ sealed class LibInfo CompressionMethod GetCompressionMethod (string fileName) { - if (uncompressedFileExtensions.Any (x => string.Compare (x.StartsWith (".", StringComparison.OrdinalIgnoreCase) ? x : $".{x}", Path.GetExtension (fileName), StringComparison.OrdinalIgnoreCase) == 0)) - return UncompressedMethod; - return CompressionMethod.Default; + return uncompressedFileExtensions.Contains (Path.GetExtension (fileName)) ? UncompressedMethod : CompressionMethod.Default; } void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemPath, string inArchiveFileName) { - string archivePath = $"lib/{abi}/{inArchiveFileName}"; + string archivePath = MakeArchiveLibPath (abi, inArchiveFileName); existingEntries.Remove (archivePath); CompressionMethod compressionMethod = GetCompressionMethod (archivePath); if (apk.SkipExistingFile (filesystemPath, archivePath, compressionMethod)) { @@ -793,7 +804,7 @@ private void AddAdditionalNativeLibraries (ArchiveFileList files, string [] supp void AddNativeLibrary (ArchiveFileList files, string path, string abi, string archiveFileName) { string fileName = string.IsNullOrEmpty (archiveFileName) ? Path.GetFileName (path) : archiveFileName; - var item = (filePath: path, archivePath: $"lib/{abi}/{fileName}"); + var item = (filePath: path, archivePath: MakeArchiveLibPath (abi, fileName)); if (files.Any (x => x.archivePath == item.archivePath)) { Log.LogCodedWarning ("XA4301", path, 0, Properties.Resources.XA4301, item.archivePath); return; @@ -817,5 +828,7 @@ void LogSanitizerError (string message) { Log.LogError (message); } + + static string MakeArchiveLibPath (string abi, string fileName) => MonoAndroidHelper.MakeZipArchivePath (ArchiveLibPath, abi, fileName); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildBaseAppBundle.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildBaseAppBundle.cs index 400a58ee43d..7e8d5f72919 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildBaseAppBundle.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildBaseAppBundle.cs @@ -12,7 +12,7 @@ public class BuildBaseAppBundle : BuildApk public override string TaskPrefix => "BBA"; /// - /// Files that need to land in the final APK need to go in `root/` + /// Files that need to land in the final APK need to go in `root` /// protected override string RootPath => "root/"; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 94bee97d49d..bae6eeddeb2 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -21,6 +21,7 @@ namespace Xamarin.Android.Tasks { public partial class MonoAndroidHelper { + static readonly char[] ZipPathTrimmedChars = {'/', '\\'}; static Lazy uname = new Lazy (GetOSBinDirName, System.Threading.LazyThreadSafetyMode.PublicationOnly); // Set in ResolveSdks.Execute(); @@ -656,5 +657,33 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } + + public static string MakeZipArchivePath (string part1, params string[]? pathParts) + { + return MakeZipArchivePath (part1, (ICollection?)pathParts); + } + + public static string MakeZipArchivePath (string part1, ICollection? pathParts) + { + var parts = new List (); + if (!String.IsNullOrEmpty (part1)) { + parts.Add (part1.TrimEnd (ZipPathTrimmedChars)); + }; + + if (pathParts != null && pathParts.Count > 0) { + foreach (string p in pathParts) { + if (String.IsNullOrEmpty (p)) { + continue; + } + parts.Add (p.TrimEnd (ZipPathTrimmedChars)); + } + } + + if (parts.Count == 0) { + return String.Empty; + } + + return String.Join ("/", parts); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 4edb130ae9e..f696ff74aca 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1448,11 +1448,11 @@ because xbuild doesn't support framework reference assemblies. DependsOnTargets="_LinkAssembliesNoShrinkInputs" Condition="'$(AndroidLinkMode)' == 'None'" Inputs="@(ResolvedAssemblies);$(_AndroidBuildPropertiesCache)" - Outputs="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')"> + Outputs="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')"> (entry_name.length ()) - prefix_len, max_name_size); From ec90aff0f22ea5af42b4f065d02cfa876cd710fa Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 14 Nov 2023 21:16:35 +0100 Subject: [PATCH 011/143] Moving on --- .../Tasks/BuildApk.cs | 9 +- .../Tasks/BuildBaseAppBundle.cs | 2 +- .../Tasks/ProcessAssemblies.cs | 133 ++++-------------- .../Utilities/AssemblyCompression.cs | 3 +- .../Utilities/AssemblyStoreAssemblyInfo.cs | 10 +- .../Utilities/AssemblyStoreGenerator.cs | 31 ++-- .../Utilities/CompressedAssemblyInfo.cs | 6 +- src/monodroid/jni/embedded-assemblies-zip.cc | 7 +- src/monodroid/jni/embedded-assemblies.cc | 13 +- src/monodroid/jni/monodroid-glue.cc | 3 - src/monodroid/jni/xamarin-app.hh | 14 +- 11 files changed, 82 insertions(+), 149 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 8dbeb1fc424..f93341fd01a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -395,14 +395,13 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary "BBA"; /// - /// Files that need to land in the final APK need to go in `root` + /// Files that need to land in the final APK need to go in `root/` /// protected override string RootPath => "root/"; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 1db37adfb6c..b99e85e2aee 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -57,15 +57,7 @@ public override bool RunTask () } } - // We only need to "dedup" assemblies when there is more than one RID - // if (RuntimeIdentifiers.Length > 1) { - // Log.LogDebugMessage ("Deduplicating assemblies per RuntimeIdentifier"); - // DeduplicateAssemblies (output, symbols); - // } else { - // Log.LogDebugMessage ("Found a single RuntimeIdentifier"); - SetMetadataForAssemblies (output, symbols); - //} - + SetMetadataForAssemblies (output, symbols); OutputAssemblies = output.ToArray (); ResolvedSymbols = symbols.Values.ToArray (); @@ -99,95 +91,35 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - void SetAssemblyAbiMetadata (string abi, string assetType, ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) + void SetAssemblyAbiMetadata (string abi, ITaskItem assembly, ITaskItem? symbol) { - if (String.IsNullOrEmpty (abi) /* || (!isDuplicate && String.Compare ("native", assetType, StringComparison.OrdinalIgnoreCase) != 0) */) { - return; + if (String.IsNullOrEmpty (abi)) { + throw new ArgumentException ("must not be null or empty", nameof (abi)); } assembly.SetMetadata ("Abi", abi); - if (symbol != null) { - symbol.SetMetadata ("Abi", abi); - } + symbol?.SetMetadata ("Abi", abi); } - void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol, bool isDuplicate) + void SetAssemblyAbiMetadata (ITaskItem assembly, ITaskItem? symbol) { - string assetType = assembly.GetMetadata ("AssetType"); string rid = assembly.GetMetadata ("RuntimeIdentifier"); - // if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { - // // Satellite assemblies are abi-agnostic, they shouldn't have the Abi metadata set - // return; - // } - SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assetType, assembly, symbol, isDuplicate); + SetAssemblyAbiMetadata (AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid), assembly, symbol); } void SetMetadataForAssemblies (List output, Dictionary symbols) { - foreach (var assembly in InputAssemblies) { - var symbol = GetOrCreateSymbolItem (symbols, assembly); - SetAssemblyAbiMetadata (assembly, symbol, isDuplicate: false); - symbol?.SetDestinationSubPath (); - assembly.SetDestinationSubPath (); + foreach (ITaskItem assembly in InputAssemblies) { + ITaskItem? symbol = GetOrCreateSymbolItem (symbols, assembly); + SetAssemblyAbiMetadata (assembly, symbol); + SetDestinationSubDirectory (assembly, symbol); assembly.SetMetadata ("FrameworkAssembly", IsFromAKnownRuntimePack (assembly).ToString ()); assembly.SetMetadata ("HasMonoAndroidReference", MonoAndroidHelper.HasMonoAndroidReference (assembly).ToString ()); output.Add (assembly); } } - void DeduplicateAssemblies (List output, Dictionary symbols) - { - // Group by assembly file name - foreach (var group in InputAssemblies.Where (Filter).GroupBy (a => Path.GetFileName (a.ItemSpec))) { - // Get the unique list of MVIDs - var mvids = new HashSet (); - bool? frameworkAssembly = null, hasMonoAndroidReference = null; - foreach (var assembly in group) { - using var pe = new PEReader (File.OpenRead (assembly.ItemSpec)); - var reader = pe.GetMetadataReader (); - var module = reader.GetModuleDefinition (); - var mvid = reader.GetGuid (module.Mvid); - mvids.Add (mvid); - - // Calculate %(FrameworkAssembly) and %(HasMonoAndroidReference) for the first - if (frameworkAssembly == null) { - frameworkAssembly = IsFromAKnownRuntimePack (assembly); - } - if (hasMonoAndroidReference == null) { - hasMonoAndroidReference = MonoAndroidHelper.IsMonoAndroidAssembly (assembly) || - MonoAndroidHelper.HasMonoAndroidReference (reader); - } - assembly.SetMetadata ("FrameworkAssembly", frameworkAssembly.ToString ()); - assembly.SetMetadata ("HasMonoAndroidReference", hasMonoAndroidReference.ToString ()); - } - // If we end up with more than 1 unique mvid, we need *all* assemblies - if (mvids.Count > 1) { - foreach (var assembly in group) { - var symbol = GetOrCreateSymbolItem (symbols, assembly); - SetDestinationSubDirectory (assembly, group.Key, symbol, isDuplicate: true); - output.Add (assembly); - } - } else { - // Otherwise only include the first assembly - bool first = true; - foreach (var assembly in group) { - if (first) { - first = false; - - var symbol = GetOrCreateSymbolItem (symbols, assembly); - symbol?.SetDestinationSubPath (); - assembly.SetDestinationSubPath (); - output.Add (assembly); - SetAssemblyAbiMetadata (assembly, symbol, false); - } else { - symbols.Remove (Path.ChangeExtension (assembly.ItemSpec, ".pdb")); - } - } - } - } - } - static bool IsFromAKnownRuntimePack (ITaskItem assembly) { string packageId = assembly.GetMetadata ("NuGetPackageId") ?? ""; @@ -219,35 +151,30 @@ bool Filter (ITaskItem item) /// /// Sets %(DestinationSubDirectory) and %(DestinationSubPath) based on %(RuntimeIdentifier) /// - void SetDestinationSubDirectory (ITaskItem assembly, string fileName, ITaskItem? symbol, bool isDuplicate) + void SetDestinationSubDirectory (ITaskItem assembly, ITaskItem? symbol) { - var rid = assembly.GetMetadata ("RuntimeIdentifier"); - string assetType = assembly.GetMetadata ("AssetType"); - - // Satellite assemblies have `RuntimeIdentifier` set, but they shouldn't - they aren't specific to any architecture, so they should have none of the - // abi-specific metadata set - // - if (!String.IsNullOrEmpty (assembly.GetMetadata ("Culture")) || - String.Compare ("resources", assetType, StringComparison.OrdinalIgnoreCase) == 0) { - rid = String.Empty; + string? rid = assembly.GetMetadata ("RuntimeIdentifier"); + if (String.IsNullOrEmpty (rid)) { + throw new InvalidOperationException ($"Assembly '{assembly}' item is missing required "); } - var abi = AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid); - if (!string.IsNullOrEmpty (abi)) { - string destination = Path.Combine (assembly.GetMetadata ("DestinationSubDirectory"), abi); - assembly.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); - assembly.SetMetadata ("DestinationSubPath", Path.Combine (destination, fileName)); - if (symbol != null) { - destination = Path.Combine (symbol.GetMetadata ("DestinationSubDirectory"), abi); - symbol.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); - symbol.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (symbol.ItemSpec))); + string? abi = AndroidRidAbiHelper.RuntimeIdentifierToAbi (rid); + if (string.IsNullOrEmpty (abi)) { + throw new InvalidOperationException ($"Unable to convert a runtime identifier '{rid}' to Android ABI for: {assembly.ItemSpec}"); + } + + SetIt (assembly); + SetIt (symbol); + + void SetIt (ITaskItem? item) + { + if (item == null) { + return; } - SetAssemblyAbiMetadata (abi, assetType, assembly, symbol, isDuplicate); - } else { - Log.LogDebugMessage ($"Android ABI not found for: {assembly.ItemSpec}"); - assembly.SetDestinationSubPath (); - symbol?.SetDestinationSubPath (); + string destination = Path.Combine (item.GetMetadata ("DestinationSubDirectory"), abi); + item.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); + item.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (item.ItemSpec))); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index 57c40136717..39a899bd31a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -69,7 +69,6 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect data.DestinationPath = Path.Combine (outputDirectory, $"{Path.GetFileName (data.SourcePath)}.lz4"); data.SourceSize = (uint)fi.Length; - Console.WriteLine ($"Compressing: {data.SourcePath} => {data.DestinationPath}; Index: {data.DescriptorIndex}"); byte[] sourceBytes = null; byte[] destBytes = null; try { @@ -79,7 +78,7 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect } destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (sourceBytes.Length)); - int encodedLength = LZ4Codec.Encode (sourceBytes, 0, checked((int)fi.Length), destBytes, 0, destBytes.Length, LZ4Level.L09_HC); + int encodedLength = LZ4Codec.Encode (sourceBytes, 0, checked((int)fi.Length), destBytes, 0, destBytes.Length, LZ4Level.L12_MAX); if (encodedLength < 0) return CompressionResult.EncodingFailed; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index 3f5a44a5840..c8922c09a2d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -8,12 +8,12 @@ namespace Xamarin.Android.Tasks; class AssemblyStoreAssemblyInfo { - public AndroidTargetArch Arch { get; } - public string InArchivePath { get; } - public FileInfo SourceFile { get; } + public AndroidTargetArch Arch { get; } + public string InArchivePath { get; } + public FileInfo SourceFile { get; } - public FileInfo? SymbolsFile { get; set; } - public FileInfo? ConfigFile { get; set; } + public FileInfo? SymbolsFile { get; set; } + public FileInfo? ConfigFile { get; set; } public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index fab2ea06bb4..60e5bc70c99 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -21,15 +21,15 @@ namespace Xamarin.Android.Tasks; // Formats of the sections above are as follows: // // HEADER (fixed size) -// [MAGIC] uint; value: 0x41424158 -// [FORMAT] uint; store format version number -// [ENTRY_COUNT] uint; number of entries in the store -// [INDEX_ENTRY_COUNT] uint; number of entries in the index -// [INDEX_SIZE] uint; index size in bytes +// [MAGIC] uint; value: 0x41424158 +// [FORMAT_VERSION] uint; store format version number +// [ENTRY_COUNT] uint; number of entries in the store +// [INDEX_ENTRY_COUNT] uint; number of entries in the index +// [INDEX_SIZE] uint; index size in bytes // // INDEX (variable size, HEADER.ENTRY_COUNT*2 entries, for assembly names with and without the extension) -// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name -// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array +// [NAME_HASH] uint on 32-bit platforms, ulong on 64-bit platforms; xxhash of the assembly name +// [DESCRIPTOR_INDEX] uint; index into in-store assembly descriptor array // // ASSEMBLY_DESCRIPTORS (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: // [MAPPING_INDEX] uint; index into a runtime array where assembly data pointers are stored @@ -49,6 +49,11 @@ partial class AssemblyStoreGenerator const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; + const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; + const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; + const uint ASSEMBLY_STORE_ABI_X64 = 0x00030000; + const uint ASSEMBLY_STORE_ABI_X86 = 0x00040000; + readonly TaskLoggingHelper log; readonly Dictionary> assemblies; @@ -82,11 +87,11 @@ public Dictionary Generate (string baseOutputDirector string Generate (string baseOutputDirectory, AndroidTargetArch arch, List infos) { - bool is64Bit = arch switch { - AndroidTargetArch.Arm => false, - AndroidTargetArch.X86 => false, - AndroidTargetArch.Arm64 => true, - AndroidTargetArch.X86_64 => true, + (bool is64Bit, uint abiFlag) = arch switch { + AndroidTargetArch.Arm => (false, ASSEMBLY_STORE_ABI_ARM), + AndroidTargetArch.X86 => (false, ASSEMBLY_STORE_ABI_X86), + AndroidTargetArch.Arm64 => (true, ASSEMBLY_STORE_ABI_AARCH64), + AndroidTargetArch.X86_64 => (true, ASSEMBLY_STORE_ABI_X64), _ => throw new NotSupportedException ($"Internal error: arch {arch} not supported") }; @@ -137,7 +142,7 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List Helpers::abort_application (); } - if (header->version > ASSEMBLY_STORE_FORMAT_VERSION) { - log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format v%u which is not understood by this version of Xamarin.Android", entry_name.get (), header->version); + if (header->version != ASSEMBLY_STORE_FORMAT_VERSION) { + log_fatal (LOG_ASSEMBLY, "Assembly store '%s' uses format version 0x%x, instead of the expected 0x%x", entry_name.get (), header->version, ASSEMBLY_STORE_FORMAT_VERSION); Helpers::abort_application (); } @@ -317,9 +317,8 @@ EmbeddedAssemblies::set_entry_data (XamarinAndroidBundledAssembly &entry, int ap if constexpr (NeedsNameAlloc) { entry.name = utils.strdup_new (entry_name.get () + prefix_len); } else { - // entry.name is preallocated on build time here and is max_name_size + 1 bytes long, filled with 0s, thus we + // entry.name is preallocated at build time here and is max_name_size + 1 bytes long, filled with 0s, thus we // don't need to append the terminating NUL even for strings of `max_name_size` characters - log_debug (LOG_ASSEMBLY, "Storing data for assembly entry '%s'", entry_name.get ()); strncpy (entry.name, entry_name.get () + prefix_len, max_name_size); } entry.name_length = std::min (static_cast(entry_name.length ()) - prefix_len, max_name_size); diff --git a/src/monodroid/jni/embedded-assemblies.cc b/src/monodroid/jni/embedded-assemblies.cc index 42fe5e41f44..061a725e36e 100644 --- a/src/monodroid/jni/embedded-assemblies.cc +++ b/src/monodroid/jni/embedded-assemblies.cc @@ -385,28 +385,23 @@ EmbeddedAssemblies::assembly_store_open_from_bundles (dynamic_local_string (assembly_runtime_info.debug_info_data), static_cast(assembly_runtime_info.descriptor->debug_data_size)); } - log_debug (LOG_ASSEMBLY, "here #1.5"); + MonoImageOpenStatus status; MonoAssembly *a = mono_assembly_load_from_full (image, name.get (), &status, ref_only); - log_debug (LOG_ASSEMBLY, "here #1.6"); if (a == nullptr || status != MonoImageOpenStatus::MONO_IMAGE_OK) { log_warn (LOG_ASSEMBLY, "Failed to load managed assembly '%s'. %s", name.get (), mono_image_strerror (status)); return nullptr; @@ -441,11 +436,11 @@ EmbeddedAssemblies::open_from_bundles (MonoAssemblyName* aname, TLoaderData load } else { a = individual_assemblies_open_from_bundles (name, loader_data, ref_only); } - log_debug (LOG_ASSEMBLY, "here #1.7"); + if (a == nullptr) { log_warn (LOG_ASSEMBLY, "open_from_bundles: failed to load assembly %s", name.get ()); } - log_debug (LOG_ASSEMBLY, "here #1.8"); + return a; } diff --git a/src/monodroid/jni/monodroid-glue.cc b/src/monodroid/jni/monodroid-glue.cc index f80915e2f42..f3006ca5bb3 100644 --- a/src/monodroid/jni/monodroid-glue.cc +++ b/src/monodroid/jni/monodroid-glue.cc @@ -1372,12 +1372,9 @@ MonodroidRuntime::monodroid_dlopen (const char *name, int flags, char **err) noe StartupAwareLock lock (dso_handle_write_lock); #if defined (RELEASE) if (androidSystem.is_embedded_dso_mode_enabled ()) { - log_debug (LOG_ASSEMBLY, "here #2.0"); DSOApkEntry *apk_entry = dso_apk_entries; for (size_t i = 0; i < application_config.number_of_shared_libraries; i++) { - log_debug (LOG_ASSEMBLY, "here #2.1"); if (apk_entry->name_hash != dso->real_name_hash) { - log_debug (LOG_ASSEMBLY, "here #2.2"); apk_entry++; continue; } diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 8ead25f41a0..42eeee0ce15 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -14,14 +14,26 @@ static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58; static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian +// The highest bit of assembly store version is a 64-bit ABI flag #if INTPTR_MAX == INT64_MAX static constexpr uint32_t ASSEMBLY_STORE_64BIT_FLAG = 0x80000000; #else static constexpr uint32_t ASSEMBLY_STORE_64BIT_FLAG = 0x00000000; #endif +// The second-to-last byte denotes the actual ABI +#if defined(__aarch64__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00010000; +#elif defined(__arm__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00020000; +#elif defined(__x86_64__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00030000; +#elif defined(__i386__) +static constexpr uint32_t ASSEMBLY_STORE_ABI = 0x00040000; +#endif + // Increase whenever an incompatible change is made to the assembly store format -static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 2 | ASSEMBLY_STORE_64BIT_FLAG; +static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 2 | ASSEMBLY_STORE_64BIT_FLAG | ASSEMBLY_STORE_ABI; static constexpr uint32_t MODULE_MAGIC_NAMES = 0x53544158; // 'XATS', little-endian static constexpr uint32_t MODULE_INDEX_MAGIC = 0x49544158; // 'XATI', little-endian From 17ac554830a8734ca43c2dfd43e94f56d7669c62 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 15 Nov 2023 12:18:49 +0100 Subject: [PATCH 012/143] Bump monodroid to fix (some) tests --- .external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.external b/.external index 63fe115fdf6..18a56db07cc 100644 --- a/.external +++ b/.external @@ -1,2 +1,2 @@ -xamarin/monodroid:main@4d3ede83f699e1d127fed4a18de992819983d715 +xamarin/monodroid:dev/grendel/blobs-in-lib@204e5bbcaa2dcb69ccdc49216a76cdfff3de5e55 mono/mono:2020-02@6dd9def57ce969ca04a0ecd9ef72c0a8f069112d From 32b647b0d9a4f05822b4e49ef7a112f9fa68f1a6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 15 Nov 2023 14:58:28 +0100 Subject: [PATCH 013/143] Bump System.Reflection.Metadata to see if it fixes some tests --- .../Xamarin.Android.Build.Tasks.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj index 92132281396..76609c9fec7 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.csproj @@ -37,7 +37,7 @@ - + From be79df0815e9bbcc8ba5e652721665fd929798bd Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 15 Nov 2023 16:16:19 +0100 Subject: [PATCH 014/143] Bump the System.Collections.Immutable nuget, to see if it fixes some tests --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 64033ca5019..bfb9cc2147d 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -47,7 +47,7 @@ 5.4.0 1.1.11 6.12.0.148 - 6.0.0 + 8.0.0 6.0.0 2.13.1 2.14.1 From e875b3e91febbd562a851174a5c0e588975a676b Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 15 Nov 2023 21:16:44 +0100 Subject: [PATCH 015/143] Restore ILRepacker, to see if that's what broke some tests --- .../Xamarin.Android.Build.Tasks.targets | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets index fe34046bab6..fbd0fd842c7 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets @@ -268,6 +268,35 @@ + + + <_InputAssembliesThatExist Include="@(InputAssemblies)" Condition="Exists('%(Identity)')" /> + <_NetstandardPath Include="@(ReferencePath->'%(RootDir)%(Directory)')" Condition="'%(FileName)%(Extension)' == 'netstandard.dll'" /> + + + <_NetstandardDir>@(_NetstandardPath) + <_ILRepackArgs>/out:"$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" /internalize + <_ILRepackArgs>$(_ILRepackArgs) /keyfile:"$(XamarinAndroidSourcePath)product.snk" + <_ILRepackArgs>$(_ILRepackArgs) "$(MSBuildThisFileDirectory)$(IntermediateOutputPath)$(AssemblyName).dll" + <_ILRepackArgs>$(_ILRepackArgs) @(_InputAssembliesThatExist->'"%(Identity)"', ' ') + <_ILRepackArgs>$(_ILRepackArgs) /lib:"$(_NetstandardDir.TrimEnd('\'))" + + + + + + + + <_ExtraPackageSource Include="$(PkgXamarin_LibZipSharp)\lib\$(TargetFrameworkNETStandard)\libZipSharp.pdb" /> <_ExtraPackageTarget Include="$(OutputPath)\libZipSharp.pdb" /> From f3e83d3bf4bd9456af46815c1dd3c100b91cd288 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 16 Nov 2023 13:34:17 +0100 Subject: [PATCH 016/143] Rely on `DestinationSubPath` metadata `ProcessAssemblies` will always add the metadata --- .../Microsoft.Android.Sdk.AssemblyResolution.targets | 8 ++++---- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 11 +++++++---- .../Xamarin.Android.Common.targets | 4 ++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index 5f0f4a8d232..b1cebb26584 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -180,10 +180,10 @@ _ResolveAssemblies MSBuild target. - <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> - <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> - <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> - <_ResolvedSymbols Include="@(ResolvedSymbols->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> + <_ResolvedSymbols Include="@(ResolvedSymbols->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ShrunkAssemblies Include="@(_ResolvedAssemblies)" /> <_ShrunkUserAssemblies Include="@(_ResolvedUserAssemblies)" /> <_ShrunkFrameworkAssemblies Include="@(_ResolvedFrameworkAssemblies)" /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index f93341fd01a..a2e5c546912 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -593,12 +593,15 @@ string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) if (haveRootPath) { parts.Add (ArchiveAssembliesPath); } - parts.Add (MonoAndroidHelper.GetAssemblyAbi (assembly)); string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); - if (!string.IsNullOrEmpty (subDirectory)) { - parts.Add (subDirectory.Replace ('\\', '/')); - } else if (!frameworkAssembly && SatelliteAssembly.TryGetSatelliteCultureAndFileName (assembly.ItemSpec, out var culture, out _)) { + if (string.IsNullOrEmpty (subDirectory)) { + throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); + } + parts.Add (subDirectory.Replace ('\\', '/')); + + // Even satellite assemblies are treated as RID-specific + if (!frameworkAssembly && SatelliteAssembly.TryGetSatelliteCultureAndFileName (assembly.ItemSpec, out var culture, out _)) { parts.Add (culture); } return MonoAndroidHelper.MakeZipArchivePath (haveRootPath ? RootPath : ArchiveAssembliesPath, parts) + "/"; diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index f696ff74aca..4edb130ae9e 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -1448,11 +1448,11 @@ because xbuild doesn't support framework reference assemblies. DependsOnTargets="_LinkAssembliesNoShrinkInputs" Condition="'$(AndroidLinkMode)' == 'None'" Inputs="@(ResolvedAssemblies);$(_AndroidBuildPropertiesCache)" - Outputs="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(RuntimeIdentifier)\%(DestinationSubPath)')"> + Outputs="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')"> Date: Thu, 16 Nov 2023 13:35:29 +0100 Subject: [PATCH 017/143] Fix the `LinkAssembliesShrink` test --- .../Xamarin.Android.Build.Tests/IncrementalBuildTest.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index b456c4d6ae1..1c9de6d7530 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -655,8 +655,10 @@ public void LinkAssembliesNoShrink () Assert.IsTrue (b.Build (proj), "build should have succeeded."); // Touch an assembly to a timestamp older than build.props - var formsViewGroup = b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", "FormsViewGroup.dll")); - File.SetLastWriteTimeUtc (formsViewGroup, new DateTime (1970, 1, 1)); + foreach (string abi in proj.GetProperty (KnownProperties.AndroidSupportedAbis).Split (';')) { + var formsViewGroup = b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, "FormsViewGroup.dll")); + File.SetLastWriteTimeUtc (formsViewGroup, new DateTime (1970, 1, 1)); + } Assert.IsTrue (b.Build (proj, doNotCleanupOnUpdate: true), "build should have succeeded."); b.Output.AssertTargetIsNotSkipped (KnownTargets.LinkAssembliesNoShrink); From 0ef099662826c7ebc7257993e6a256c8a198aaff Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 16 Nov 2023 22:20:31 +0100 Subject: [PATCH 018/143] [WIP] New assembly store reader Like the old one, it will be used not only for the command line utility but also from tests, but with support for both old (v1) and new (v2) assembly blob formats. --- Xamarin.Android.sln | 2 +- ...pplicationConfigNativeAssemblyGenerator.cs | 4 +- .../Utilities/AssemblyStoreAssemblyInfo.cs | 25 ++- .../AssemblyStoreGenerator.Classes.cs | 7 + .../Utilities/AssemblyStoreGenerator.cs | 204 +++++++++++++----- .../LlvmIrGenerator/LlvmIrComposer.cs | 13 -- .../MarshalMethodsNativeAssemblyGenerator.cs | 8 +- .../Utilities/MonoAndroidHelper.cs | 14 ++ .../AssemblyStoreExplorer.cs | 32 +++ .../AssemblyStoreItem.cs | 23 ++ .../AssemblyStoreReader.cs | 58 +++++ .../Directory.Build.targets | 6 + tools/assembly-store-reader-mk2/FileFormat.cs | 11 + tools/assembly-store-reader-mk2/Log.cs | 12 ++ tools/assembly-store-reader-mk2/Program.cs | 74 +++++++ .../StorePrettyPrinter.cs | 89 ++++++++ .../StoreReader_V1.cs | 22 ++ .../StoreReader_V2.Classes.cs | 92 ++++++++ .../StoreReader_V2.cs | 149 +++++++++++++ tools/assembly-store-reader-mk2/Utils.cs | 85 ++++++++ .../assembly-store-reader.csproj | 28 +++ tools/assembly-store-reader/Program.cs | 2 +- .../assembly-store-reader.csproj | 4 +- 23 files changed, 880 insertions(+), 84 deletions(-) create mode 100644 tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs create mode 100644 tools/assembly-store-reader-mk2/AssemblyStoreItem.cs create mode 100644 tools/assembly-store-reader-mk2/AssemblyStoreReader.cs create mode 100644 tools/assembly-store-reader-mk2/Directory.Build.targets create mode 100644 tools/assembly-store-reader-mk2/FileFormat.cs create mode 100644 tools/assembly-store-reader-mk2/Log.cs create mode 100644 tools/assembly-store-reader-mk2/Program.cs create mode 100644 tools/assembly-store-reader-mk2/StorePrettyPrinter.cs create mode 100644 tools/assembly-store-reader-mk2/StoreReader_V1.cs create mode 100644 tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs create mode 100644 tools/assembly-store-reader-mk2/StoreReader_V2.cs create mode 100644 tools/assembly-store-reader-mk2/Utils.cs create mode 100644 tools/assembly-store-reader-mk2/assembly-store-reader.csproj diff --git a/Xamarin.Android.sln b/Xamarin.Android.sln index af81dc76da7..6c5e6091e87 100644 --- a/Xamarin.Android.sln +++ b/Xamarin.Android.sln @@ -115,7 +115,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "decompress-assemblies", "to EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "tmt", "tools\tmt\tmt.csproj", "{1A273ED2-AE84-48E9-9C23-E978C2D0CB34}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-store-reader", "tools\assembly-store-reader\assembly-store-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "assembly-store-reader", "tools\assembly-store-reader-mk2\assembly-store-reader.csproj", "{DA50FC92-7FE7-48B5-BDB6-CDA57B37BB51}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Java.Interop.Tools.JavaTypeSystem", "external\Java.Interop\src\Java.Interop.Tools.JavaTypeSystem\Java.Interop.Tools.JavaTypeSystem.csproj", "{4EFCED6E-9A6B-453A-94E4-CE4B736EC684}" EndProject diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index c65eced6f5e..49d6aa2296c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -323,8 +323,8 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob throw new InvalidOperationException ($"Internal error: DSO cache entry has unexpected type {instance.Obj.GetType ()}"); } - entry.hash = GetXxHash (entry.HashedName, is64Bit); - entry.real_name_hash = GetXxHash (entry.name, is64Bit); + entry.hash = MonoAndroidHelper.GetXxHash (entry.HashedName, is64Bit); + entry.real_name_hash = MonoAndroidHelper.GetXxHash (entry.name, is64Bit); } cache.Sort ((StructureInstance a, StructureInstance b) => a.Instance.hash.CompareTo (b.Instance.hash)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index c8922c09a2d..f3ca88f7d79 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -8,12 +8,13 @@ namespace Xamarin.Android.Tasks; class AssemblyStoreAssemblyInfo { - public AndroidTargetArch Arch { get; } - public string InArchivePath { get; } - public FileInfo SourceFile { get; } - - public FileInfo? SymbolsFile { get; set; } - public FileInfo? ConfigFile { get; set; } + public AndroidTargetArch Arch { get; } + public string InArchivePath { get; } + public FileInfo SourceFile { get; } + public string AssemblyName { get; } + public byte[] AssemblyNameBytes { get; } + public FileInfo? SymbolsFile { get; set; } + public FileInfo? ConfigFile { get; set; } public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) { @@ -24,5 +25,17 @@ public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssembl SourceFile = new FileInfo (sourceFilePath); InArchivePath = inArchiveAssemblyPath; + + string? name = Path.GetFileName (SourceFile.Name); + if (name == null) { + throw new InvalidOperationException ("Internal error: info without assembly name"); + } + + if (name.EndsWith (".lz4", StringComparison.OrdinalIgnoreCase)) { + name = Path.GetFileNameWithoutExtension (name); + } + + AssemblyName = name; + AssemblyNameBytes = MonoAndroidHelper.Utf8StringToBytes (name); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs index 18877bc0a7a..680d8966070 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.Classes.cs @@ -21,6 +21,13 @@ public AssemblyStoreHeader (uint version, uint entry_count, uint index_entry_cou this.index_entry_count = index_entry_count; this.index_size = index_size; } +#if XABT_TESTS + public AssemblyStoreHeader (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) + : this (version, entry_count, index_entry_count, index_size) + { + this.magic = magic; + } +#endif } sealed class AssemblyStoreIndexEntry diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index 60e5bc70c99..fe36b5ac775 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -40,12 +40,16 @@ namespace Xamarin.Android.Tasks; // [CONFIG_DATA_OFFSET] uint; offset from the beginning of the store to the start of assembly .config contents, 0 if absent // [CONFIG_DATA_SIZE] uint; size of the stored assembly .config contents, 0 if absent // +// ASSEMBLY_NAMES (variable size, HEADER.ENTRY_COUNT entries), each entry formatted as follows: +// [NAME_LENGTH] uint: length of assembly name +// [NAME] byte: UTF-8 bytes of assembly name, without the NUL terminator +// partial class AssemblyStoreGenerator { // The two constants below must match their counterparts in src/monodroid/jni/xamarin-app.hh const uint ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian, must match the BUNDLED_ASSEMBLIES_BLOB_MAGIC native constant - // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones + // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; @@ -100,9 +104,15 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List (); var descriptors = new List (); + ulong namesSize = 0; + + foreach (AssemblyStoreAssemblyInfo info in infos) { + namesSize += (ulong)info.AssemblyNameBytes.Length; + namesSize += sizeof (uint); + } - ulong assemblyDataStart = (infoCount * IndexEntrySize () * 2) + (AssemblyStoreEntryDescriptor.NativeSize * infoCount) + AssemblyStoreHeader.NativeSize; - // We'll start writing to the stream after we seek to the position just after the header, index and descriptors data. + ulong assemblyDataStart = (infoCount * IndexEntrySize () * 2) + (AssemblyStoreEntryDescriptor.NativeSize * infoCount) + AssemblyStoreHeader.NativeSize + namesSize; + // We'll start writing to the stream after we seek to the position just after the header, index, descriptors and name data. ulong curPos = assemblyDataStart; using var fs = File.Open (storePath, FileMode.Create, FileAccess.Write, FileShare.Read); @@ -117,21 +127,11 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List a.name_hash.CompareTo (b.name_hash)); + WriteHeader (writer, header); using var manifestFs = File.Open ($"{storePath}.manifest", FileMode.Create, FileAccess.Write, FileShare.Read); using var mw = new StreamWriter (manifestFs, new System.Text.UTF8Encoding (false)); - foreach (AssemblyStoreIndexEntry entry in index) { - if (is64Bit) { - writer.Write (entry.name_hash); - mw.Write ($"0x{entry.name_hash:x}"); - } else { - writer.Write ((uint)entry.name_hash); - mw.Write ($"0x{(uint)entry.name_hash:x}"); - } - writer.Write (entry.descriptor_index); - mw.Write ($" di:{entry.descriptor_index}"); - - AssemblyStoreEntryDescriptor desc = descriptors[(int)entry.descriptor_index]; - mw.Write ($" mi:{desc.mapping_index}"); - mw.Write ($" do:{desc.data_offset}"); - mw.Write ($" ds:{desc.data_size}"); - mw.Write ($" ddo:{desc.debug_data_offset}"); - mw.Write ($" dds:{desc.debug_data_size}"); - mw.Write ($" cdo:{desc.config_data_offset}"); - mw.Write ($" cds:{desc.config_data_size}"); - mw.WriteLine ($" {entry.name}"); - } - writer.Flush (); + WriteIndex (writer, mw, index, descriptors, is64Bit); mw.Flush (); Console.WriteLine ($"Number of descriptors: {descriptors.Count}; index entries: {index.Count}"); Console.WriteLine ($"Header size: {AssemblyStoreHeader.NativeSize}; index entry size: {IndexEntrySize ()}; descriptor size: {AssemblyStoreEntryDescriptor.NativeSize}"); - foreach (AssemblyStoreEntryDescriptor desc in descriptors) { - writer.Write (desc.mapping_index); - writer.Write (desc.data_offset); - writer.Write (desc.data_size); - writer.Write (desc.debug_data_offset); - writer.Write (desc.debug_data_size); - writer.Write (desc.config_data_offset); - writer.Write (desc.config_data_size); - } + WriteDescriptors (writer, descriptors); + WriteNames (writer, infos); writer.Flush (); if (fs.Position != (long)assemblyDataStart) { @@ -248,4 +214,132 @@ uint GetDataLength (FileInfo? info) { return (uint)info.Length; } } + + void WriteHeader (BinaryWriter writer, AssemblyStoreHeader header) + { + writer.Write (header.magic); + writer.Write (header.version); + writer.Write (header.entry_count); + writer.Write (header.index_entry_count); + writer.Write (header.index_size); + } +#if XABT_TESTS + AssemblyStoreHeader ReadHeader (BinaryReader reader) + { + reader.BaseStream.Seek (0, SeekOrigin.Begin); + uint magic = reader.ReadUInt32 (); + uint version = reader.ReadUInt32 (); + uint entry_count = reader.ReadUInt32 (); + uint index_entry_count = reader.ReadUInt32 (); + uint index_size = reader.ReadUInt32 (); + + return new AssemblyStoreHeader (magic, version, entry_count, index_entry_count, index_size); + } +#endif + + void WriteIndex (BinaryWriter writer, StreamWriter manifestWriter, List index, List descriptors, bool is64Bit) + { + index.Sort ((AssemblyStoreIndexEntry a, AssemblyStoreIndexEntry b) => a.name_hash.CompareTo (b.name_hash)); + + foreach (AssemblyStoreIndexEntry entry in index) { + if (is64Bit) { + writer.Write (entry.name_hash); + manifestWriter.Write ($"0x{entry.name_hash:x}"); + } else { + writer.Write ((uint)entry.name_hash); + manifestWriter.Write ($"0x{(uint)entry.name_hash:x}"); + } + writer.Write (entry.descriptor_index); + manifestWriter.Write ($" di:{entry.descriptor_index}"); + + AssemblyStoreEntryDescriptor desc = descriptors[(int)entry.descriptor_index]; + manifestWriter.Write ($" mi:{desc.mapping_index}"); + manifestWriter.Write ($" do:{desc.data_offset}"); + manifestWriter.Write ($" ds:{desc.data_size}"); + manifestWriter.Write ($" ddo:{desc.debug_data_offset}"); + manifestWriter.Write ($" dds:{desc.debug_data_size}"); + manifestWriter.Write ($" cdo:{desc.config_data_offset}"); + manifestWriter.Write ($" cds:{desc.config_data_size}"); + manifestWriter.WriteLine ($" {entry.name}"); + } + } + + List ReadIndex (BinaryReader reader, AssemblyStoreHeader header) + { + if (header.index_entry_count > Int32.MaxValue) { + throw new InvalidOperationException ("Assembly store index is too big"); + } + + var index = new List ((int)header.index_entry_count); + reader.BaseStream.Seek (AssemblyStoreHeader.NativeSize, SeekOrigin.Begin); + + bool is64Bit = (header.version & ASSEMBLY_STORE_FORMAT_VERSION_64BIT) == ASSEMBLY_STORE_FORMAT_VERSION_64BIT; + for (int i = 0; i < (int)header.index_entry_count; i++) { + ulong name_hash; + if (is64Bit) { + name_hash = reader.ReadUInt64 (); + } else { + name_hash = reader.ReadUInt32 (); + } + + uint descriptor_index = reader.ReadUInt32 (); + index.Add (new AssemblyStoreIndexEntry (String.Empty, name_hash, descriptor_index)); + } + + return index; + } + + void WriteDescriptors (BinaryWriter writer, List descriptors) + { + foreach (AssemblyStoreEntryDescriptor desc in descriptors) { + writer.Write (desc.mapping_index); + writer.Write (desc.data_offset); + writer.Write (desc.data_size); + writer.Write (desc.debug_data_offset); + writer.Write (desc.debug_data_size); + writer.Write (desc.config_data_offset); + writer.Write (desc.config_data_size); + } + } + + List ReadDescriptors (BinaryReader reader, AssemblyStoreHeader header) + { + if (header.entry_count > Int32.MaxValue) { + throw new InvalidOperationException ("Assembly store descriptor table is too big"); + } + + var descriptors = new List (); + reader.BaseStream.Seek (AssemblyStoreHeader.NativeSize + header.index_size, SeekOrigin.Begin); + + for (int i = 0; i < (int)header.entry_count; i++) { + uint mapping_index = reader.ReadUInt32 (); + uint data_offset = reader.ReadUInt32 (); + uint data_size = reader.ReadUInt32 (); + uint debug_data_offset = reader.ReadUInt32 (); + uint debug_data_size = reader.ReadUInt32 (); + uint config_data_offset = reader.ReadUInt32 (); + uint config_data_size = reader.ReadUInt32 (); + + var desc = new AssemblyStoreEntryDescriptor { + mapping_index = mapping_index, + data_offset = data_offset, + data_size = data_size, + debug_data_offset = debug_data_offset, + debug_data_size = debug_data_size, + config_data_offset = config_data_offset, + config_data_size = config_data_size, + }; + descriptors.Add (desc); + } + + return descriptors; + } + + void WriteNames (BinaryWriter writer, List infos) + { + foreach (AssemblyStoreAssemblyInfo info in infos) { + writer.Write ((uint)info.AssemblyNameBytes.Length); + writer.Write (info.AssemblyNameBytes); + } + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index 2baa5c0fcc6..ed9dacb6352 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -39,19 +39,6 @@ public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter protected virtual void CleanupAfterGeneration (AndroidTargetArch arch) {} - public static byte[] StringToBytes (string str) => Encoding.UTF8.GetBytes (str); - - public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (StringToBytes (str), is64Bit); - - public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) - { - if (is64Bit) { - return XxHash64.HashToUInt64 (stringBytes); - } - - return (ulong)XxHash32.HashToUInt32 (stringBytes); - } - protected LlvmIrGlobalVariable EnsureGlobalVariable (LlvmIrVariable variable) { var gv = variable as LlvmIrGlobalVariable; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 92e557c50d0..b399bfb6f1a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -982,10 +982,10 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) foreach (string name in uniqueAssemblyNames) { // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here string clippedName = Path.Combine (Path.GetDirectoryName (name) ?? String.Empty, Path.GetFileNameWithoutExtension (name)); - ulong hashFull32 = GetXxHash (name, is64Bit: false); - ulong hashClipped32 = GetXxHash (clippedName, is64Bit: false); - ulong hashFull64 = GetXxHash (name, is64Bit: true); - ulong hashClipped64 = GetXxHash (clippedName, is64Bit: true); + ulong hashFull32 = MonoAndroidHelper.GetXxHash (name, is64Bit: false); + ulong hashClipped32 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: false); + ulong hashFull64 = MonoAndroidHelper.GetXxHash (name, is64Bit: true); + ulong hashClipped64 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: true); // // If the number of name forms changes, xamarin-app.hh MUST be updated to set value of the diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index bae6eeddeb2..ee47c087afe 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.Linq; using System.IO; +using System.IO.Hashing; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using System.Security.Cryptography; @@ -685,5 +686,18 @@ public static string MakeZipArchivePath (string part1, ICollection? path return String.Join ("/", parts); } + + public static byte[] Utf8StringToBytes (string str) => Encoding.UTF8.GetBytes (str); + + public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (Utf8StringToBytes (str), is64Bit); + + public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) + { + if (is64Bit) { + return XxHash64.HashToUInt64 (stringBytes); + } + + return (ulong)XxHash32.HashToUInt32 (stringBytes); + } } } diff --git a/tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs new file mode 100644 index 00000000000..da64aed81df --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +class AssemblyStoreExplorer +{ + readonly AssemblyStoreReader reader; + + public string StorePath { get; } + public AndroidTargetArch? TargetArch => reader.TargetArch; + public uint AssemblyCount => reader.AssemblyCount; + public uint IndexEntryCount => reader.IndexEntryCount; + public IList? Assemblies => reader.Assemblies; + public bool Is64Bit => reader.Is64Bit; + + public AssemblyStoreExplorer (FileInfo storeInfo) + { + Stream storeStream = storeInfo.OpenRead (); + var storeReader = AssemblyStoreReader.Create (storeStream, storeInfo.FullName); + if (storeReader == null) { + storeStream.Dispose (); + throw new NotSupportedException ($"Format of assembly store '{storeInfo.FullName}' is unsupported"); + } + + reader = storeReader; + StorePath = storeInfo.FullName; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStoreItem.cs b/tools/assembly-store-reader-mk2/AssemblyStoreItem.cs new file mode 100644 index 00000000000..3336825eb9e --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStoreItem.cs @@ -0,0 +1,23 @@ +using System.Collections.Generic; + +namespace Xamarin.Android.AssemblyStore; + +abstract class AssemblyStoreItem +{ + public string Name { get; } + public IList Hashes { get; } + public bool Is64Bit { get; } + public uint DataOffset { get; protected set; } + public uint DataSize { get; protected set; } + public uint DebugOffset { get; protected set; } + public uint DebugSize { get; protected set; } + public uint ConfigOffset { get; protected set; } + public uint ConfigSize { get; protected set; } + + protected AssemblyStoreItem (string name, bool is64Bit, List hashes) + { + Name = name; + Hashes = hashes.AsReadOnly (); + Is64Bit = is64Bit; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStoreReader.cs b/tools/assembly-store-reader-mk2/AssemblyStoreReader.cs new file mode 100644 index 00000000000..39e4dcdc2fe --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStoreReader.cs @@ -0,0 +1,58 @@ +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +abstract class AssemblyStoreReader +{ + static readonly UTF8Encoding ReaderEncoding = new UTF8Encoding (false); + + protected Stream StoreStream { get; } + public abstract string Description { get; } + public string StorePath { get; } + + public AndroidTargetArch? TargetArch { get; protected set; } + public uint AssemblyCount { get; protected set; } + public uint IndexEntryCount { get; protected set; } + public IList? Assemblies { get; protected set; } + public bool Is64Bit { get; protected set; } + + protected AssemblyStoreReader (Stream store, string path) + { + StoreStream = store; + StorePath = path; + } + + public static AssemblyStoreReader? Create (Stream store, string path) + { + AssemblyStoreReader? reader = MakeReaderReady (new StoreReader_V1 (store, path)); + if (reader != null) { + return reader; + } + + reader = MakeReaderReady (new StoreReader_V2 (store, path)); + if (reader != null) { + return reader; + } + + return null; + } + + static AssemblyStoreReader? MakeReaderReady (AssemblyStoreReader reader) + { + if (!reader.IsSupported ()) { + return null; + } + + reader.Prepare (); + return reader; + } + + protected BinaryReader CreateReader () => new BinaryReader (StoreStream, ReaderEncoding, leaveOpen: true); + + protected abstract bool IsSupported (); + protected abstract void Prepare (); +} diff --git a/tools/assembly-store-reader-mk2/Directory.Build.targets b/tools/assembly-store-reader-mk2/Directory.Build.targets new file mode 100644 index 00000000000..e58eed5ca2c --- /dev/null +++ b/tools/assembly-store-reader-mk2/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/tools/assembly-store-reader-mk2/FileFormat.cs b/tools/assembly-store-reader-mk2/FileFormat.cs new file mode 100644 index 00000000000..4a02e0ae0c3 --- /dev/null +++ b/tools/assembly-store-reader-mk2/FileFormat.cs @@ -0,0 +1,11 @@ +namespace Xamarin.Android.AssemblyStore; + +enum FileFormat +{ + Aab, + AabBase, + Apk, + AssemblyStore, + Zip, + Unknown, +} diff --git a/tools/assembly-store-reader-mk2/Log.cs b/tools/assembly-store-reader-mk2/Log.cs new file mode 100644 index 00000000000..497b6c430d6 --- /dev/null +++ b/tools/assembly-store-reader-mk2/Log.cs @@ -0,0 +1,12 @@ +using System; + +namespace Xamarin.Android.AssemblyStore; + +static class Log +{ + public static void Debug (string message) + { + // TODO: verbosity + Console.WriteLine (message); + } +} diff --git a/tools/assembly-store-reader-mk2/Program.cs b/tools/assembly-store-reader-mk2/Program.cs new file mode 100644 index 00000000000..d3582beffad --- /dev/null +++ b/tools/assembly-store-reader-mk2/Program.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace Xamarin.Android.AssemblyStore; + +class Program +{ + static void ShowHelp () + { + Console.WriteLine ("Usage: read-assembly-store BLOB_PATH [BLOB_PATH ...]"); + Console.WriteLine (); + Console.WriteLine (@" where each BLOB_PATH can point to: + * aab file + * apk file + * index store file (e.g. base_assemblies.blob or assemblies.arm64_v8a.blob.so) + * arch store file (e.g. base_assemblies.arm64_v8a.blob) + * store manifest file (e.g. base_assemblies.manifest) + * store base name (e.g. base or base_assemblies) + + In each case the whole set of stores and manifests will be read (if available). Search for the + various members of the store set (common/main store, arch stores, manifest) is based on this naming + convention: + + {BASE_NAME}[.ARCH_NAME].{blob|manifest} + + Whichever file is referenced in `BLOB_PATH`, the BASE_NAME component is extracted and all the found files are read. + If `BLOB_PATH` points to an aab or an apk, BASE_NAME will always be `assemblies` + +"); + } + + static int WriteErrorAndReturn (string message) + { + Console.Error.WriteLine (message); + return 1; + } + + static int Main (string[] args) + { + if (args.Length == 0) { + ShowHelp (); + return 1; + } + + string inputFile = args[0]; + (FileFormat format, FileInfo? info) = Utils.DetectFileFormat (inputFile); + if (info == null) { + return WriteErrorAndReturn ($"File '{inputFile}' does not exist."); + } + + var stores = new List (); + switch (format) { + case FileFormat.Unknown: + return WriteErrorAndReturn ($"File '{inputFile}' has an unknown format."); + + case FileFormat.Zip: + return WriteErrorAndReturn ($"File '{inputFile}' is a ZIP archive, but not an Android one."); + + case FileFormat.AssemblyStore: + stores.Add (new AssemblyStoreExplorer (info)); + break; + } + + foreach (AssemblyStoreExplorer store in stores) { + var printer = new StorePrettyPrinter (store); + printer.Show (); + } + + return 0; + } + + +} diff --git a/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs b/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs new file mode 100644 index 00000000000..f216238794e --- /dev/null +++ b/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +class StorePrettyPrinter +{ + AssemblyStoreExplorer explorer; + + public StorePrettyPrinter (AssemblyStoreExplorer storeExplorer) + { + explorer = storeExplorer; + } + + public void Show () + { + Console.WriteLine ($"Store: {explorer.StorePath}"); + Console.WriteLine ($" Target architecture: {GetTargetArch (explorer)} ({GetBitness (explorer.Is64Bit)}-bit)"); + Console.WriteLine ($" Assembly count: {explorer.AssemblyCount}"); + Console.WriteLine ($" Index entry count: {explorer.IndexEntryCount}"); + Console.WriteLine (); + + if (explorer.Assemblies == null || explorer.Assemblies.Count == 0) { + Console.WriteLine ("NO ASSEMBLIES!"); + return; + } + Console.WriteLine ("Assemblies:"); + var line = new StringBuilder (); + foreach (AssemblyStoreItem assembly in explorer.Assemblies) { + line.Clear (); + line.Append (" "); + line.Append (assembly.Name); + line.Append (' '); + line.Append (FormatOffsetAndSize (assembly.DataOffset, assembly.DataSize)); + line.Append (' '); + line.Append (FormatOffsetAndSize (assembly.DebugOffset, assembly.DebugSize)); + line.Append (' '); + line.Append (FormatOffsetAndSize (assembly.ConfigOffset, assembly.ConfigSize)); + line.Append (" ["); + line.Append (FormatHashes (assembly.Hashes)); + line.Append (']'); + Console.WriteLine (line.ToString ()); + } + Console.WriteLine (); + } + + static string FormatOffsetAndSize (uint offset, uint size) + { + if (offset == 0) { + return "none"; + } + + return $"{offset} ({size})"; + } + + static string FormatHashes (IList hashes) + { + if (hashes.Count == 0) { + return "none"; + } + + var ret = new List (); + foreach (ulong hash in hashes) { + ret.Add ($"0x{hash:x}"); + } + + return String.Join (',', ret); + } + + static string GetBitness (bool is64bit) => is64bit ? "64" : "32"; + + static string GetTargetArch (AssemblyStoreExplorer storeExplorer) + { + if (storeExplorer.TargetArch == null) { + return "ABI agnostic"; + } + + return storeExplorer.TargetArch switch { + AndroidTargetArch.Arm64 => "Arm64", + AndroidTargetArch.Arm => "Arm32", + AndroidTargetArch.X86_64 => "x64", + AndroidTargetArch.X86 => "x86", + _ => throw new NotSupportedException ($"Unsupported target architecture {storeExplorer.TargetArch}") + }; + } +} diff --git a/tools/assembly-store-reader-mk2/StoreReader_V1.cs b/tools/assembly-store-reader-mk2/StoreReader_V1.cs new file mode 100644 index 00000000000..87850329750 --- /dev/null +++ b/tools/assembly-store-reader-mk2/StoreReader_V1.cs @@ -0,0 +1,22 @@ +using System; +using System.IO; + +namespace Xamarin.Android.AssemblyStore; + +class StoreReader_V1 : AssemblyStoreReader +{ + public override string Description => "Assembly store v1"; + + public StoreReader_V1 (Stream store, string path) + : base (store, path) + {} + + protected override bool IsSupported () + { + return false; + } + + protected override void Prepare () + { + } +} diff --git a/tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs b/tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs new file mode 100644 index 00000000000..0c56e9318c2 --- /dev/null +++ b/tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; + +namespace Xamarin.Android.AssemblyStore; + +partial class StoreReader_V2 +{ + sealed class Header + { + public const uint NativeSize = 5 * sizeof (uint); + + public readonly uint magic; + public readonly uint version; + public readonly uint entry_count; + public readonly uint index_entry_count; + + // Index size in bytes + public readonly uint index_size; + + public Header (uint magic, uint version, uint entry_count, uint index_entry_count, uint index_size) + { + this.magic = magic; + this.version = version; + this.entry_count = entry_count; + this.index_entry_count = index_entry_count; + this.index_size = index_size; + } + } + + sealed class IndexEntry + { + public string? name; + public readonly ulong name_hash; + public readonly uint descriptor_index; + + public IndexEntry (ulong name_hash, uint descriptor_index) + { + this.name_hash = name_hash; + this.descriptor_index = descriptor_index; + } + } + + sealed class EntryDescriptor + { + public uint mapping_index; + + public uint data_offset; + public uint data_size; + + public uint debug_data_offset; + public uint debug_data_size; + + public uint config_data_offset; + public uint config_data_size; + } + + sealed class StoreItem_V2 : AssemblyStoreItem + { + public StoreItem_V2 (string name, bool is64Bit, List indexEntries, EntryDescriptor descriptor) + : base (name, is64Bit, IndexToHashes (indexEntries)) + { + DataOffset = descriptor.data_offset; + DataSize = descriptor.data_size; + DebugOffset = descriptor.debug_data_offset; + DebugSize = descriptor.debug_data_size; + ConfigOffset = descriptor.config_data_offset; + ConfigSize = descriptor.config_data_size; + } + + static List IndexToHashes (List indexEntries) + { + var ret = new List (); + foreach (IndexEntry ie in indexEntries) { + ret.Add (ie.name_hash); + } + + return ret; + } + } + + sealed class TemporaryItem + { + public readonly string Name; + public readonly List IndexEntries = new List (); + public readonly EntryDescriptor Descriptor; + + public TemporaryItem (string name, EntryDescriptor descriptor) + { + Name = name; + Descriptor = descriptor; + } + } +} diff --git a/tools/assembly-store-reader-mk2/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/StoreReader_V2.cs new file mode 100644 index 00000000000..6a83513565f --- /dev/null +++ b/tools/assembly-store-reader-mk2/StoreReader_V2.cs @@ -0,0 +1,149 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +partial class StoreReader_V2 : AssemblyStoreReader +{ + // Bit 31 is set for 64-bit platforms, cleared for the 32-bit ones + const uint ASSEMBLY_STORE_FORMAT_VERSION_64BIT = 0x80000002; // Must match the ASSEMBLY_STORE_FORMAT_VERSION native constant + const uint ASSEMBLY_STORE_FORMAT_VERSION_32BIT = 0x00000002; + const uint ASSEMBLY_STORE_FORMAT_VERSION_MASK = 0xF0000000; + + const uint ASSEMBLY_STORE_ABI_AARCH64 = 0x00010000; + const uint ASSEMBLY_STORE_ABI_ARM = 0x00020000; + const uint ASSEMBLY_STORE_ABI_X64 = 0x00030000; + const uint ASSEMBLY_STORE_ABI_X86 = 0x00040000; + const uint ASSEMBLY_STORE_ABI_MASK = 0x00FF0000; + + public override string Description => "Assembly store v2"; + + readonly HashSet supportedVersions; + + Header? header; + + public StoreReader_V2 (Stream store, string path) + : base (store, path) + { + supportedVersions = new HashSet { + ASSEMBLY_STORE_FORMAT_VERSION_64BIT | ASSEMBLY_STORE_ABI_AARCH64, + ASSEMBLY_STORE_FORMAT_VERSION_64BIT | ASSEMBLY_STORE_ABI_X64, + ASSEMBLY_STORE_FORMAT_VERSION_32BIT | ASSEMBLY_STORE_ABI_ARM, + ASSEMBLY_STORE_FORMAT_VERSION_32BIT | ASSEMBLY_STORE_ABI_X86, + }; + } + + protected override bool IsSupported () + { + StoreStream.Seek (0, SeekOrigin.Begin); + using var reader = CreateReader (); + + uint magic = reader.ReadUInt32 (); + if (magic != Utils.ASSEMBLY_STORE_MAGIC) { + Log.Debug ($"Store '{StorePath}' has invalid header magic number."); + return false; + } + + uint version = reader.ReadUInt32 (); + if (!supportedVersions.Contains (version)) { + Log.Debug ($"Store '{StorePath}' has unsupported version 0x{version:x}"); + return false; + } + + uint entry_count = reader.ReadUInt32 (); + uint index_entry_count = reader.ReadUInt32 (); + uint index_size = reader.ReadUInt32 (); + + header = new Header (magic, version, entry_count, index_entry_count, index_size); + return true; + } + + protected override void Prepare () + { + if (header == null) { + throw new InvalidOperationException ("Internal error: header not set, was IsSupported() called?"); + } + + TargetArch = (header.version & ASSEMBLY_STORE_ABI_MASK) switch { + ASSEMBLY_STORE_ABI_AARCH64 => AndroidTargetArch.Arm64, + ASSEMBLY_STORE_ABI_ARM => AndroidTargetArch.Arm, + ASSEMBLY_STORE_ABI_X64 => AndroidTargetArch.X86_64, + ASSEMBLY_STORE_ABI_X86 => AndroidTargetArch.X86, + _ => throw new NotSupportedException ($"Unsupported ABI in store version: 0x{header.version:x}") + }; + + Is64Bit = (header.version & ASSEMBLY_STORE_FORMAT_VERSION_MASK) != 0; + AssemblyCount = header.entry_count; + IndexEntryCount = header.index_entry_count; + + StoreStream.Seek (Header.NativeSize, SeekOrigin.Begin); + using var reader = CreateReader (); + + var index = new List (); + for (uint i = 0; i < header.index_entry_count; i++) { + ulong name_hash; + if (Is64Bit) { + name_hash = reader.ReadUInt64 (); + } else { + name_hash = (ulong)reader.ReadUInt32 (); + } + + uint descriptor_index = reader.ReadUInt32 (); + index.Add (new IndexEntry (name_hash, descriptor_index)); + } + + var descriptors = new List (); + for (uint i = 0; i < header.entry_count; i++) { + uint mapping_index = reader.ReadUInt32 (); + uint data_offset = reader.ReadUInt32 (); + uint data_size = reader.ReadUInt32 (); + uint debug_data_offset = reader.ReadUInt32 (); + uint debug_data_size = reader.ReadUInt32 (); + uint config_data_offset = reader.ReadUInt32 (); + uint config_data_size = reader.ReadUInt32 (); + + var desc = new EntryDescriptor { + mapping_index = mapping_index, + data_offset = data_offset, + data_size = data_size, + debug_data_offset = debug_data_offset, + debug_data_size = debug_data_size, + config_data_offset = config_data_offset, + config_data_size = config_data_size, + }; + descriptors.Add (desc); + } + + var names = new List (); + for (uint i = 0; i < header.entry_count; i++) { + uint name_length = reader.ReadUInt32 (); + byte[] name_bytes = reader.ReadBytes ((int)name_length); + names.Add (Encoding.UTF8.GetString (name_bytes)); + } + + var tempItems = new Dictionary (); + foreach (IndexEntry ie in index) { + if (!tempItems.TryGetValue (ie.descriptor_index, out TemporaryItem? item)) { + item = new TemporaryItem (names[(int)ie.descriptor_index], descriptors[(int)ie.descriptor_index]); + tempItems.Add (ie.descriptor_index, item); + } + item.IndexEntries.Add (ie); + } + + if (tempItems.Count != descriptors.Count) { + throw new InvalidOperationException ($"Assembly store '{StorePath}' index is corrupted."); + } + + var storeItems = new List (); + foreach (var kvp in tempItems) { + TemporaryItem ti = kvp.Value; + var item = new StoreItem_V2 (ti.Name, Is64Bit, ti.IndexEntries, ti.Descriptor); + storeItems.Add (item); + } + Assemblies = storeItems.AsReadOnly (); + } +} diff --git a/tools/assembly-store-reader-mk2/Utils.cs b/tools/assembly-store-reader-mk2/Utils.cs new file mode 100644 index 00000000000..d36d86fbd71 --- /dev/null +++ b/tools/assembly-store-reader-mk2/Utils.cs @@ -0,0 +1,85 @@ +using System; +using System.IO; +using System.Buffers; + +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.AssemblyStore; + +static class Utils +{ + static readonly string[] aabZipEntries = { + "base/manifest/AndroidManifest.xml", + "BundleConfig.pb", + }; + + static readonly string[] aabBaseZipEntries = { + "manifest/AndroidManifest.xml", + }; + + static readonly string[] apkZipEntries = { + "AndroidManifest.xml", + }; + + public const uint ZIP_MAGIC = 0x4034b50; + public const uint ASSEMBLY_STORE_MAGIC = 0x41424158; + + public static readonly ArrayPool BytePool = ArrayPool.Shared; + + public static (FileFormat format, FileInfo? info) DetectFileFormat (string path) + { + if (String.IsNullOrEmpty (path)) { + return (FileFormat.Unknown, null); + } + + var info = new FileInfo (path); + if (!info.Exists) { + return (FileFormat.Unknown, null); + } + + using var reader = new BinaryReader (info.OpenRead ()); + + // ATM, all formats we recognize have 4-byte magic at the start + FileFormat format = reader.ReadUInt32 () switch { + Utils.ZIP_MAGIC => FileFormat.Zip, + Utils.ASSEMBLY_STORE_MAGIC => FileFormat.AssemblyStore, + _ => FileFormat.Unknown + }; + + if (format == FileFormat.Unknown || format != FileFormat.Zip) { + return (format, info); + } + + return (DetectAndroidArchive (info, format), info); + } + + static FileFormat DetectAndroidArchive (FileInfo info, FileFormat defaultFormat) + { + using var zip = ZipArchive.Open (info.FullName, FileMode.Open); + + if (HasAllEntries (zip, aabZipEntries)) { + return FileFormat.Aab; + } + + if (HasAllEntries (zip, apkZipEntries)) { + return FileFormat.Apk; + } + + if (HasAllEntries (zip, aabBaseZipEntries)) { + return FileFormat.AabBase; + } + + return defaultFormat; + } + + static bool HasAllEntries (ZipArchive zip, string[] entries) + { + foreach (string entry in entries) { + if (!zip.ContainsEntry (entry, caseSensitive: true)) { + return false; + } + } + + return true; + } +} diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj new file mode 100644 index 00000000000..b6fb7474b07 --- /dev/null +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -0,0 +1,28 @@ + + + + + Microsoft Corporation + 2023 Microsoft Corporation + 0.0.2 + false + ../../bin/$(Configuration)/bin/assembly-store-reader + Exe + $(DotNetStableTargetFramework) + Xamarin.Android.AssemblyStoreReader + disable + enable + + + + + + + + + + + + + + diff --git a/tools/assembly-store-reader/Program.cs b/tools/assembly-store-reader/Program.cs index 6052ed396ce..84dbe5703c6 100644 --- a/tools/assembly-store-reader/Program.cs +++ b/tools/assembly-store-reader/Program.cs @@ -123,7 +123,7 @@ static int Main (string[] args) Console.Error.WriteLine (@" where each BLOB_PATH can point to: * aab file * apk file - * index store file (e.g. base_assemblies.blob) + * index store file (e.g. base_assemblies.blob or assemblies.arm64_v8a.blob.so) * arch store file (e.g. base_assemblies.arm64_v8a.blob) * store manifest file (e.g. base_assemblies.manifest) * store base name (e.g. base or base_assemblies) diff --git a/tools/assembly-store-reader/assembly-store-reader.csproj b/tools/assembly-store-reader/assembly-store-reader.csproj index 4e85b5fc9d5..9b0e604b1da 100644 --- a/tools/assembly-store-reader/assembly-store-reader.csproj +++ b/tools/assembly-store-reader/assembly-store-reader.csproj @@ -3,8 +3,8 @@ Microsoft Corporation - 2021 Microsoft Corporation - 0.0.1 + 2023 Microsoft Corporation + 0.0.2 false ../../bin/$(Configuration)/bin/assembly-store-reader Exe From 7721aaece138157b6bf0e086be2bbcc512f9adbb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 17 Nov 2023 22:06:49 +0100 Subject: [PATCH 019/143] Continued work on new assembly store reader + tests --- .../Utilities/ArchiveAssemblyHelper.cs | 159 +++++++-------- .../Xamarin.Android.Build.Tests.csproj | 2 +- .../Utilities/MonoAndroidHelper.Simple.cs | 123 +++++++++++ .../Utilities/MonoAndroidHelper.cs | 134 +----------- .../MSBuildDeviceIntegration.csproj | 2 +- .../AssemblyStore/AssemblyStoreExplorer.cs | 192 ++++++++++++++++++ .../AssemblyStore/AssemblyStoreItem.cs | 26 +++ .../AssemblyStoreReader.cs | 2 +- .../{ => AssemblyStore}/FileFormat.cs | 0 .../{ => AssemblyStore}/Log.cs | 0 .../AssemblyStore/StoreReader_V1.cs | 33 +++ .../StoreReader_V2.Classes.cs | 6 +- .../{ => AssemblyStore}/StoreReader_V2.cs | 47 ++++- .../{ => AssemblyStore}/Utils.cs | 0 .../AssemblyStoreExplorer.cs | 32 --- .../AssemblyStoreItem.cs | 23 --- tools/assembly-store-reader-mk2/Program.cs | 16 +- .../StorePrettyPrinter.cs | 58 ++++-- .../StoreReader_V1.cs | 22 -- .../assembly-store-reader.csproj | 5 + 20 files changed, 561 insertions(+), 321 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs create mode 100644 tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs create mode 100644 tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs rename tools/assembly-store-reader-mk2/{ => AssemblyStore}/AssemblyStoreReader.cs (94%) rename tools/assembly-store-reader-mk2/{ => AssemblyStore}/FileFormat.cs (100%) rename tools/assembly-store-reader-mk2/{ => AssemblyStore}/Log.cs (100%) create mode 100644 tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs rename tools/assembly-store-reader-mk2/{ => AssemblyStore}/StoreReader_V2.Classes.cs (91%) rename tools/assembly-store-reader-mk2/{ => AssemblyStore}/StoreReader_V2.cs (78%) rename tools/assembly-store-reader-mk2/{ => AssemblyStore}/Utils.cs (100%) delete mode 100644 tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs delete mode 100644 tools/assembly-store-reader-mk2/AssemblyStoreItem.cs delete mode 100644 tools/assembly-store-reader-mk2/StoreReader_V1.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 3966c85180a..9f4657b9699 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -5,6 +5,8 @@ using System.Linq; using Xamarin.Android.AssemblyStore; +using Xamarin.Android.Tools; +using Xamarin.Android.Tasks; using Xamarin.ProjectTools; using Xamarin.Tools.Zip; @@ -21,13 +23,6 @@ public class ArchiveAssemblyHelper ".pdb", }; - static readonly Dictionary ArchToAbi = new Dictionary (StringComparer.OrdinalIgnoreCase) { - {"x86", "x86"}, - {"x86_64", "x86_64"}, - {"armeabi_v7a", "armeabi-v7a"}, - {"arm64_v8a", "arm64-v8a"}, - }; - static readonly ArrayPool buffers = ArrayPool.Shared; readonly string archivePath; @@ -60,16 +55,16 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, } } - public Stream ReadEntry (string path) + public Stream? ReadEntry (string path, AndroidTargetArch arch = AndroidTargetArch.None, bool uncompressIfNecessary = false) { if (useAssemblyStores) { - return ReadStoreEntry (path); + return ReadStoreEntry (path, arch, uncompressIfNecessary); } - return ReadZipEntry (path); + return ReadZipEntry (path, arch, uncompressIfNecessary); } - Stream ReadZipEntry (string path) + Stream? ReadZipEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) { using (var zip = ZipHelper.OpenZip (archivePath)) { ZipEntry entry = zip.ReadEntry (path); @@ -80,61 +75,39 @@ Stream ReadZipEntry (string path) } } - Stream ReadStoreEntry (string path) + Stream? ReadStoreEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) { - AssemblyStoreReader storeReader = null; - AssemblyStoreAssembly assembly = null; string name = Path.GetFileNameWithoutExtension (path); - var explorer = new AssemblyStoreExplorer (archivePath); - - foreach (var asm in explorer.Assemblies) { - if (String.Compare (name, asm.Name, StringComparison.Ordinal) != 0) { - continue; - } - assembly = asm; - storeReader = asm.Store; - break; - } - - if (storeReader == null) { - Console.WriteLine ($"Store for entry {path} not found, will try a standard Zip read"); - return ReadZipEntry (path); - } - - string storeEntryName; - if (String.IsNullOrEmpty (storeReader.Arch)) { - storeEntryName = $"{assembliesRootDir}assemblies.blob"; - } else { - storeEntryName = $"{assembliesRootDir}assemblies_{storeReader.Arch}.blob"; + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); + AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); + if (explorer == null) { + Console.WriteLine ($"Failed to read assembly '{name}' from '{archivePath}'. {errorMessage}"); + return null; } - Stream store = ReadZipEntry (storeEntryName); - if (store == null) { - Console.WriteLine ($"Store zip entry {storeEntryName} does not exist"); + IList? assemblies = explorer.Find (name, arch); + if (assemblies == null) { + Console.WriteLine ($"Failed to locate assembly '{name}' in assembly store for architecture '{arch}', in archive '{archivePath}'"); return null; } - store.Seek (assembly.DataOffset, SeekOrigin.Begin); - var ret = new MemoryStream (); - byte[] buffer = buffers.Rent (AssemblyStoreReadBufferSize); - int toRead = (int)assembly.DataSize; - while (toRead > 0) { - int nread = store.Read (buffer, 0, AssemblyStoreReadBufferSize); - if (nread <= 0) { + AssemblyStoreItem? assembly = null; + foreach (AssemblyStoreItem item in assemblies) { + if (arch == AndroidTargetArch.None || item.TargetArch == arch) { + assembly = item; break; } + } - ret.Write (buffer, 0, nread); - toRead -= nread; + if (assembly == null) { + Console.WriteLine ($"Failed to find assembly '{name}' in assembly store for architecture '{arch}', in archive '{archivePath}'"); + return null; } - ret.Flush (); - store.Dispose (); - buffers.Return (buffer); - return ret; + return explorer.Read (assembly, uncompressIfNecessary); } - public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false) + public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) { if (!forceRefresh && archiveContents != null) { return archiveContents; @@ -158,22 +131,24 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb } Console.WriteLine ($"Creating AssemblyStoreExplorer for archive '{archivePath}'"); - var explorer = new AssemblyStoreExplorer (archivePath); - Console.WriteLine ($"Explorer found {explorer.Assemblies.Count} assemblies"); - foreach (var asm in explorer.Assemblies) { + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); + AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); + Console.WriteLine ($"Explorer found {explorer.AssemblyCount} assemblies"); + + foreach (AssemblyStoreItem asm in explorer.Assemblies) { string prefix = storeEntryPrefix; - if (haveMultipleRids && !String.IsNullOrEmpty (asm.Store.Arch)) { - string arch = ArchToAbi[asm.Store.Arch]; - prefix = $"{prefix}{arch}/"; + if (haveMultipleRids && asm.TargetArch != AndroidTargetArch.None) { + string abi = MonoAndroidHelper.ArchToAbi (asm.TargetArch); + prefix = $"{prefix}{abi}/"; } entries.Add ($"{prefix}{asm.Name}.dll"); - if (asm.DebugDataOffset > 0) { + if (asm.DebugOffset > 0) { entries.Add ($"{prefix}{asm.Name}.pdb"); } - if (asm.ConfigDataOffset > 0) { + if (asm.ConfigOffset > 0) { entries.Add ($"{prefix}{asm.Name}.dll.config"); } } @@ -186,6 +161,30 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb return entries; } + AssemblyStoreExplorer? SelectExplorer (IList? explorers, AndroidTargetArch arch) + { + if (explorers == null || explorers.Count == 0) { + return null; + } + + // If we don't care about target architecture, we check the first store, since all of them will have the same + // assemblies. Otherwise we try to locate the correct store. + if (arch == AndroidTargetArch.None) { + return explorers[0]; + } + + foreach (AssemblyStoreExplorer e in explorers) { + if (e.TargetArch == null || e.TargetArch != arch) { + continue; + } + return e; + } + + + Console.WriteLine ($"Failed to find assembly store for architecture '{arch}' in archive '{archivePath}'"); + return null; + } + public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool forceRefresh = false) { List contents = ListArchiveContents (assembliesRootDir, forceRefresh); @@ -217,7 +216,7 @@ public bool Exists (string entryPath, bool forceRefresh = false) return contents.Contains (entryPath); } - public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch = AndroidTargetArch.None) { if (fileNames == null) { throw new ArgumentNullException (nameof (fileNames)); @@ -228,13 +227,13 @@ public void Contains (string[] fileNames, out List existingFiles, out Li } if (useAssemblyStores) { - StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, arch); } else { - ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles); + ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, arch); } } - void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch) { using (var zip = ZipHelper.OpenZip (archivePath)) { existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); @@ -243,11 +242,11 @@ void ArchiveContains (string[] fileNames, out List existingFiles, out Li } } - void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles) + void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch) { var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); - var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase) || x.EndsWith (".mdb", StringComparison.OrdinalIgnoreCase)).ToList (); + var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase)).ToList (); var otherFiles = fileNames.Where (x => !SpecialExtensions.Contains (Path.GetExtension (x))).ToList (); existingFiles = new List (); @@ -265,38 +264,38 @@ void StoreContains (string[] fileNames, out List existingFiles, out List } } - var explorer = new AssemblyStoreExplorer (archivePath, customLogger: (a, s) => { - Console.WriteLine ($"DEBUG! {s}"); - }); + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); + AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); + if (explorer == null) { + return; + } foreach (var f in explorer.AssembliesByName) { Console.WriteLine ($"DEBUG!\tKey:{f.Key}"); } - // Assembly stores don't store the assembly extension - var storeAssemblies = explorer.AssembliesByName.Keys.Select (x => $"{x}.dll"); if (explorer.AssembliesByName.Count != 0) { - existingFiles.AddRange (storeAssemblies); + existingFiles.AddRange (explorer.AssembliesByName.Keys); // We need to fake config and debug files since they have no named entries in the storeReader foreach (string file in configFiles) { - AssemblyStoreAssembly asm = GetStoreAssembly (file); + AssemblyStoreItem asm = GetStoreAssembly (file); if (asm == null) { continue; } - if (asm.ConfigDataOffset > 0) { + if (asm.ConfigOffset > 0) { existingFiles.Add (file); } } foreach (string file in debugFiles) { - AssemblyStoreAssembly asm = GetStoreAssembly (file); + AssemblyStoreItem asm = GetStoreAssembly (file); if (asm == null) { continue; } - if (asm.DebugDataOffset > 0) { + if (asm.DebugOffset > 0) { existingFiles.Add (file); } } @@ -311,14 +310,10 @@ void StoreContains (string[] fileNames, out List existingFiles, out List additionalFiles = existingFiles.Where (x => !fileNames.Contains (x)).ToList (); - AssemblyStoreAssembly GetStoreAssembly (string file) + AssemblyStoreItem GetStoreAssembly (string file) { string assemblyName = Path.GetFileNameWithoutExtension (file); - if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - assemblyName = Path.GetFileNameWithoutExtension (assemblyName); - } - - if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreAssembly asm) || asm == null) { + if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreItem asm) || asm == null) { return null; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj index ad22a1aeaed..7d287882eef 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Xamarin.Android.Build.Tests.csproj @@ -35,7 +35,7 @@ - + ..\Expected\GenerateDesignerFileExpected.cs PreserveNewest diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs new file mode 100644 index 00000000000..e1634df2fed --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO.Hashing; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +partial class MonoAndroidHelper +{ + static readonly char[] ZipPathTrimmedChars = {'/', '\\'}; + + static readonly Dictionary ClangAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"arm64-v8a", "aarch64"}, + {"armeabi-v7a", "arm"}, + {"x86", "i686"}, + {"x86_64", "x86_64"} + }; + + public static AndroidTargetArch AbiToTargetArch (string abi) + { + return abi switch { + "armeabi-v7a" => AndroidTargetArch.Arm, + "arm64-v8a" => AndroidTargetArch.Arm64, + "x86_64" => AndroidTargetArch.X86_64, + "x86" => AndroidTargetArch.X86, + _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") + }; + } + + public static string AbiToRid (string abi) + { + return abi switch { + "armeabi-v7a" => "android-arm", + "arm64-v8a" => "android-arm64", + "x86_64" => "android-x64", + "x86" => "android-x86", + _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") + }; + } + + public static string ArchToRid (AndroidTargetArch arch) + { + return arch switch { + AndroidTargetArch.Arm64 => "android-arm64", + AndroidTargetArch.Arm => "android-arm", + AndroidTargetArch.X86 => "android-x86", + AndroidTargetArch.X86_64 => "android-x64", + _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") + }; + } + + public static string ArchToAbi (AndroidTargetArch arch) + { + return arch switch { + AndroidTargetArch.Arm64 => "arm64-v8a", + AndroidTargetArch.Arm => "armeabi-v7a", + AndroidTargetArch.X86 => "x86", + AndroidTargetArch.X86_64 => "x86_64", + _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") + }; + } + + public static string? CultureInvariantToString (object? obj) + { + if (obj == null) { + return null; + } + + return Convert.ToString (obj, CultureInfo.InvariantCulture); + } + + public static string MapAndroidAbiToClang (string androidAbi) + { + if (ClangAbiMap.TryGetValue (androidAbi, out string clangAbi)) { + return clangAbi; + } + return null; + } + + public static string MakeZipArchivePath (string part1, params string[]? pathParts) + { + return MakeZipArchivePath (part1, (ICollection?)pathParts); + } + + public static string MakeZipArchivePath (string part1, ICollection? pathParts) + { + var parts = new List (); + if (!String.IsNullOrEmpty (part1)) { + parts.Add (part1.TrimEnd (ZipPathTrimmedChars)); + }; + + if (pathParts != null && pathParts.Count > 0) { + foreach (string p in pathParts) { + if (String.IsNullOrEmpty (p)) { + continue; + } + parts.Add (p.TrimEnd (ZipPathTrimmedChars)); + } + } + + if (parts.Count == 0) { + return String.Empty; + } + + return String.Join ("/", parts); + } + + public static byte[] Utf8StringToBytes (string str) => Encoding.UTF8.GetBytes (str); + + public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (Utf8StringToBytes (str), is64Bit); + + public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) + { + if (is64Bit) { + return XxHash64.HashToUInt64 (stringBytes); + } + + return (ulong)XxHash32.HashToUInt32 (stringBytes); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index ee47c087afe..2593ac422e0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -1,19 +1,16 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; using System.Linq; using System.IO; -using System.IO.Hashing; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; -using System.Security.Cryptography; using System.Text; using Xamarin.Android.Tools; using Xamarin.Tools.Zip; -using Microsoft.Android.Build.Tasks; #if MSBUILD +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; #endif @@ -22,7 +19,6 @@ namespace Xamarin.Android.Tasks { public partial class MonoAndroidHelper { - static readonly char[] ZipPathTrimmedChars = {'/', '\\'}; static Lazy uname = new Lazy (GetOSBinDirName, System.Threading.LazyThreadSafetyMode.PublicationOnly); // Set in ResolveSdks.Execute(); @@ -279,22 +275,6 @@ public static void LogWarning (object log, string msg, params object [] args) } #if MSBUILD - static readonly Dictionary ClangAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { - {"arm64-v8a", "aarch64"}, - {"armeabi-v7a", "arm"}, - {"x86", "i686"}, - {"x86_64", "x86_64"} - }; - - public static string MapAndroidAbiToClang (string androidAbi) - { - if (ClangAbiMap.TryGetValue (androidAbi, out string clangAbi)) { - return clangAbi; - } - return null; - } -#endif - public static bool IsMonoAndroidAssembly (ITaskItem assembly) { var tfi = assembly.GetMetadata ("TargetFrameworkIdentifier"); @@ -315,6 +295,7 @@ public static bool HasMonoAndroidReference (ITaskItem assembly) var reader = pe.GetMetadataReader (); return HasMonoAndroidReference (reader); } +#endif public static bool HasMonoAndroidReference (MetadataReader reader) { @@ -389,7 +370,6 @@ internal static IEnumerable GetFrameworkAssembliesToTreatAsUserAssemb return ret; } -#endif public static bool SaveMapFile (IBuildEngine4 engine, string mapFile, Dictionary map) { @@ -450,6 +430,7 @@ public static bool SaveCustomViewMapFile (IBuildEngine4 engine, string mapFile, return Files.CopyIfStreamChanged (writer.BaseStream, mapFile); } } +#endif // MSBUILD public static string [] GetProguardEnvironmentVaribles (string proguardHome) { @@ -484,6 +465,7 @@ public static IEnumerable Executables (string executable) yield return executable; } +#if MSBUILD public static string TryGetAndroidJarPath (TaskLoggingHelper log, string platform, bool designTimeBuild = false) { var platformPath = MonoAndroidHelper.AndroidSdk.TryGetPlatformDirectoryFromApiLevel (platform, MonoAndroidHelper.SupportedVersions); @@ -505,7 +487,7 @@ public static void SaveResourceCaseMap (IBuildEngine4 engine, Dictionary LoadResourceCaseMap (IBuildEngine4 engine, Func keyCallback) => engine.GetRegisteredTaskObjectAssemblyLocal> (keyCallback (ResourceCaseMapKey), RegisteredTaskObjectLifetime.Build) ?? new Dictionary (0); - +#endif // MSBUILD public static string FixUpAndroidResourcePath (string file, string resourceDirectory, string resourceDirectoryFullPath, Dictionary resource_name_case_map) { string newfile = null; @@ -525,7 +507,7 @@ public static string FixUpAndroidResourcePath (string file, string resourceDirec } static readonly char [] DirectorySeparators = new [] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; - +#if MSBUILD /// /// Returns the relative path that should be used for an @(AndroidAsset) item /// @@ -537,68 +519,9 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa path = head.Length == path.Length ? path : path.Substring ((head.Length == 0 ? 0 : head.Length + 1) + assetsDirectory.Length).TrimStart (DirectorySeparators); return path; } +#endif // MSBUILD - public static AndroidTargetArch AbiToTargetArch (string abi) - { - return abi switch { - "armeabi-v7a" => AndroidTargetArch.Arm, - "arm64-v8a" => AndroidTargetArch.Arm64, - "x86_64" => AndroidTargetArch.X86_64, - "x86" => AndroidTargetArch.X86, - _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") - }; - } - - public static string AbiToRid (string abi) - { - switch (abi) { - case "arm64-v8a": - return "android-arm64"; - - case "armeabi-v7a": - return "android-arm"; - - case "x86": - return "android-x86"; - - case "x86_64": - return "android-x64"; - - default: - throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); - } - } - - public static string ArchToRid (AndroidTargetArch arch) - { - return arch switch { - AndroidTargetArch.Arm64 => "android-arm64", - AndroidTargetArch.Arm => "android-arm", - AndroidTargetArch.X86 => "android-x86", - AndroidTargetArch.X86_64 => "android-x64", - _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") - }; - } - - public static string ArchToAbi (AndroidTargetArch arch) - { - return arch switch { - AndroidTargetArch.Arm64 => "arm64-v8a", - AndroidTargetArch.Arm => "armeabi-v7a", - AndroidTargetArch.X86 => "x86", - AndroidTargetArch.X86_64 => "x86_64", - _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") - }; - } - - public static string? CultureInvariantToString (object? obj) - { - if (obj == null) { - return null; - } - return Convert.ToString (obj, CultureInfo.InvariantCulture); - } /// /// Converts $(SupportedOSPlatformVersion) to an API level, as it can be a version (21.0), or an int (21). @@ -617,6 +540,7 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) return apiLevel; } +#if MSBUILD public static string GetAssemblyAbi (ITaskItem asmItem) { string? abi = asmItem.GetMetadata ("Abi"); @@ -628,6 +552,7 @@ public static string GetAssemblyAbi (ITaskItem asmItem) } public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) => AbiToTargetArch (GetAssemblyAbi (asmItem)); +#endif // MSBUILD static string GetToolsRootDirectoryRelativePath (string androidBinUtilsDirectory) { @@ -658,46 +583,5 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } - - public static string MakeZipArchivePath (string part1, params string[]? pathParts) - { - return MakeZipArchivePath (part1, (ICollection?)pathParts); - } - - public static string MakeZipArchivePath (string part1, ICollection? pathParts) - { - var parts = new List (); - if (!String.IsNullOrEmpty (part1)) { - parts.Add (part1.TrimEnd (ZipPathTrimmedChars)); - }; - - if (pathParts != null && pathParts.Count > 0) { - foreach (string p in pathParts) { - if (String.IsNullOrEmpty (p)) { - continue; - } - parts.Add (p.TrimEnd (ZipPathTrimmedChars)); - } - } - - if (parts.Count == 0) { - return String.Empty; - } - - return String.Join ("/", parts); - } - - public static byte[] Utf8StringToBytes (string str) => Encoding.UTF8.GetBytes (str); - - public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (Utf8StringToBytes (str), is64Bit); - - public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) - { - if (is64Bit) { - return XxHash64.HashToUInt64 (stringBytes); - } - - return (ulong)XxHash32.HashToUInt32 (stringBytes); - } } } diff --git a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj index 70616562284..eb53ded7436 100644 --- a/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj +++ b/tests/MSBuildDeviceIntegration/MSBuildDeviceIntegration.csproj @@ -24,7 +24,7 @@ - + diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs new file mode 100644 index 00000000000..c36dc9df7a1 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs @@ -0,0 +1,192 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Xamarin.Android.Tools; +using Xamarin.Tools.Zip; + +namespace Xamarin.Android.AssemblyStore; + +class AssemblyStoreExplorer +{ + readonly AssemblyStoreReader reader; + + public string StorePath { get; } + public AndroidTargetArch? TargetArch { get; } + public uint AssemblyCount { get; } + public uint IndexEntryCount { get; } + public IList? Assemblies { get; } + public IDictionary? AssembliesByName { get; } + public bool Is64Bit { get; } + + protected AssemblyStoreExplorer (Stream storeStream, string path) + { + StorePath = path; + var storeReader = AssemblyStoreReader.Create (storeStream, path); + if (storeReader == null) { + storeStream.Dispose (); + throw new NotSupportedException ($"Format of assembly store '{path}' is unsupported"); + } + + reader = storeReader; + TargetArch = reader.TargetArch; + AssemblyCount = reader.AssemblyCount; + IndexEntryCount = reader.IndexEntryCount; + Assemblies = reader.Assemblies; + Is64Bit = reader.Is64Bit; + + var dict = new Dictionary (StringComparer.Ordinal); + foreach (AssemblyStoreItem item in Assemblies) { + dict.Add (item.Name, item); + } + AssembliesByName = dict.AsReadOnly (); + } + + protected AssemblyStoreExplorer (FileInfo storeInfo) + : this (storeInfo.OpenRead (), storeInfo.FullName) + {} + + public static (IList? explorers, string? errorMessage) Open (string inputFile) + { + (FileFormat format, FileInfo? info) = Utils.DetectFileFormat (inputFile); + if (info == null) { + return (null, $"File '{inputFile}' does not exist."); + } + + switch (format) { + case FileFormat.Unknown: + return (null, $"File '{inputFile}' has an unknown format."); + + case FileFormat.Zip: + return (null, $"File '{inputFile}' is a ZIP archive, but not an Android one."); + + case FileFormat.AssemblyStore: + return (new List { new AssemblyStoreExplorer (info)}, null); + + case FileFormat.Aab: + return OpenAab (info); + + case FileFormat.AabBase: + return OpenAabBase (info); + + case FileFormat.Apk: + return OpenApk (info); + + default: + return (null, $"File '{inputFile}' has an unsupported format '{format}'"); + } + } + + static (IList? explorers, string? errorMessage) OpenAab (FileInfo fi) + { + return OpenCommon ( + fi, + new List> { + StoreReader_V2.AabPaths, + StoreReader_V1.AabPaths, + } + ); + } + + static (IList? explorers, string? errorMessage) OpenAabBase (FileInfo fi) + { + return OpenCommon ( + fi, + new List> { + StoreReader_V2.AabBasePaths, + StoreReader_V1.AabBasePaths, + } + ); + } + + static (IList? explorers, string? errorMessage) OpenApk (FileInfo fi) + { + return OpenCommon ( + fi, + new List> { + StoreReader_V2.ApkPaths, + StoreReader_V1.ApkPaths, + } + ); + } + + static (IList? explorers, string? errorMessage) OpenCommon (FileInfo fi, List> pathLists) + { + using var zip = ZipArchive.Open (fi.FullName, FileMode.Open); + IList? explorers; + string? errorMessage; + bool pathsFound; + + foreach (IList paths in pathLists) { + (explorers, errorMessage, pathsFound) = TryLoad (fi, zip, paths); + if (pathsFound) { + return (explorers, errorMessage); + } + } + + return (null, "Unable to find any blob entries"); + } + + static (IList? explorers, string? errorMessage, bool pathsFound) TryLoad (FileInfo fi, ZipArchive zip, IList paths) + { + var ret = new List (); + + foreach (string path in paths) { + if (!zip.ContainsEntry (path)) { + continue; + } + + ZipEntry entry = zip.ReadEntry (path); + var stream = new MemoryStream (); + entry.Extract (stream); + ret.Add (new AssemblyStoreExplorer (stream, $"{fi.FullName}!{path}")); + } + + if (ret.Count == 0) { + return (null, null, false); + } + + return (ret, null, true); + } + + public Stream Read (AssemblyStoreItem item, bool uncompressIfNeeded = false) + { + throw new NotImplementedException (); + } + + public IList? Find (string assemblyName, AndroidTargetArch? targetArch = null) + { + if (Assemblies == null) { + return null; + } + + var items = new List (); + foreach (AssemblyStoreItem item in Assemblies) { + if (String.CompareOrdinal (assemblyName, item.Name) != 0) { + continue; + } + + if (targetArch != null && item.TargetArch != targetArch) { + continue; + } + + items.Add (item); + } + + if (items.Count == 0) { + return null; + } + + return items; + } + + public bool Contains (string assemblyName, AndroidTargetArch? targetArch = null) + { + IList? items = Find (assemblyName, targetArch); + if (items == null || items.Count == 0) { + return false; + } + + return true; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs new file mode 100644 index 00000000000..d2ee02cf0a4 --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreItem.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +abstract class AssemblyStoreItem +{ + public string Name { get; } + public IList Hashes { get; } + public bool Is64Bit { get; } + public uint DataOffset { get; protected set; } + public uint DataSize { get; protected set; } + public uint DebugOffset { get; protected set; } + public uint DebugSize { get; protected set; } + public uint ConfigOffset { get; protected set; } + public uint ConfigSize { get; protected set; } + public AndroidTargetArch TargetArch { get; protected set; } + + protected AssemblyStoreItem (string name, bool is64Bit, List hashes) + { + Name = name; + Hashes = hashes.AsReadOnly (); + Is64Bit = is64Bit; + } +} diff --git a/tools/assembly-store-reader-mk2/AssemblyStoreReader.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs similarity index 94% rename from tools/assembly-store-reader-mk2/AssemblyStoreReader.cs rename to tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs index 39e4dcdc2fe..93764e34d4e 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStoreReader.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs @@ -14,7 +14,7 @@ abstract class AssemblyStoreReader public abstract string Description { get; } public string StorePath { get; } - public AndroidTargetArch? TargetArch { get; protected set; } + public AndroidTargetArch TargetArch { get; protected set; } = AndroidTargetArch.Arm; public uint AssemblyCount { get; protected set; } public uint IndexEntryCount { get; protected set; } public IList? Assemblies { get; protected set; } diff --git a/tools/assembly-store-reader-mk2/FileFormat.cs b/tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs similarity index 100% rename from tools/assembly-store-reader-mk2/FileFormat.cs rename to tools/assembly-store-reader-mk2/AssemblyStore/FileFormat.cs diff --git a/tools/assembly-store-reader-mk2/Log.cs b/tools/assembly-store-reader-mk2/AssemblyStore/Log.cs similarity index 100% rename from tools/assembly-store-reader-mk2/Log.cs rename to tools/assembly-store-reader-mk2/AssemblyStore/Log.cs diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs new file mode 100644 index 00000000000..fa8922e3d5d --- /dev/null +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using System.IO; + +namespace Xamarin.Android.AssemblyStore; + +class StoreReader_V1 : AssemblyStoreReader +{ + public override string Description => "Assembly store v1"; + + public static IList ApkPaths { get; } + public static IList AabPaths { get; } + public static IList AabBasePaths { get; } + + static StoreReader_V1 () + { + ApkPaths = new List ().AsReadOnly (); + AabPaths = new List ().AsReadOnly (); + AabBasePaths = new List ().AsReadOnly (); + } + + public StoreReader_V1 (Stream store, string path) + : base (store, path) + {} + + protected override bool IsSupported () + { + return false; + } + + protected override void Prepare () + { + } +} diff --git a/tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs similarity index 91% rename from tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs rename to tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs index 0c56e9318c2..9dd8a19a054 100644 --- a/tools/assembly-store-reader-mk2/StoreReader_V2.Classes.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.Classes.cs @@ -1,5 +1,7 @@ using System.Collections.Generic; +using Xamarin.Android.Tools; + namespace Xamarin.Android.AssemblyStore; partial class StoreReader_V2 @@ -28,7 +30,6 @@ public Header (uint magic, uint version, uint entry_count, uint index_entry_coun sealed class IndexEntry { - public string? name; public readonly ulong name_hash; public readonly uint descriptor_index; @@ -55,7 +56,7 @@ sealed class EntryDescriptor sealed class StoreItem_V2 : AssemblyStoreItem { - public StoreItem_V2 (string name, bool is64Bit, List indexEntries, EntryDescriptor descriptor) + public StoreItem_V2 (AndroidTargetArch targetArch, string name, bool is64Bit, List indexEntries, EntryDescriptor descriptor) : base (name, is64Bit, IndexToHashes (indexEntries)) { DataOffset = descriptor.data_offset; @@ -64,6 +65,7 @@ public StoreItem_V2 (string name, bool is64Bit, List indexEntries, E DebugSize = descriptor.debug_data_size; ConfigOffset = descriptor.config_data_offset; ConfigSize = descriptor.config_data_size; + TargetArch = targetArch; } static List IndexToHashes (List indexEntries) diff --git a/tools/assembly-store-reader-mk2/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs similarity index 78% rename from tools/assembly-store-reader-mk2/StoreReader_V2.cs rename to tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index 6a83513565f..d4fb86833fd 100644 --- a/tools/assembly-store-reader-mk2/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -4,6 +4,7 @@ using System.Text; using Xamarin.Android.Tools; +using Xamarin.Android.Tasks; namespace Xamarin.Android.AssemblyStore; @@ -22,10 +23,52 @@ partial class StoreReader_V2 : AssemblyStoreReader public override string Description => "Assembly store v2"; + public static IList ApkPaths { get; } + public static IList AabPaths { get; } + public static IList AabBasePaths { get; } + readonly HashSet supportedVersions; Header? header; + static StoreReader_V2 () + { + var paths = new List { + GetArchPath (AndroidTargetArch.Arm64), + GetArchPath (AndroidTargetArch.Arm), + GetArchPath (AndroidTargetArch.X86_64), + GetArchPath (AndroidTargetArch.X86), + }; + ApkPaths = paths.AsReadOnly (); + AabBasePaths = ApkPaths; + + const string AabBaseDir = "base"; + paths = new List { + GetArchPath (AndroidTargetArch.Arm64, AabBaseDir), + GetArchPath (AndroidTargetArch.Arm, AabBaseDir), + GetArchPath (AndroidTargetArch.X86_64, AabBaseDir), + GetArchPath (AndroidTargetArch.X86, AabBaseDir), + }; + AabPaths = paths.AsReadOnly (); + + string GetArchPath (AndroidTargetArch arch, string? root = null) + { + const string LibDirName = "lib"; + + string abi = MonoAndroidHelper.ArchToAbi (arch); + var parts = new List (); + if (!String.IsNullOrEmpty (root)) { + parts.Add (LibDirName); + } else { + root = LibDirName; + } + parts.Add (abi); + parts.Add (GetBlobName (abi)); + + return MonoAndroidHelper.MakeZipArchivePath (root, parts); + } + } + public StoreReader_V2 (Stream store, string path) : base (store, path) { @@ -37,6 +80,8 @@ public StoreReader_V2 (Stream store, string path) }; } + static string GetBlobName (string abi) => $"assemblies.{abi}.blob.so"; + protected override bool IsSupported () { StoreStream.Seek (0, SeekOrigin.Begin); @@ -141,7 +186,7 @@ protected override void Prepare () var storeItems = new List (); foreach (var kvp in tempItems) { TemporaryItem ti = kvp.Value; - var item = new StoreItem_V2 (ti.Name, Is64Bit, ti.IndexEntries, ti.Descriptor); + var item = new StoreItem_V2 (TargetArch, ti.Name, Is64Bit, ti.IndexEntries, ti.Descriptor); storeItems.Add (item); } Assemblies = storeItems.AsReadOnly (); diff --git a/tools/assembly-store-reader-mk2/Utils.cs b/tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs similarity index 100% rename from tools/assembly-store-reader-mk2/Utils.cs rename to tools/assembly-store-reader-mk2/AssemblyStore/Utils.cs diff --git a/tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs deleted file mode 100644 index da64aed81df..00000000000 --- a/tools/assembly-store-reader-mk2/AssemblyStoreExplorer.cs +++ /dev/null @@ -1,32 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -using Xamarin.Android.Tools; - -namespace Xamarin.Android.AssemblyStore; - -class AssemblyStoreExplorer -{ - readonly AssemblyStoreReader reader; - - public string StorePath { get; } - public AndroidTargetArch? TargetArch => reader.TargetArch; - public uint AssemblyCount => reader.AssemblyCount; - public uint IndexEntryCount => reader.IndexEntryCount; - public IList? Assemblies => reader.Assemblies; - public bool Is64Bit => reader.Is64Bit; - - public AssemblyStoreExplorer (FileInfo storeInfo) - { - Stream storeStream = storeInfo.OpenRead (); - var storeReader = AssemblyStoreReader.Create (storeStream, storeInfo.FullName); - if (storeReader == null) { - storeStream.Dispose (); - throw new NotSupportedException ($"Format of assembly store '{storeInfo.FullName}' is unsupported"); - } - - reader = storeReader; - StorePath = storeInfo.FullName; - } -} diff --git a/tools/assembly-store-reader-mk2/AssemblyStoreItem.cs b/tools/assembly-store-reader-mk2/AssemblyStoreItem.cs deleted file mode 100644 index 3336825eb9e..00000000000 --- a/tools/assembly-store-reader-mk2/AssemblyStoreItem.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Collections.Generic; - -namespace Xamarin.Android.AssemblyStore; - -abstract class AssemblyStoreItem -{ - public string Name { get; } - public IList Hashes { get; } - public bool Is64Bit { get; } - public uint DataOffset { get; protected set; } - public uint DataSize { get; protected set; } - public uint DebugOffset { get; protected set; } - public uint DebugSize { get; protected set; } - public uint ConfigOffset { get; protected set; } - public uint ConfigSize { get; protected set; } - - protected AssemblyStoreItem (string name, bool is64Bit, List hashes) - { - Name = name; - Hashes = hashes.AsReadOnly (); - Is64Bit = is64Bit; - } -} diff --git a/tools/assembly-store-reader-mk2/Program.cs b/tools/assembly-store-reader-mk2/Program.cs index d3582beffad..6d8d07166c6 100644 --- a/tools/assembly-store-reader-mk2/Program.cs +++ b/tools/assembly-store-reader-mk2/Program.cs @@ -49,20 +49,12 @@ static int Main (string[] args) return WriteErrorAndReturn ($"File '{inputFile}' does not exist."); } - var stores = new List (); - switch (format) { - case FileFormat.Unknown: - return WriteErrorAndReturn ($"File '{inputFile}' has an unknown format."); - - case FileFormat.Zip: - return WriteErrorAndReturn ($"File '{inputFile}' is a ZIP archive, but not an Android one."); - - case FileFormat.AssemblyStore: - stores.Add (new AssemblyStoreExplorer (info)); - break; + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (inputFile); + if (explorers == null) { + return WriteErrorAndReturn (errorMessage ?? "Unknown error"); } - foreach (AssemblyStoreExplorer store in stores) { + foreach (AssemblyStoreExplorer store in explorers) { var printer = new StorePrettyPrinter (store); printer.Show (); } diff --git a/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs b/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs index f216238794e..8efd16fd162 100644 --- a/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs +++ b/tools/assembly-store-reader-mk2/StorePrettyPrinter.cs @@ -27,47 +27,67 @@ public void Show () Console.WriteLine ("NO ASSEMBLIES!"); return; } + + var assemblies = new List (explorer.Assemblies); + assemblies.Sort ((AssemblyStoreItem a, AssemblyStoreItem b) => a.Name.CompareTo (b.Name)); + Console.WriteLine ("Assemblies:"); var line = new StringBuilder (); - foreach (AssemblyStoreItem assembly in explorer.Assemblies) { + foreach (AssemblyStoreItem assembly in assemblies) { line.Clear (); line.Append (" "); - line.Append (assembly.Name); - line.Append (' '); - line.Append (FormatOffsetAndSize (assembly.DataOffset, assembly.DataSize)); - line.Append (' '); - line.Append (FormatOffsetAndSize (assembly.DebugOffset, assembly.DebugSize)); - line.Append (' '); - line.Append (FormatOffsetAndSize (assembly.ConfigOffset, assembly.ConfigSize)); - line.Append (" ["); - line.Append (FormatHashes (assembly.Hashes)); - line.Append (']'); + line.AppendLine (assembly.Name); + line.Append (" PE image data: "); + FormatOffsetAndSize (line, assembly.DataOffset, assembly.DataSize); + line.AppendLine (); + line.Append (" Debug data: "); + FormatOffsetAndSize (line, assembly.DebugOffset, assembly.DebugSize); + line.AppendLine (); + line.Append (" Config data: "); + FormatOffsetAndSize (line, assembly.ConfigOffset, assembly.ConfigSize); + line.AppendLine (); + line.Append (" Name hashes: "); + FormatHashes (line, assembly.Hashes); + line.AppendLine (); Console.WriteLine (line.ToString ()); } Console.WriteLine (); } - static string FormatOffsetAndSize (uint offset, uint size) + static void FormatOffsetAndSize (StringBuilder sb, uint offset, uint size) { if (offset == 0) { - return "none"; + FormatNone (sb); + return; } - return $"{offset} ({size})"; + sb.Append ("offset "); + sb.Append (offset); + sb.Append (", size "); + sb.Append (size); } - static string FormatHashes (IList hashes) + static void FormatHashes (StringBuilder sb, IList hashes) { if (hashes.Count == 0) { - return "none"; + FormatNone (sb); + return; } - var ret = new List (); + bool first = true; foreach (ulong hash in hashes) { - ret.Add ($"0x{hash:x}"); + if (first) { + first = false; + } else { + sb.Append (", "); + } + sb.Append ($"0x{hash:x}"); } + } - return String.Join (',', ret); + static void FormatNone (StringBuilder sb) + { + sb.Append ("none"); } static string GetBitness (bool is64bit) => is64bit ? "64" : "32"; diff --git a/tools/assembly-store-reader-mk2/StoreReader_V1.cs b/tools/assembly-store-reader-mk2/StoreReader_V1.cs deleted file mode 100644 index 87850329750..00000000000 --- a/tools/assembly-store-reader-mk2/StoreReader_V1.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System; -using System.IO; - -namespace Xamarin.Android.AssemblyStore; - -class StoreReader_V1 : AssemblyStoreReader -{ - public override string Description => "Assembly store v1"; - - public StoreReader_V1 (Stream store, string path) - : base (store, path) - {} - - protected override bool IsSupported () - { - return false; - } - - protected override void Prepare () - { - } -} diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj index b6fb7474b07..92315248cc2 100644 --- a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -18,6 +18,11 @@ + + + + + From 29747b6ad5365292d4e0a90ea466176b293cb342 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 20 Nov 2023 17:17:14 +0100 Subject: [PATCH 020/143] Fix a handful of tests --- .../IncrementalBuildTest.cs | 3 +- .../Utilities/ArchiveAssemblyHelper.cs | 139 ++++++++++++++++-- .../Utilities/MonoAndroidHelper.Simple.cs | 109 +++++++++++--- .../AssemblyStore/AssemblyStoreExplorer.cs | 21 ++- .../AssemblyStore/AssemblyStoreReader.cs | 26 ++++ .../AssemblyStore/StoreReader_V1.cs | 1 + .../AssemblyStore/StoreReader_V2.cs | 1 + 7 files changed, 260 insertions(+), 40 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index 1c9de6d7530..36bced767f2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -655,7 +655,8 @@ public void LinkAssembliesNoShrink () Assert.IsTrue (b.Build (proj), "build should have succeeded."); // Touch an assembly to a timestamp older than build.props - foreach (string abi in proj.GetProperty (KnownProperties.AndroidSupportedAbis).Split (';')) { + foreach (string rid in proj.GetProperty (KnownProperties.RuntimeIdentifiers).Split (';')) { + string abi = MonoAndroidHelper.RidToAbi (rid); var formsViewGroup = b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, "FormsViewGroup.dll")); File.SetLastWriteTimeUtc (formsViewGroup, new DateTime (1970, 1, 1)); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 9f4657b9699..f6dfbe73551 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -66,13 +66,25 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, Stream? ReadZipEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) { - using (var zip = ZipHelper.OpenZip (archivePath)) { - ZipEntry entry = zip.ReadEntry (path); + List? potentialEntries = TransformArchiveAssemblyPath (path, arch); + if (potentialEntries == null || potentialEntries.Count == 0) { + return null; + } + + using var zip = ZipHelper.OpenZip (archivePath); + foreach (string assemblyPath in potentialEntries) { + if (!zip.ContainsEntry (assemblyPath)) { + continue; + } + + ZipEntry entry = zip.ReadEntry (assemblyPath); var ret = new MemoryStream (); entry.Extract (ret); ret.Flush (); return ret; } + + return null; } Stream? ReadStoreEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) @@ -85,6 +97,15 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, return null; } + if (arch == AndroidTargetArch.None) { + if (explorer.TargetArch == null) { + throw new InvalidOperationException ($"Internal error: explorer should not have its TargetArch unset"); + } + + arch = (AndroidTargetArch)explorer.TargetArch; + } + + Console.WriteLine ($"Trying to read store entry: {name}"); IList? assemblies = explorer.Find (name, arch); if (assemblies == null) { Console.WriteLine ($"Failed to locate assembly '{name}' in assembly store for architecture '{arch}', in archive '{archivePath}'"); @@ -104,7 +125,7 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, return null; } - return explorer.Read (assembly, uncompressIfNecessary); + return explorer.ReadImageData (assembly, uncompressIfNecessary); } public List ListArchiveContents (string storeEntryPrefix = DefaultAssemblyStoreEntryPrefix, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) @@ -138,18 +159,18 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb foreach (AssemblyStoreItem asm in explorer.Assemblies) { string prefix = storeEntryPrefix; - if (haveMultipleRids && asm.TargetArch != AndroidTargetArch.None) { +// if (haveMultipleRids && asm.TargetArch != AndroidTargetArch.None) { string abi = MonoAndroidHelper.ArchToAbi (asm.TargetArch); prefix = $"{prefix}{abi}/"; - } +// } - entries.Add ($"{prefix}{asm.Name}.dll"); + entries.Add ($"{prefix}{asm.Name}"); if (asm.DebugOffset > 0) { - entries.Add ($"{prefix}{asm.Name}.pdb"); + entries.Add ($"{prefix}{Path.GetFileNameWithoutExtension (asm.Name)}.pdb"); } if (asm.ConfigOffset > 0) { - entries.Add ($"{prefix}{asm.Name}.dll.config"); + entries.Add ($"{prefix}{asm.Name}.config"); } } @@ -185,7 +206,7 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb return null; } - public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool forceRefresh = false) + public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) { List contents = ListArchiveContents (assembliesRootDir, forceRefresh); var dlls = contents.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)); @@ -206,14 +227,106 @@ public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool force }).Count (); } - public bool Exists (string entryPath, bool forceRefresh = false) + /// + /// Takes "old style" `assemblies/assembly.dll` path and returns (if possible) a set of paths that reflect the new + /// location of `assemblies/{ARCH}/assembly.dll`. A list is returned because, if `arch` is `None`, we'll return all + /// the possible architectural paths. + /// An exception is thrown if we cannot transform the path for some reason. It should **not** be handled. + /// + static List? TransformArchiveAssemblyPath (string path, AndroidTargetArch arch) { - List contents = ListArchiveContents (assembliesRootDir, forceRefresh); - if (contents.Count == 0) { + const string AssembliesPath = "assemblies"; + const string AssembliesPathTerminated = AssembliesPath + "/"; + + if (String.IsNullOrEmpty (path)) { + throw new ArgumentException (nameof (path), "must not be null or empty"); + } + + if (!path.StartsWith (AssembliesPathTerminated, StringComparison.Ordinal)) { + throw new InvalidOperationException ($"Path '{path}' does not start with '{AssembliesPathTerminated}'"); + } + + string[] parts = path.Split ('/'); + if (parts.Length < 2) { + throw new InvalidOperationException ($"Path '{path}' must consist of at least two segments separated by `/`"); + } + + // We accept: + // assemblies/assembly.dll + // assemblies/{CULTURE}/assembly.dll + // assemblies/{ARCH}/assembly.dll + // assemblies/{ARCH}/{CULTURE}/assembly.dll + if (parts.Length > 4) { + throw new InvalidOperationException ($"Path '{path}' must not consist of more than 4 segments separated by `/`"); + } + + var ret = new List (); + if (parts.Length == 4) { + // It's a full satellite assembly path that includes the ABI, no need to change anything + ret.Add (path); + return ret; + } + + if (parts.Length == 3) { + // We need to check whether the middle part is a culture or an ABI + if (MonoAndroidHelper.IsValidAbi (parts[1])) { + // Nothing more to do + ret.Add (path); + return ret; + } + } + + // We need to add the ABI(s) + var newParts = new List { + String.Empty, // ABI placeholder + }; + + for (int i = 1; i < parts.Length; i++) { + newParts.Add (parts[i]); + } + + if (arch != AndroidTargetArch.None) { + ret.Add (MakeAbiArchivePath (arch)); + } else { + foreach (AndroidTargetArch targetArch in MonoAndroidHelper.SupportedTargetArchitectures) { + ret.Add (MakeAbiArchivePath (targetArch)); + } + } + + return ret; + + string MakeAbiArchivePath (AndroidTargetArch targetArch) + { + newParts[0] = MonoAndroidHelper.ArchToAbi (targetArch); + return MonoAndroidHelper.MakeZipArchivePath (AssembliesPath, newParts); + } + } + + static bool ArchiveContains (List archiveContents, string entryPath, AndroidTargetArch arch) + { + if (archiveContents.Count == 0) { return false; } - return contents.Contains (entryPath); + List? potentialEntries = TransformArchiveAssemblyPath (entryPath, arch); + if (potentialEntries == null || potentialEntries.Count == 0) { + return false; + } + + foreach (string existingEntry in archiveContents) { + foreach (string wantedEntry in potentialEntries) { + if (String.Compare (existingEntry, wantedEntry, StringComparison.Ordinal) == 0) { + return true; + } + } + } + + return false; + } + + public bool Exists (string entryPath, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) + { + return ArchiveContains (ListArchiveContents (assembliesRootDir, forceRefresh), entryPath, arch); } public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch = AndroidTargetArch.None) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs index e1634df2fed..5e2c0c32a10 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs @@ -10,6 +10,29 @@ namespace Xamarin.Android.Tasks; partial class MonoAndroidHelper { + public static class AndroidAbi + { + public const string Arm32 = "armeabi-v7a"; + public const string Arm64 = "arm64-v8a"; + public const string X86 = "x86"; + public const string X64 = "x86_64"; + } + + public static class RuntimeIdentifier + { + public const string Arm32 = "android-arm"; + public const string Arm64 = "android-arm64"; + public const string X86 = "android-x86"; + public const string X64 = "android-x64"; + } + + public static readonly HashSet SupportedTargetArchitectures = new HashSet { + AndroidTargetArch.Arm, + AndroidTargetArch.Arm64, + AndroidTargetArch.X86, + AndroidTargetArch.X86_64, + }; + static readonly char[] ZipPathTrimmedChars = {'/', '\\'}; static readonly Dictionary ClangAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { @@ -19,50 +42,88 @@ partial class MonoAndroidHelper {"x86_64", "x86_64"} }; + static readonly Dictionary AbiToArchMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { AndroidAbi.Arm32, AndroidTargetArch.Arm }, + { AndroidAbi.Arm64, AndroidTargetArch.Arm64 }, + { AndroidAbi.X86, AndroidTargetArch.X86 }, + { AndroidAbi.X64, AndroidTargetArch.X86_64 }, + }; + + static readonly Dictionary AbiToRidMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { AndroidAbi.Arm32, RuntimeIdentifier.Arm32 }, + { AndroidAbi.Arm64, RuntimeIdentifier.Arm64 }, + { AndroidAbi.X86, RuntimeIdentifier.X86 }, + { AndroidAbi.X64, RuntimeIdentifier.X64 }, + }; + + static readonly Dictionary RidToAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { RuntimeIdentifier.Arm32, AndroidAbi.Arm32 }, + { RuntimeIdentifier.Arm64, AndroidAbi.Arm64 }, + { RuntimeIdentifier.X86, AndroidAbi.X86 }, + { RuntimeIdentifier.X64, AndroidAbi.X64 }, + }; + + static readonly Dictionary ArchToRidMap = new Dictionary { + { AndroidTargetArch.Arm, RuntimeIdentifier.Arm32 }, + { AndroidTargetArch.Arm64, RuntimeIdentifier.Arm64 }, + { AndroidTargetArch.X86, RuntimeIdentifier.X86 }, + { AndroidTargetArch.X86_64, RuntimeIdentifier.X64 }, + }; + + static readonly Dictionary ArchToAbiMap = new Dictionary { + { AndroidTargetArch.Arm, AndroidAbi.Arm32 }, + { AndroidTargetArch.Arm64, AndroidAbi.Arm64 }, + { AndroidTargetArch.X86, AndroidAbi.X86 }, + { AndroidTargetArch.X86_64, AndroidAbi.X64 }, + }; + public static AndroidTargetArch AbiToTargetArch (string abi) { - return abi switch { - "armeabi-v7a" => AndroidTargetArch.Arm, - "arm64-v8a" => AndroidTargetArch.Arm64, - "x86_64" => AndroidTargetArch.X86_64, - "x86" => AndroidTargetArch.X86, - _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") + if (!AbiToArchMap.TryGetValue (abi, out AndroidTargetArch arch)) { + throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'"); }; + + return arch; } public static string AbiToRid (string abi) { - return abi switch { - "armeabi-v7a" => "android-arm", - "arm64-v8a" => "android-arm64", - "x86_64" => "android-x64", - "x86" => "android-x86", - _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") + if (!AbiToRidMap.TryGetValue (abi, out string rid)) { + throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'"); + }; + + return rid; + } + + public static string RidToAbi (string rid) + { + if (!RidToAbiMap.TryGetValue (rid, out string abi)) { + throw new NotSupportedException ($"Internal error: unsupported Runtime Identifier '{rid}'"); }; + + return abi; } public static string ArchToRid (AndroidTargetArch arch) { - return arch switch { - AndroidTargetArch.Arm64 => "android-arm64", - AndroidTargetArch.Arm => "android-arm", - AndroidTargetArch.X86 => "android-x86", - AndroidTargetArch.X86_64 => "android-x64", - _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") + if (!ArchToRidMap.TryGetValue (arch, out string rid)) { + throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'"); }; + + return rid; } public static string ArchToAbi (AndroidTargetArch arch) { - return arch switch { - AndroidTargetArch.Arm64 => "arm64-v8a", - AndroidTargetArch.Arm => "armeabi-v7a", - AndroidTargetArch.X86 => "x86", - AndroidTargetArch.X86_64 => "x86_64", - _ => throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'") + if (!ArchToAbiMap.TryGetValue (arch, out string abi)) { + throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'"); }; + + return abi; } + public static bool IsValidAbi (string abi) => AbiToRidMap.ContainsKey (abi); + public static string? CultureInvariantToString (object? obj) { if (obj == null) { diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs index c36dc9df7a1..c99a23d0636 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreExplorer.cs @@ -149,9 +149,25 @@ public static (IList? explorers, string? errorMessage) Op return (ret, null, true); } - public Stream Read (AssemblyStoreItem item, bool uncompressIfNeeded = false) + public Stream? ReadImageData (AssemblyStoreItem item, bool uncompressIfNeeded = false) { - throw new NotImplementedException (); + return reader.ReadEntryImageData (item, uncompressIfNeeded); + } + + string EnsureCorrectAssemblyName (string assemblyName) + { + assemblyName = Path.GetFileName (assemblyName); + if (reader.NeedsExtensionInName) { + if (!assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + return $"{assemblyName}.dll"; + } + } else { + if (assemblyName.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + return Path.GetFileNameWithoutExtension (assemblyName); + } + } + + return assemblyName; } public IList? Find (string assemblyName, AndroidTargetArch? targetArch = null) @@ -160,6 +176,7 @@ public Stream Read (AssemblyStoreItem item, bool uncompressIfNeeded = false) return null; } + assemblyName = EnsureCorrectAssemblyName (assemblyName); var items = new List (); foreach (AssemblyStoreItem item in Assemblies) { if (String.CompareOrdinal (assemblyName, item.Name) != 0) { diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs index 93764e34d4e..cc39baa6ecc 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/AssemblyStoreReader.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; using System.Text; @@ -11,7 +12,9 @@ abstract class AssemblyStoreReader static readonly UTF8Encoding ReaderEncoding = new UTF8Encoding (false); protected Stream StoreStream { get; } + public abstract string Description { get; } + public abstract bool NeedsExtensionInName { get; } public string StorePath { get; } public AndroidTargetArch TargetArch { get; protected set; } = AndroidTargetArch.Arm; @@ -55,4 +58,27 @@ protected AssemblyStoreReader (Stream store, string path) protected abstract bool IsSupported (); protected abstract void Prepare (); + + public Stream ReadEntryImageData (AssemblyStoreItem entry, bool uncompressIfNeeded = false) + { + StoreStream.Seek (entry.DataOffset, SeekOrigin.Begin); + var stream = new MemoryStream (); + + if (uncompressIfNeeded) { + throw new NotImplementedException (); + } + + const long BufferSize = 65535; + byte[] buffer = Utils.BytePool.Rent ((int)BufferSize); + long remainingToRead = entry.DataSize; + + while (remainingToRead > 0) { + int nread = StoreStream.Read (buffer, 0, (int)Math.Min (BufferSize, remainingToRead)); + stream.Write (buffer, 0, nread); + remainingToRead -= (long)nread; + } + stream.Flush (); + + return stream; + } } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs index fa8922e3d5d..d907721c088 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V1.cs @@ -6,6 +6,7 @@ namespace Xamarin.Android.AssemblyStore; class StoreReader_V1 : AssemblyStoreReader { public override string Description => "Assembly store v1"; + public override bool NeedsExtensionInName => false; public static IList ApkPaths { get; } public static IList AabPaths { get; } diff --git a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs index d4fb86833fd..45e74e22752 100644 --- a/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs +++ b/tools/assembly-store-reader-mk2/AssemblyStore/StoreReader_V2.cs @@ -22,6 +22,7 @@ partial class StoreReader_V2 : AssemblyStoreReader const uint ASSEMBLY_STORE_ABI_MASK = 0x00FF0000; public override string Description => "Assembly store v2"; + public override bool NeedsExtensionInName => true; public static IList ApkPaths { get; } public static IList AabPaths { get; } From 446fbecc313f983520cea035e6e32e98be2addea Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 20 Nov 2023 17:51:07 +0100 Subject: [PATCH 021/143] More test fixes + potential test perf improvement --- .../IncrementalBuildTest.cs | 2 +- .../Tasks/LinkerTests.cs | 4 +- .../Xamarin.ProjectTools/Common/Builder.cs | 39 +++++++++++++++++-- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index 36bced767f2..8f1156729b0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -655,7 +655,7 @@ public void LinkAssembliesNoShrink () Assert.IsTrue (b.Build (proj), "build should have succeeded."); // Touch an assembly to a timestamp older than build.props - foreach (string rid in proj.GetProperty (KnownProperties.RuntimeIdentifiers).Split (';')) { + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { string abi = MonoAndroidHelper.RidToAbi (rid); var formsViewGroup = b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, "FormsViewGroup.dll")); File.SetLastWriteTimeUtc (formsViewGroup, new DateTime (1970, 1, 1)); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index bc0a493e58c..86efceed4ab 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -565,8 +565,8 @@ void Assert64Bit(string rid, bool expected64) /* * IL snippet - * .method private hidebysig specialname rtspecialname static - * void .cctor () cil managed + * .method private hidebysig specialname rtspecialname static + * void .cctor () cil managed * { * // Is64Bits = 4 >= 8; * IL_0000: ldc.i4 4 diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index c583e2b6192..f79bd5a4a81 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -20,6 +20,8 @@ public class Builder : IDisposable string root; string buildLogFullPath; + IEnumerable? lastBuildOutput; + public bool IsUnix { get; set; } /// /// This passes /p:BuildingInsideVisualStudio=True, command-line to MSBuild @@ -32,10 +34,17 @@ public class Builder : IDisposable public LoggerVerbosity Verbosity { get; set; } = LoggerVerbosity.Diagnostic; public IEnumerable LastBuildOutput { get { + if (lastBuildOutput != null) { + return lastBuildOutput; + } + if (!string.IsNullOrEmpty (buildLogFullPath) && File.Exists (buildLogFullPath)) { - return File.ReadLines (buildLogFullPath, Encoding.UTF8); + lastBuildOutput = File.ReadLines (buildLogFullPath, Encoding.UTF8); + } else { + lastBuildOutput = Enumerable.Empty (); } - return Enumerable.Empty (); + + return lastBuildOutput; } } public TimeSpan LastBuildTime { get; protected set; } @@ -120,6 +129,30 @@ public void GetTargetFrameworkVersionRange (out string firstApiLevel, out string allFrameworkVersions = allTFVs.ToArray (); } + public HashSet GetBuildRuntimeIdentifiers () + { + var ret = new HashSet (StringComparer.OrdinalIgnoreCase); + + foreach (string l in LastBuildOutput) { + string line = l.Trim (); + if (line.Length == 0 || line[0] != 'R') { + continue; + } + + // Here's hoping MSBuild doesn't change the property reporting format + if (!line.StartsWith ("RuntimeIdentifiers =", StringComparison.Ordinal)) { + continue; + } + + foreach (string r in line.Split ('=')[1].Split (';')) { + ret.Add (r.Trim ()); + } + break; + } + + return ret; + } + static string GetApiInfoElementValue (string androidApiInfo, string elementPath) { if (!File.Exists (androidApiInfo)) @@ -159,6 +192,7 @@ protected virtual void Dispose (bool disposing) protected bool BuildInternal (string projectOrSolution, string target, string [] parameters = null, Dictionary environmentVariables = null, bool restore = true, string binlogName = "msbuild") { + lastBuildOutput = null; // make sure we don't return the previous build's cached output buildLogFullPath = (!string.IsNullOrEmpty (BuildLogFile)) ? Path.GetFullPath (Path.Combine (XABuildPaths.TestOutputDirectory, Path.GetDirectoryName (projectOrSolution), BuildLogFile)) : null; @@ -372,4 +406,3 @@ string QuoteFileName (string fileName) } } - From 52e324f37e3e87d5766766c9d1ab207dde331323 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 20 Nov 2023 21:43:24 +0100 Subject: [PATCH 022/143] Make sure assemblies have all the required RID metadata --- .../Tasks/PrepareSatelliteAssemblies.cs | 75 +++++++++++ ...r.Simple.cs => MonoAndroidHelper.Basic.cs} | 0 .../Xamarin.Android.Common.targets | 16 +-- tools/assembly-store-reader-mk2/Main.cs | 118 ++++++++++++++++++ tools/assembly-store-reader-mk2/Program.cs | 66 ---------- .../assembly-store-reader.csproj | 2 +- 6 files changed, 203 insertions(+), 74 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs rename src/Xamarin.Android.Build.Tasks/Utilities/{MonoAndroidHelper.Simple.cs => MonoAndroidHelper.Basic.cs} (100%) create mode 100644 tools/assembly-store-reader-mk2/Main.cs delete mode 100644 tools/assembly-store-reader-mk2/Program.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs new file mode 100644 index 00000000000..8fe1e98a6c8 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.IO; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks; + +public class PrepareSatelliteAssemblies : AndroidTask +{ + public override string TaskPrefix => "PSA"; + + [Required] + public string[] BuildTargetAbis { get; set; } = Array.Empty (); + + [Required] + public ITaskItem[] ReferenceSatellitePaths { get; set; } = Array.Empty (); + + [Required] + public ITaskItem[] IntermediateSatelliteAssemblies { get; set; } = Array.Empty (); + + [Output] + public ITaskItem[] ProcessedSatelliteAssemblies { get; set; } + + public override bool RunTask () + { + var output = new List (); + + SetMetadata (ReferenceSatellitePaths, output); + SetMetadata (IntermediateSatelliteAssemblies, output); + + ProcessedSatelliteAssemblies = output.ToArray (); + return !Log.HasLoggedErrors; + } + + void SetMetadata (ITaskItem[] items, List output) + { + foreach (ITaskItem item in items) { + SetMetadata (item, output); + } + } + + void SetMetadata (ITaskItem item, List output) + { + string? culture = item.GetMetadata ("Culture"); + if (String.IsNullOrEmpty (culture)) { + throw new InvalidOperationException ($"Assembly item '{item}' is missing the 'Culture' metadata"); + } + + string? targetPath = item.GetMetadata ("TargetPath"); + bool haveTargetPath = !String.IsNullOrEmpty (targetPath); + string assemblyName = Path.GetFileName (item.ItemSpec); + char sep = Path.DirectorySeparatorChar; + + foreach (string abi in BuildTargetAbis) { + var newItem = new TaskItem (item); + newItem.SetMetadata ("Abi", abi); + + if (haveTargetPath) { + SetDestinationPathsMetadata (item, targetPath); + } else { + SetDestinationPathsMetadata (item, culture + sep + assemblyName); + } + + } + + void SetDestinationPathsMetadata (ITaskItem item, string path) + { + item.SetMetadata ("DestinationSubDirectory", path); + item.SetMetadata ("DestinationSubPath", path); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs similarity index 100% rename from src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Simple.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets index 4edb130ae9e..118295f137d 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Common.targets @@ -94,6 +94,7 @@ Copyright (C) 2011-2012 Xamarin. All rights reserved. + - <_AndroidResolvedSatellitePaths Include="@(IntermediateSatelliteAssembliesWithTargetPath->'$(OutDir)%(Culture)\$(TargetName).resources.dll')" /> - + DependsOnTargets="_ResolveAssemblies"> + + + + diff --git a/tools/assembly-store-reader-mk2/Main.cs b/tools/assembly-store-reader-mk2/Main.cs new file mode 100644 index 00000000000..24767dfae88 --- /dev/null +++ b/tools/assembly-store-reader-mk2/Main.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +using Mono.Options; +using Xamarin.Android.Tasks; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.AssemblyStore; + +class App +{ + static void ShowHelp () + { + } + + static int WriteErrorAndReturn (string message) + { + Console.Error.WriteLine (message); + return 1; + } + + static HashSet? ParseArchList (string values) + { + if (String.IsNullOrEmpty (values)) { + return null; + } + + var ret = new HashSet (); + foreach (string a in values.Split (',')) { + string archName = a.Trim (); + if (Enum.TryParse (archName, out AndroidTargetArch arch)) { + ret.Add (arch); + continue; + } + + arch = archName.ToLowerInvariant () switch { + "aarch64" => AndroidTargetArch.Arm64, + "arm32" => AndroidTargetArch.Arm, + "arm64" => AndroidTargetArch.Arm64, + "armv7a" => AndroidTargetArch.Arm, + "armv8a" => AndroidTargetArch.Arm64, + "x64" => AndroidTargetArch.X86_64, + _ => throw new InvalidOperationException ($"Unknown architecture name '{archName}'") + }; + ret.Add (arch); + } + + return ret; + } + + static string GetArchNames () + { + return String.Join (", ", MonoAndroidHelper.SupportedTargetArchitectures.Select (a => a.ToString ().ToLowerInvariant ())); + } + + static int Main (string[] args) + { + HashSet? arches = null; + bool showHelp = false; + + var options = new OptionSet { + "Usage: read-assembly-store [OPTIONS] BLOB_PATH", + "", + " where each BLOB_PATH can point to:", + " * aab file", + " * apk file", + " * index store file (e.g. base_assemblies.blob or assemblies.arm64_v8a.blob.so)", + " * arch store file (e.g. base_assemblies.arm64_v8a.blob)", + " * store manifest file (e.g. base_assemblies.manifest)", + " * store base name (e.g. base or base_assemblies)", + "", + " In each case the whole set of stores and manifests will be read (if available). Search for the", + " various members of the store set (common/main store, arch stores, manifest) is based on this naming", + " convention:", + "", + " {BASE_NAME}[.ARCH_NAME].{blob|blob.so|manifest}", + "", + " Whichever file is referenced in `BLOB_PATH`, the BASE_NAME component is extracted and all the found files are read.", + " If `BLOB_PATH` points to an aab or an apk, BASE_NAME will always be `assemblies`", + "", + {"a|arch=", $"Limit listing of assemblies to these {{ARCHITECTURES}} only. A comma-separated list of one or more of: {GetArchNames ()}", v => arches = ParseArchList (v) }, + "", + {"?|h|help", "Show this help screen", v => showHelp = true}, + }; + + List? theRest = options.Parse (args); + if (theRest == null || theRest.Count == 0 || showHelp) { + options.WriteOptionDescriptions (Console.Out); + return showHelp ? 0 : 1; + } + + string inputFile = theRest[0]; + (FileFormat format, FileInfo? info) = Utils.DetectFileFormat (inputFile); + if (info == null) { + return WriteErrorAndReturn ($"File '{inputFile}' does not exist."); + } + + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (inputFile); + if (explorers == null) { + return WriteErrorAndReturn (errorMessage ?? "Unknown error"); + } + + foreach (AssemblyStoreExplorer store in explorers) { + if (arches != null && store.TargetArch.HasValue && !arches.Contains (store.TargetArch.Value)) { + continue; + } + + var printer = new StorePrettyPrinter (store); + printer.Show (); + } + + return 0; + } + + +} diff --git a/tools/assembly-store-reader-mk2/Program.cs b/tools/assembly-store-reader-mk2/Program.cs deleted file mode 100644 index 6d8d07166c6..00000000000 --- a/tools/assembly-store-reader-mk2/Program.cs +++ /dev/null @@ -1,66 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; - -namespace Xamarin.Android.AssemblyStore; - -class Program -{ - static void ShowHelp () - { - Console.WriteLine ("Usage: read-assembly-store BLOB_PATH [BLOB_PATH ...]"); - Console.WriteLine (); - Console.WriteLine (@" where each BLOB_PATH can point to: - * aab file - * apk file - * index store file (e.g. base_assemblies.blob or assemblies.arm64_v8a.blob.so) - * arch store file (e.g. base_assemblies.arm64_v8a.blob) - * store manifest file (e.g. base_assemblies.manifest) - * store base name (e.g. base or base_assemblies) - - In each case the whole set of stores and manifests will be read (if available). Search for the - various members of the store set (common/main store, arch stores, manifest) is based on this naming - convention: - - {BASE_NAME}[.ARCH_NAME].{blob|manifest} - - Whichever file is referenced in `BLOB_PATH`, the BASE_NAME component is extracted and all the found files are read. - If `BLOB_PATH` points to an aab or an apk, BASE_NAME will always be `assemblies` - -"); - } - - static int WriteErrorAndReturn (string message) - { - Console.Error.WriteLine (message); - return 1; - } - - static int Main (string[] args) - { - if (args.Length == 0) { - ShowHelp (); - return 1; - } - - string inputFile = args[0]; - (FileFormat format, FileInfo? info) = Utils.DetectFileFormat (inputFile); - if (info == null) { - return WriteErrorAndReturn ($"File '{inputFile}' does not exist."); - } - - (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (inputFile); - if (explorers == null) { - return WriteErrorAndReturn (errorMessage ?? "Unknown error"); - } - - foreach (AssemblyStoreExplorer store in explorers) { - var printer = new StorePrettyPrinter (store); - printer.Show (); - } - - return 0; - } - - -} diff --git a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj index 92315248cc2..81520d99671 100644 --- a/tools/assembly-store-reader-mk2/assembly-store-reader.csproj +++ b/tools/assembly-store-reader-mk2/assembly-store-reader.csproj @@ -22,7 +22,7 @@ - + From 4363e0f090c3743d09fcf72385483427ad16e6aa Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 21 Nov 2023 16:55:11 +0100 Subject: [PATCH 023/143] Fixes for a handful more tests --- .../Tasks/PrepareSatelliteAssemblies.cs | 6 +- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 109 +++++++++++++----- .../Tasks/GeneratePackageManagerJavaTests.cs | 9 +- .../Utilities/ArchiveAssemblyHelper.cs | 8 +- .../Xamarin.ProjectTools/Common/Builder.cs | 13 ++- .../Xamarin.ProjectTools.csproj | 2 + 6 files changed, 109 insertions(+), 38 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs index 8fe1e98a6c8..8849c98a92a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs @@ -59,11 +59,11 @@ void SetMetadata (ITaskItem item, List output) newItem.SetMetadata ("Abi", abi); if (haveTargetPath) { - SetDestinationPathsMetadata (item, targetPath); + SetDestinationPathsMetadata (newItem, targetPath); } else { - SetDestinationPathsMetadata (item, culture + sep + assemblyName); + SetDestinationPathsMetadata (newItem, culture + sep + assemblyName); } - + output.Add (newItem); } void SetDestinationPathsMetadata (ITaskItem item, string path) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 2a52a44cea6..44eaf1ad847 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -1236,8 +1236,12 @@ public void BuildBasicApplicationCheckPdb () var proj = new XamarinAndroidApplicationProject (); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); - Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/assets/UnnamedProject.pdb")), - "UnnamedProject.pdb must be copied to the Intermediate directory"); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + + Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, $"android/assets/{abi}/UnnamedProject.pdb")), + $"UnnamedProject.pdb must be copied to the Intermediate directory for ABI {abi}"); + } } } @@ -1247,11 +1251,22 @@ public void BuildBasicApplicationCheckPdbRepeatBuild () var proj = new XamarinAndroidApplicationProject (); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); - Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/assets/UnnamedProject.pdb")), - "UnnamedProject.pdb must be copied to the Intermediate directory"); + + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + + Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, $"android/assets/{abi}/UnnamedProject.pdb")), + $"UnnamedProject.pdb must be copied to the Intermediate directory for ABI {abi}"); + } + Assert.IsTrue (b.Build (proj), "second build failed"); - Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android/assets/UnnamedProject.pdb")), - "UnnamedProject.pdb must be copied to the Intermediate directory"); + + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + + Assert.IsTrue (File.Exists (Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, $"android/assets/{abi}/UnnamedProject.pdb")), + $"UnnamedProject.pdb must be copied to the Intermediate directory for ABI {abi}"); + } } } @@ -1304,35 +1319,69 @@ public Class2 () Assert.IsTrue (b.Build (proj), "App1 Build should have succeeded."); var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); var outputPath = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath); - var assetsPdb = Path.Combine (intermediate, "android", "assets", "Library1.pdb"); - var binSrc = Path.Combine (outputPath, "Library1.pdb"); - Assert.IsTrue ( - File.Exists (Path.Combine (intermediate, "android", "assets", "Mono.Android.pdb")), - "Mono.Android.pdb must be copied to Intermediate directory"); + + const string LibraryBaseName = "Library1"; + const string AppBaseName = "App1"; + const string LibraryPdbName = LibraryBaseName + ".pdb"; + const string LibraryDllName = LibraryBaseName + ".dll"; + const string AppPdbName = AppBaseName + ".pdb"; + const string AppDllName = AppBaseName + ".dll"; + + string libraryPdbBinSrc = Path.Combine (outputPath, LibraryPdbName); + string appPdbBinSrc = Path.Combine (outputPath, AppPdbName); + Assert.IsTrue ( - File.Exists (assetsPdb), - "Library1.pdb must be copied to Intermediate directory"); + File.Exists (libraryPdbBinSrc), + $"{LibraryPdbName} must be copied to bin directory"); + Assert.IsTrue ( - File.Exists (binSrc), - "Library1.pdb must be copied to bin directory"); - using (var apk = ZipHelper.OpenZip (Path.Combine (outputPath, proj.PackageName + "-Signed.apk"))) { - var data = ZipHelper.ReadFileFromZip (apk, "assemblies/Library1.pdb"); - if (data == null) - data = File.ReadAllBytes (assetsPdb); - var filedata = File.ReadAllBytes (binSrc); - Assert.AreEqual (filedata.Length, data.Length, "Library1.pdb in the apk should match {0}", binSrc); + File.Exists (appPdbBinSrc), + $"{AppPdbName} must be copied to bin directory"); + + var fileNames = new List { + "Mono.Android.pdb", + AppPdbName, + LibraryPdbName, + AppDllName, + LibraryDllName, + }; + + string apkPath = Path.Combine (outputPath, proj.PackageName + "-Signed.apk"); + var helper = new ArchiveAssemblyHelper (apkPath, useAssemblyStores: false, b.GetBuildRuntimeIdentifiers ().ToArray ()); + foreach (string abi in b.GetBuildAbis ()) { + foreach (string fileName in fileNames) { + EnsureFilesAreTheSame (intermediate, outputPath, fileName, abi, helper, fileName.EndsWith (".dll", StringComparison.Ordinal)); + } } - var androidAssets = Path.Combine (intermediate, "android", "assets", "App1.pdb"); - binSrc = Path.Combine (outputPath, "App1.pdb"); - Assert.IsTrue ( - File.Exists (binSrc), - "App1.pdb must be copied to bin directory"); - FileAssert.AreEqual (binSrc, androidAssets, "{0} and {1} should not differ.", binSrc, androidAssets); - androidAssets = Path.Combine (intermediate, "android", "assets", "App1.dll"); - binSrc = Path.Combine (outputPath, "App1.dll"); - FileAssert.AreEqual (binSrc, androidAssets, "{0} and {1} should match.", binSrc, androidAssets); } } + + void EnsureFilesAreTheSame (string intermediatePath, string binPath, string fileName, string abi, ArchiveAssemblyHelper helper, bool uncompressIfNecessary) + { + string assetsPath = Path.Combine (intermediatePath, "android", "assets", abi, fileName); + Assert.IsTrue (File.Exists (assetsPath), $"{fileName} must be copied to Intermediate directory for ABI {abi}"); + + string apkPath = MonoAndroidHelper.MakeZipArchivePath ("assemblies", abi, fileName); + Stream? apkFileStream = helper.ReadEntry (apkPath, MonoAndroidHelper.AbiToTargetArch (abi), uncompressIfNecessary); + Assert.NotNull (apkFileStream, $"'{apkPath}' not found in the APK"); + + using var assetsFileStream = File.OpenRead (assetsPath); + FileAssert.AreEqual (apkFileStream, assetsFileStream, $"{0} and {1} should not differ", apkPath, assetsPath); + + // TODO: compare `bin` copies with those in `assets` and the apk. Right now: + // we mustn't compare against the pdb copy in `bin/` since we don't know which ABI was copied there, and the DLLs (and thus their debug data) + // may differ between ABIs/RIDs + + // assetsFileStream.Seek (0, SeekOrigin.Begin); + // apkFileStream.Seek (0, SeekOrigin.Begin); + + // string outputBinPath = Path.Combine (binPath, abi, fileName); + // using var outputBinStream = File.OpenRead (outputBinPath); + // FileAssert.AreEqual (assetsFileStream, outputBinStream, $"{0} and {1} should not differ", assetsPath, outputBinPath); + + // outputBinStream.Seek (0, SeekOrigin.Begin); + // FileAssert.AreEqual (apkFileStream, outputBinStream, $"{0} and {1} should not differ", apkPath, outputBinPath); + } } [Test] diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs index 4f75c742ab2..e61f0cec2f3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/GeneratePackageManagerJavaTests.cs @@ -1,4 +1,5 @@ using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -61,8 +62,12 @@ public void CheckPackageManagerAssemblyOrder (string[] resolvedUserAssemblies, s File.WriteAllText (Path.Combine (path, "AndroidManifest.xml"), $@""); - var resolvedUserAssembliesList = resolvedUserAssemblies.Select (x => new TaskItem (x)); - var resolvedAssembliesList = resolvedAssemblies.Select (x => new TaskItem (x)); + var metadata = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"Abi", "arm64-v8a"}, + }; + + var resolvedUserAssembliesList = resolvedUserAssemblies.Select (x => new TaskItem (x, metadata)); + var resolvedAssembliesList = resolvedAssemblies.Select (x => new TaskItem (x, metadata)); var task = new GeneratePackageManagerJava { BuildEngine = new MockBuildEngine (TestContext.Out), diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index f6dfbe73551..d8f146f0c4e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -57,11 +57,15 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, public Stream? ReadEntry (string path, AndroidTargetArch arch = AndroidTargetArch.None, bool uncompressIfNecessary = false) { + Stream? ret; if (useAssemblyStores) { - return ReadStoreEntry (path, arch, uncompressIfNecessary); + ret = ReadStoreEntry (path, arch, uncompressIfNecessary); + } else { + ret = ReadZipEntry (path, arch, uncompressIfNecessary); } - return ReadZipEntry (path, arch, uncompressIfNecessary); + ret?.Seek (0, SeekOrigin.Begin); + return ret; } Stream? ReadZipEntry (string path, AndroidTargetArch arch, bool uncompressIfNecessary) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs index f79bd5a4a81..8407a3af198 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Common/Builder.cs @@ -10,6 +10,8 @@ using System.Xml.XPath; using System.Xml.Linq; +using Xamarin.Android.Tasks; + namespace Xamarin.ProjectTools { public class Builder : IDisposable @@ -132,7 +134,6 @@ public void GetTargetFrameworkVersionRange (out string firstApiLevel, out string public HashSet GetBuildRuntimeIdentifiers () { var ret = new HashSet (StringComparer.OrdinalIgnoreCase); - foreach (string l in LastBuildOutput) { string line = l.Trim (); if (line.Length == 0 || line[0] != 'R') { @@ -153,6 +154,16 @@ public HashSet GetBuildRuntimeIdentifiers () return ret; } + public HashSet GetBuildAbis () + { + var ret = new HashSet (StringComparer.OrdinalIgnoreCase); + foreach (string rid in GetBuildRuntimeIdentifiers ()) { + ret.Add (MonoAndroidHelper.RidToAbi (rid)); + } + + return ret; + } + static string GetApiInfoElementValue (string androidApiInfo, string elementPath) { if (!File.Exists (androidApiInfo)) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj index 1f10e632927..d30eba4a101 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Xamarin.ProjectTools.csproj @@ -14,6 +14,7 @@ + %(RecursiveDir)appicon.png @@ -26,6 +27,7 @@ + {E34BCFA0-CAA4-412C-AA1C-75DB8D67D157} Xamarin.Android.Tools.AndroidSdk From 4cab3b7f9a2b196b358aae5426e099047a708048 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 21 Nov 2023 19:43:31 +0100 Subject: [PATCH 024/143] Another set of test fixes --- .../Tasks/BuildApk.cs | 6 +-- .../Tasks/PrepareSatelliteAssemblies.cs | 16 ++----- .../Utilities/ArchiveAssemblyHelper.cs | 47 ++++++++++++------- .../Utilities/AssemblyStoreAssemblyInfo.cs | 5 ++ 4 files changed, 42 insertions(+), 32 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 35d5473317f..1dcfcd342ad 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -610,16 +610,14 @@ string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) parts.Add (ArchiveAssembliesPath); } + // There's no need anymore to treat satellite assemblies specially here. The PrepareSatelliteAssemblies task takes care of + // properly setting `DestinationSubDirectory`, so we can just use it here. string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); if (string.IsNullOrEmpty (subDirectory)) { throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); } parts.Add (subDirectory.Replace ('\\', '/')); - // Even satellite assemblies are treated as RID-specific - if (!frameworkAssembly && SatelliteAssembly.TryGetSatelliteCultureAndFileName (assembly.ItemSpec, out var culture, out _)) { - parts.Add (culture); - } return MonoAndroidHelper.MakeZipArchivePath (haveRootPath ? RootPath : ArchiveAssembliesPath, parts) + "/"; } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs index 8849c98a92a..52e82043d6b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs @@ -49,27 +49,19 @@ void SetMetadata (ITaskItem item, List output) throw new InvalidOperationException ($"Assembly item '{item}' is missing the 'Culture' metadata"); } - string? targetPath = item.GetMetadata ("TargetPath"); - bool haveTargetPath = !String.IsNullOrEmpty (targetPath); string assemblyName = Path.GetFileName (item.ItemSpec); - char sep = Path.DirectorySeparatorChar; - foreach (string abi in BuildTargetAbis) { var newItem = new TaskItem (item); newItem.SetMetadata ("Abi", abi); - if (haveTargetPath) { - SetDestinationPathsMetadata (newItem, targetPath); - } else { - SetDestinationPathsMetadata (newItem, culture + sep + assemblyName); - } + SetDestinationPathsMetadata (newItem, MonoAndroidHelper.MakeZipArchivePath (abi, culture, assemblyName)); output.Add (newItem); } - void SetDestinationPathsMetadata (ITaskItem item, string path) + void SetDestinationPathsMetadata (ITaskItem item, string zipArchivePath) { - item.SetMetadata ("DestinationSubDirectory", path); - item.SetMetadata ("DestinationSubPath", path); + item.SetMetadata ("DestinationSubPath", zipArchivePath); + item.SetMetadata ("DestinationSubDirectory", Path.GetDirectoryName (zipArchivePath)); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index d8f146f0c4e..509981d9e19 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -157,25 +157,17 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb Console.WriteLine ($"Creating AssemblyStoreExplorer for archive '{archivePath}'"); (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); - AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); - Console.WriteLine ($"Explorer found {explorer.AssemblyCount} assemblies"); - - foreach (AssemblyStoreItem asm in explorer.Assemblies) { - string prefix = storeEntryPrefix; - -// if (haveMultipleRids && asm.TargetArch != AndroidTargetArch.None) { - string abi = MonoAndroidHelper.ArchToAbi (asm.TargetArch); - prefix = $"{prefix}{abi}/"; -// } - entries.Add ($"{prefix}{asm.Name}"); - if (asm.DebugOffset > 0) { - entries.Add ($"{prefix}{Path.GetFileNameWithoutExtension (asm.Name)}.pdb"); + if (arch == AndroidTargetArch.None) { + if (explorers == null || explorers.Count == 0) { + return entries; } - if (asm.ConfigOffset > 0) { - entries.Add ($"{prefix}{asm.Name}.config"); + foreach (AssemblyStoreExplorer? explorer in explorers) { + SynthetizeAssemblies (explorer); } + } else { + SynthetizeAssemblies (SelectExplorer (explorers, arch)); } Console.WriteLine ("Archive entries with synthetised assembly storeReader entries:"); @@ -184,6 +176,29 @@ public List ListArchiveContents (string storeEntryPrefix = DefaultAssemb } return entries; + + void SynthetizeAssemblies (AssemblyStoreExplorer? explorer) + { + if (explorer == null) { + return; + } + + Console.WriteLine ($"Explorer for {explorer.TargetArch} found {explorer.AssemblyCount} assemblies"); + foreach (AssemblyStoreItem asm in explorer.Assemblies) { + string prefix = storeEntryPrefix; + string abi = MonoAndroidHelper.ArchToAbi (asm.TargetArch); + prefix = $"{prefix}{abi}/"; + + entries.Add ($"{prefix}{asm.Name}"); + if (asm.DebugOffset > 0) { + entries.Add ($"{prefix}{Path.GetFileNameWithoutExtension (asm.Name)}.pdb"); + } + + if (asm.ConfigOffset > 0) { + entries.Add ($"{prefix}{asm.Name}.config"); + } + } + } } AssemblyStoreExplorer? SelectExplorer (IList? explorers, AndroidTargetArch arch) @@ -247,7 +262,7 @@ public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool force } if (!path.StartsWith (AssembliesPathTerminated, StringComparison.Ordinal)) { - throw new InvalidOperationException ($"Path '{path}' does not start with '{AssembliesPathTerminated}'"); + return new List { path }; } string[] parts = path.Split ('/'); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index f3ca88f7d79..002a84d9bf3 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -35,6 +35,11 @@ public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssembl name = Path.GetFileNameWithoutExtension (name); } + string? culture = assembly.GetMetadata ("Culture"); + if (!String.IsNullOrEmpty (culture)) { + name = $"{culture}/{name}"; + } + AssemblyName = name; AssemblyNameBytes = MonoAndroidHelper.Utf8StringToBytes (name); } From 3d780b1bc2989c44b76f3f3a2704f33a7e4bfd78 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 21 Nov 2023 22:07:48 +0100 Subject: [PATCH 025/143] Fix another test --- .../Tasks/PrepareSatelliteAssemblies.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs index 52e82043d6b..9bf049d723f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/PrepareSatelliteAssemblies.cs @@ -5,6 +5,7 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; @@ -46,7 +47,10 @@ void SetMetadata (ITaskItem item, List output) { string? culture = item.GetMetadata ("Culture"); if (String.IsNullOrEmpty (culture)) { - throw new InvalidOperationException ($"Assembly item '{item}' is missing the 'Culture' metadata"); + if (!SatelliteAssembly.TryGetSatelliteCultureAndFileName (item.ItemSpec, out culture, out _)) { + throw new InvalidOperationException ($"Assembly item '{item}' is missing the 'Culture' metadata and it wasn't possible to obtain it from the path"); + } + item.SetMetadata ("Culture", culture); } string assemblyName = Path.GetFileName (item.ItemSpec); From 17dfba32483402d7503dfdc74230d4d07de24ee8 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 21 Nov 2023 22:37:09 +0100 Subject: [PATCH 026/143] More fixes --- .../Tasks/ProcessAssemblies.cs | 2 +- .../Tests/Xamarin.Android.Build.Tests/PackagingTest.cs | 10 ++++++++-- .../Utilities/ArchiveAssemblyHelper.cs | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index b99e85e2aee..1dbd2ca2efd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -172,7 +172,7 @@ void SetIt (ITaskItem? item) return; } - string destination = Path.Combine (item.GetMetadata ("DestinationSubDirectory"), abi); + string destination = Path.Combine (abi, item.GetMetadata ("DestinationSubDirectory")); item.SetMetadata ("DestinationSubDirectory", destination + Path.DirectorySeparatorChar); item.SetMetadata ("DestinationSubPath", Path.Combine (destination, Path.GetFileName (item.ItemSpec))); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 526f8dbe9b8..5c0942c8156 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -59,11 +59,10 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto proj.OtherBuildItems.Add (new BuildItem ("Using", "System.Globalization")); proj.OtherBuildItems.Add (new BuildItem ("Using", "Humanizer")); - var expectedFiles = new [] { + var expectedFilesList = new List { "Java.Interop.dll", "Mono.Android.dll", "Mono.Android.Runtime.dll", - "rc.bin", "System.Console.dll", "System.Private.CoreLib.dll", "System.Runtime.dll", @@ -78,6 +77,13 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto "System.Text.RegularExpressions.dll", }; + if (usesAssemblyStores) { + expectedFilesList.Add ("arc.bin.so"); + } else { + expectedFilesList.Add ("rc.bin"); + } + + var expectedFiles = expectedFilesList.ToArray (); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "build should have succeeded."); var apk = Path.Combine (Root, b.ProjectDirectory, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 509981d9e19..3839dd8e3ea 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -367,6 +367,7 @@ public void Contains (string[] fileNames, out List existingFiles, out Li void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch) { + // TODO: make it work with ABIs when arch == None using (var zip = ZipHelper.OpenZip (archivePath)) { existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); missingFiles = fileNames.Where (x => !zip.ContainsEntry (assembliesRootDir + x)).ToList (); @@ -376,6 +377,7 @@ void ArchiveContains (string[] fileNames, out List existingFiles, out Li void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch) { + // TODO: make it work with ABIs when arch == None var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase)).ToList (); From 15be134cdd5c79f9ae015e49f89448e90bfdef07 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 22 Nov 2023 17:04:48 +0100 Subject: [PATCH 027/143] More test fixes --- ...oft.Android.Sdk.AssemblyResolution.targets | 2 + .../Tasks/ProcessAssemblies.cs | 8 ++- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 68 ++++++++++++------- .../IncrementalBuildTest.cs | 8 ++- .../SingleProjectTest.cs | 35 +++++----- .../Tasks/LinkerTests.cs | 47 +++++++------ .../BuildReleaseArm64SimpleDotNet.apkdesc | 58 ++++++++-------- 7 files changed, 133 insertions(+), 93 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets index b1cebb26584..22c04ec80f3 100644 --- a/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets +++ b/src/Xamarin.Android.Build.Tasks/Microsoft.Android.Sdk/targets/Microsoft.Android.Sdk.AssemblyResolution.targets @@ -124,6 +124,7 @@ _ResolveAssemblies MSBuild target. + <_ResolvedAssemblies Include="@(ResolvedAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ResolvedUserAssemblies Include="@(ResolvedUserAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> <_ResolvedFrameworkAssemblies Include="@(ResolvedFrameworkAssemblies->'$(MonoAndroidIntermediateAssemblyDir)%(DestinationSubPath)')" Condition=" '%(DestinationSubPath)' != '' " /> diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs index 1dbd2ca2efd..f61180045e8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ProcessAssemblies.cs @@ -26,6 +26,8 @@ public class ProcessAssemblies : AndroidTask [Required] public string [] RuntimeIdentifiers { get; set; } = Array.Empty(); + public bool DesignTimeBuild { get; set; } + public bool AndroidIncludeDebugSymbols { get; set; } public bool PublishTrimmed { get; set; } @@ -115,7 +117,11 @@ void SetMetadataForAssemblies (List output, Dictionary { - "Mono.Android.pdb", - AppPdbName, - LibraryPdbName, - AppDllName, - LibraryDllName, + var fileNames = new List<(string path, bool existsInBin)> { + ("Mono.Android.pdb", false), + (AppPdbName, true), + (LibraryPdbName, true), + (AppDllName, true), + (LibraryDllName, true), }; string apkPath = Path.Combine (outputPath, proj.PackageName + "-Signed.apk"); var helper = new ArchiveAssemblyHelper (apkPath, useAssemblyStores: false, b.GetBuildRuntimeIdentifiers ().ToArray ()); foreach (string abi in b.GetBuildAbis ()) { - foreach (string fileName in fileNames) { - EnsureFilesAreTheSame (intermediate, outputPath, fileName, abi, helper, fileName.EndsWith (".dll", StringComparison.Ordinal)); + foreach ((string fileName, bool existsInBin) in fileNames) { + EnsureFilesAreTheSame (intermediate, existsInBin ? outputPath : null, fileName, abi, helper, uncompressIfNecessary: fileName.EndsWith (".dll", StringComparison.Ordinal)); } } } } - void EnsureFilesAreTheSame (string intermediatePath, string binPath, string fileName, string abi, ArchiveAssemblyHelper helper, bool uncompressIfNecessary) + void EnsureFilesAreTheSame (string intermediatePath, string? binPath, string fileName, string abi, ArchiveAssemblyHelper helper, bool uncompressIfNecessary) { - string assetsPath = Path.Combine (intermediatePath, "android", "assets", abi, fileName); - Assert.IsTrue (File.Exists (assetsPath), $"{fileName} must be copied to Intermediate directory for ABI {abi}"); + string assetsFilePath = Path.Combine (intermediatePath, "android", "assets", abi, fileName); + Assert.IsTrue (File.Exists (assetsFilePath), $"'{fileName}' must be copied to Intermediate directory for ABI {abi}"); - string apkPath = MonoAndroidHelper.MakeZipArchivePath ("assemblies", abi, fileName); - Stream? apkFileStream = helper.ReadEntry (apkPath, MonoAndroidHelper.AbiToTargetArch (abi), uncompressIfNecessary); - Assert.NotNull (apkFileStream, $"'{apkPath}' not found in the APK"); + using var assetsFileStream = File.OpenRead (assetsFilePath); + string apkEntryPath = MonoAndroidHelper.MakeZipArchivePath ("assemblies", abi, fileName); + using Stream? apkEntryStream = helper.ReadEntry (apkEntryPath, MonoAndroidHelper.AbiToTargetArch (abi), uncompressIfNecessary); - using var assetsFileStream = File.OpenRead (assetsPath); - FileAssert.AreEqual (apkFileStream, assetsFileStream, $"{0} and {1} should not differ", apkPath, assetsPath); + if (apkEntryStream != null) { // FastDev won't put assemblies in the APK + FileAssert.AreEqual (apkEntryStream, assetsFileStream, $"'{apkEntryPath}' and '{assetsFilePath}' should not differ"); + } + + if (String.IsNullOrEmpty (binPath)) { + return; + } - // TODO: compare `bin` copies with those in `assets` and the apk. Right now: - // we mustn't compare against the pdb copy in `bin/` since we don't know which ABI was copied there, and the DLLs (and thus their debug data) - // may differ between ABIs/RIDs + // This is a bit fragile. We don't know which RID the `bin/` files were copied from, so we'll do our best to compare + // oranges to oranges by looking at file sizes before attempting the compare. This is a very weak predicate, because + // the files may differ in e.g. the MVID and still have the same size. The real fix for this is to have per-rid `bin/` + // subdirectories. + string binFilePath = Path.Combine (binPath, fileName); + Assert.IsTrue (File.Exists (binFilePath), $"'{fileName}' must be copied to the Output directory"); - // assetsFileStream.Seek (0, SeekOrigin.Begin); - // apkFileStream.Seek (0, SeekOrigin.Begin); + var assetsInfo = new FileInfo (assetsFilePath); + var binInfo = new FileInfo (binFilePath); - // string outputBinPath = Path.Combine (binPath, abi, fileName); - // using var outputBinStream = File.OpenRead (outputBinPath); - // FileAssert.AreEqual (assetsFileStream, outputBinStream, $"{0} and {1} should not differ", assetsPath, outputBinPath); + if (assetsInfo.Length != binInfo.Length) { + Assert.Warn ($"Ignoring comparison of '{binFilePath}' with '{assetsFilePath}' because their sizes differ"); + return; + } + + using var binFileStream = File.OpenRead (binFilePath); + assetsFileStream.Seek (0, SeekOrigin.Begin); + FileAssert.AreEqual (assetsFileStream, binFileStream, $"'{assetsFilePath}' and '{binFilePath}' should not differ"); - // outputBinStream.Seek (0, SeekOrigin.Begin); - // FileAssert.AreEqual (apkFileStream, outputBinStream, $"{0} and {1} should not differ", apkPath, outputBinPath); + if (apkEntryStream != null) { + binFileStream.Seek (0, SeekOrigin.Begin); + apkEntryStream.Seek (0, SeekOrigin.Begin); + FileAssert.AreEqual (apkEntryStream, binFileStream, $"'{apkEntryPath}' and '{binFilePath}' should not differ"); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index 8f1156729b0..05ce2386d18 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -430,14 +430,18 @@ public void AppProjectTargetsDoNotBreak () var output = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath); var intermediate = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); - var filesToTouch = new [] { + var filesToTouch = new List { Path.Combine (intermediate, "..", "project.assets.json"), Path.Combine (intermediate, "build.props"), Path.Combine (intermediate, $"{proj.ProjectName}.dll"), Path.Combine (intermediate, $"{proj.ProjectName}.pdb"), - Path.Combine (intermediate, "android", "assets", $"{proj.ProjectName}.dll"), Path.Combine (output, $"{proj.ProjectName}.dll.config"), }; + + foreach (string abi in b.GetBuildAbis ()) { + filesToTouch.Add (Path.Combine (intermediate, "android", "assets", abi, $"{proj.ProjectName}.dll")); + } + foreach (var file in filesToTouch) { FileAssert.Exists (file); File.SetLastWriteTimeUtc (file, DateTime.UtcNow); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs index aabd090f01e..cce32043203 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/SingleProjectTest.cs @@ -84,22 +84,25 @@ public void AndroidManifestProperties (string versionName, string versionCode, s var versionNumber = index == -1 ? $"{versionName}.0.0" : $"{versionName.Substring (0, index)}.0.0"; - var assemblyPath = b.Output.GetIntermediaryPath ($"android/assets/{proj.ProjectName}.dll"); - FileAssert.Exists (assemblyPath); - using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); - - // System.Reflection.AssemblyVersion - Assert.AreEqual (versionNumber, assembly.Name.Version.ToString ()); - - // System.Reflection.AssemblyFileVersion - var assemblyInfoVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyInformationalVersionAttribute"); - Assert.IsNotNull (assemblyInfoVersion, "Should find AssemblyInformationalVersionAttribute!"); - Assert.AreEqual (versionName, assemblyInfoVersion.ConstructorArguments [0].Value); - - // System.Reflection.AssemblyInformationalVersion - var assemblyFileVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyFileVersionAttribute"); - Assert.IsNotNull (assemblyFileVersion, "Should find AssemblyFileVersionAttribute!"); - Assert.AreEqual (versionNumber, assemblyFileVersion.ConstructorArguments [0].Value); + + foreach (string abi in b.GetBuildAbis ()) { + var assemblyPath = b.Output.GetIntermediaryPath ($"android/assets/{abi}/{proj.ProjectName}.dll"); + FileAssert.Exists (assemblyPath); + using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); + + // System.Reflection.AssemblyVersion + Assert.AreEqual (versionNumber, assembly.Name.Version.ToString ()); + + // System.Reflection.AssemblyFileVersion + var assemblyInfoVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyInformationalVersionAttribute"); + Assert.IsNotNull (assemblyInfoVersion, "Should find AssemblyInformationalVersionAttribute!"); + Assert.AreEqual (versionName, assemblyInfoVersion.ConstructorArguments [0].Value); + + // System.Reflection.AssemblyInformationalVersion + var assemblyFileVersion = assembly.CustomAttributes.FirstOrDefault (a => a.AttributeType.FullName == "System.Reflection.AssemblyFileVersionAttribute"); + Assert.IsNotNull (assemblyFileVersion, "Should find AssemblyFileVersionAttribute!"); + Assert.AreEqual (versionNumber, assemblyFileVersion.ConstructorArguments [0].Value); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 86efceed4ab..80b92522828 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -423,33 +423,42 @@ public unsafe bool MyMethod (Android.OS.IBinder windowToken, [global::Android.Ru Assert.IsTrue (b.Build (proj), "Building a project should have succeded."); var assemblyFile = "UnnamedProject.dll"; - var assemblyPath = (!isRelease || setLinkModeNone) ? b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", assemblyFile)) : BuildTest.GetLinkedPath (b, true, assemblyFile); - using (var assembly = AssemblyDefinition.ReadAssembly (assemblyPath)) { - Assert.IsTrue (assembly != null); + if (!isRelease || setLinkModeNone) { + foreach (string abi in b.GetBuildAbis ()) { + CheckAssembly (b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, assemblyFile))); + } + } else { + CheckAssembly (BuildTest.GetLinkedPath (b, true, assemblyFile)); + } + } - var td = assembly.MainModule.GetType ("UnnamedProject.MyClass"); - Assert.IsTrue (td != null); + void CheckAssembly (string assemblyPath) + { + using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); + Assert.IsTrue (assembly != null); - var mr = td.GetMethods ().Where (m => m.Name == "MyMethod").FirstOrDefault (); - Assert.IsTrue (mr != null); + var td = assembly.MainModule.GetType ("UnnamedProject.MyClass"); + Assert.IsTrue (td != null); - var md = mr.Resolve (); - Assert.IsTrue (md != null); + var mr = td.GetMethods ().Where (m => m.Name == "MyMethod").FirstOrDefault (); + Assert.IsTrue (mr != null); - bool hasKeepAliveCall = false; - foreach (var i in md.Body.Instructions) { - if (i.OpCode.Code != Mono.Cecil.Cil.Code.Call) - continue; + var md = mr.Resolve (); + Assert.IsTrue (md != null); - if (!i.Operand.ToString ().Contains ("System.GC::KeepAlive")) - continue; + bool hasKeepAliveCall = false; + foreach (var i in md.Body.Instructions) { + if (i.OpCode.Code != Mono.Cecil.Cil.Code.Call) + continue; - hasKeepAliveCall = true; - break; - } + if (!i.Operand.ToString ().Contains ("System.GC::KeepAlive")) + continue; - Assert.IsTrue (hasKeepAliveCall == shouldAddKeepAlives); + hasKeepAliveCall = true; + break; } + + Assert.IsTrue (hasKeepAliveCall == shouldAddKeepAlives); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 1dbc7d0c32b..26788c43c02 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -4,47 +4,47 @@ "AndroidManifest.xml": { "Size": 3036 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 1024 + "assemblies/arm64-v8a/_Microsoft.Android.Resource.Designer.dll": { + "Size": 1023 }, - "assemblies/Java.Interop.dll": { - "Size": 61350 + "assemblies/arm64-v8a/Java.Interop.dll": { + "Size": 61449 }, - "assemblies/Mono.Android.dll": { - "Size": 90818 + "assemblies/arm64-v8a/Mono.Android.dll": { + "Size": 90520 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5514 + "assemblies/arm64-v8a/Mono.Android.Runtime.dll": { + "Size": 5519 }, - "assemblies/rc.bin": { - "Size": 1512 - }, - "assemblies/System.Console.dll": { - "Size": 6536 + "assemblies/arm64-v8a/System.Console.dll": { + "Size": 6532 }, - "assemblies/System.Linq.dll": { - "Size": 7118 + "assemblies/arm64-v8a/System.Linq.dll": { + "Size": 7113 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 550324 + "assemblies/arm64-v8a/System.Private.CoreLib.dll": { + "Size": 548984 }, - "assemblies/System.Runtime.dll": { - "Size": 2614 + "assemblies/arm64-v8a/System.Runtime.dll": { + "Size": 2611 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 3851 + "assemblies/arm64-v8a/System.Runtime.InteropServices.dll": { + "Size": 3843 }, - "assemblies/UnnamedProject.dll": { - "Size": 2932 + "assemblies/arm64-v8a/UnnamedProject.dll": { + "Size": 2930 }, "classes.dex": { "Size": 377956 }, + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 + }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 97080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 334784 + "Size": 334712 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3193200 @@ -59,16 +59,16 @@ "Size": 154904 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 11648 + "Size": 11888 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1221 + "Size": 1213 }, "META-INF/BNDLTOOL.SF": { - "Size": 3037 + "Size": 3144 }, "META-INF/MANIFEST.MF": { - "Size": 2910 + "Size": 3017 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2783562 + "PackageSize": 2783669 } \ No newline at end of file From ef6fa4995f03f8407d9bb30f24dee26479eb329f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 23 Nov 2023 15:59:25 +0100 Subject: [PATCH 028/143] More test fixes --- .../PackagingTest.cs | 53 ++-- .../Utilities/ArchiveAssemblyHelper.cs | 161 +++++++--- .../BuildReleaseArm64XFormsDotNet.apkdesc | 280 +++++++++--------- .../Utilities/ProjectExtensions.cs | 17 +- .../Utilities/MonoAndroidHelper.Basic.cs | 16 + 5 files changed, 312 insertions(+), 215 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 5c0942c8156..54c0ee636f5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -39,8 +39,13 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto var proj = new XamarinAndroidApplicationProject { IsRelease = true }; + + AndroidTargetArch[] supportedArches = new[] { + AndroidTargetArch.Arm, + }; + proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyStores.ToString ()); - proj.SetAndroidSupportedAbis ("armeabi-v7a"); + proj.SetRuntimeIdentifiers (supportedArches); proj.PackageReferences.Add (new Package { Id = "Humanizer.Core", Version = "2.14.1", @@ -59,31 +64,25 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto proj.OtherBuildItems.Add (new BuildItem ("Using", "System.Globalization")); proj.OtherBuildItems.Add (new BuildItem ("Using", "Humanizer")); - var expectedFilesList = new List { - "Java.Interop.dll", - "Mono.Android.dll", - "Mono.Android.Runtime.dll", - "System.Console.dll", - "System.Private.CoreLib.dll", - "System.Runtime.dll", - "System.Runtime.InteropServices.dll", - "System.Linq.dll", - "UnnamedProject.dll", - "_Microsoft.Android.Resource.Designer.dll", - "Humanizer.dll", - "es/Humanizer.resources.dll", - "System.Collections.dll", - "System.Collections.Concurrent.dll", - "System.Text.RegularExpressions.dll", + var expectedFiles = new HashSet { + "Java.Interop.dll", + "Mono.Android.dll", + "Mono.Android.Runtime.dll", + "System.Console.dll", + "System.Private.CoreLib.dll", + "System.Runtime.dll", + "System.Runtime.InteropServices.dll", + "System.Linq.dll", + "UnnamedProject.dll", + "_Microsoft.Android.Resource.Designer.dll", + "Humanizer.dll", + "es/Humanizer.resources.dll", + "System.Collections.dll", + "System.Collections.Concurrent.dll", + "System.Text.RegularExpressions.dll", + "arc.bin.so", }; - if (usesAssemblyStores) { - expectedFilesList.Add ("arc.bin.so"); - } else { - expectedFilesList.Add ("rc.bin"); - } - - var expectedFiles = expectedFilesList.ToArray (); using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "build should have succeeded."); var apk = Path.Combine (Root, b.ProjectDirectory, @@ -93,15 +92,11 @@ public void CheckIncludedAssemblies ([Values (false, true)] bool usesAssemblySto List missingFiles; List additionalFiles; - helper.Contains (expectedFiles, out existingFiles, out missingFiles, out additionalFiles); + helper.Contains (expectedFiles, out existingFiles, out missingFiles, out additionalFiles, supportedArches); Assert.IsTrue (missingFiles == null || missingFiles.Count == 0, string.Format ("The following Expected files are missing. {0}", string.Join (Environment.NewLine, missingFiles))); - - Assert.IsTrue (additionalFiles == null || additionalFiles.Count == 0, - string.Format ("Unexpected Files found! {0}", - string.Join (Environment.NewLine, additionalFiles))); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 3839dd8e3ea..90f9e32136c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -201,6 +201,11 @@ void SynthetizeAssemblies (AssemblyStoreExplorer? explorer) } } + AssemblyStoreExplorer? SelectExplorer (IList? explorers, string rid) + { + return SelectExplorer (explorers, MonoAndroidHelper.RidToArch (rid)); + } + AssemblyStoreExplorer? SelectExplorer (IList? explorers, AndroidTargetArch arch) { if (explorers == null || explorers.Count == 0) { @@ -348,36 +353,92 @@ public bool Exists (string entryPath, bool forceRefresh = false, AndroidTargetAr return ArchiveContains (ListArchiveContents (assembliesRootDir, forceRefresh), entryPath, arch); } - public void Contains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch = AndroidTargetArch.None) + public void Contains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) { if (fileNames == null) { throw new ArgumentNullException (nameof (fileNames)); } - if (fileNames.Length == 0) { + if (fileNames.Count == 0) { throw new ArgumentException ("must not be empty", nameof (fileNames)); } if (useAssemblyStores) { - StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, arch); + StoreContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, targetArches); } else { - ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, arch); + ArchiveContains (fileNames, out existingFiles, out missingFiles, out additionalFiles, targetArches); + } + } + + List GetSupportedArches (IEnumerable? runtimeIdentifiers) + { + var rids = new List (); + if (runtimeIdentifiers != null) { + rids.AddRange (runtimeIdentifiers); + } + + if (rids.Count == 0) { + rids.AddRange (MonoAndroidHelper.SupportedTargetArchitectures); + } + + return rids; + } + + void ListFiles (List existingFiles, List missingFiles, List additionalFiles) + { + Console.WriteLine ("Archive contents:"); + ListFiles ("existing files", existingFiles); + ListFiles ("missing files", missingFiles); + ListFiles ("additional files", additionalFiles); + + void ListFiles (string label, List list) + { + Console.WriteLine ($" {label}:"); + if (list.Count == 0) { + Console.WriteLine (" none"); + return; + } + + foreach (string file in list) { + Console.WriteLine ($" {file}"); + } } } - void ArchiveContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch) + (string prefixAssemblies, string prefixLib) GetArchivePrefixes (string abi) => ($"{MonoAndroidHelper.MakeZipArchivePath (assembliesRootDir, abi)}/", $"lib/{abi}/"); + + void ArchiveContains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) { - // TODO: make it work with ABIs when arch == None - using (var zip = ZipHelper.OpenZip (archivePath)) { - existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); - missingFiles = fileNames.Where (x => !zip.ContainsEntry (assembliesRootDir + x)).ToList (); - additionalFiles = existingFiles.Where (x => !fileNames.Contains (x.Replace (assembliesRootDir, string.Empty))).ToList (); + using var zip = ZipHelper.OpenZip (archivePath); + existingFiles = zip.Where (a => a.FullName.StartsWith (assembliesRootDir, StringComparison.InvariantCultureIgnoreCase)).Select (a => a.FullName).ToList (); + existingFiles.AddRange (zip.Where (a => a.FullName.StartsWith ("lib/", StringComparison.OrdinalIgnoreCase)).Select (a => a.FullName)); + + List arches = GetSupportedArches (targetArches); + + missingFiles = new List (); + additionalFiles = new List (); + foreach (AndroidTargetArch arch in arches) { + string abi = MonoAndroidHelper.ArchToAbi (arch); + missingFiles.AddRange (GetMissingFilesForAbi (abi)); + additionalFiles.AddRange (GetAdditionalFilesForAbi (abi, existingFiles)); + } + ListFiles (existingFiles, missingFiles, additionalFiles); + + IEnumerable GetMissingFilesForAbi (string abi) + { + (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (abi); + return fileNames.Where (x => !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixAssemblies, x)) && !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixLib, x))); + } + + IEnumerable GetAdditionalFilesForAbi (string abi, List existingFiles) + { + (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (abi); + return existingFiles.Where (x => !fileNames.Contains (x.Replace (prefixAssemblies, string.Empty)) && !fileNames.Contains (x.Replace (prefixLib, String.Empty))); } } - void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, AndroidTargetArch arch) + void StoreContains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) { - // TODO: make it work with ABIs when arch == None var assemblyNames = fileNames.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)).ToList (); var configFiles = fileNames.Where (x => x.EndsWith (".config", StringComparison.OrdinalIgnoreCase)).ToList (); var debugFiles = fileNames.Where (x => x.EndsWith (".pdb", StringComparison.OrdinalIgnoreCase)).ToList (); @@ -387,50 +448,61 @@ void StoreContains (string[] fileNames, out List existingFiles, out List missingFiles = new List (); additionalFiles = new List (); - if (otherFiles.Count > 0) { - using (var zip = ZipHelper.OpenZip (archivePath)) { + using ZipArchive? zip = ZipHelper.OpenZip (archivePath); + + List arches = GetSupportedArches (targetArches); + (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); + + foreach (AndroidTargetArch arch in arches) { + AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); + if (explorer == null) { + continue; + } + + if (otherFiles.Count > 0) { + (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (MonoAndroidHelper.ArchToAbi (arch)); + foreach (string file in otherFiles) { - string fullPath = assembliesRootDir + file; + string fullPath = prefixAssemblies + file; + if (zip.ContainsEntry (fullPath)) { + existingFiles.Add (file); + } + + fullPath = prefixLib + file; if (zip.ContainsEntry (fullPath)) { existingFiles.Add (file); } } } - } - (IList? explorers, string? errorMessage) = AssemblyStoreExplorer.Open (archivePath); - AssemblyStoreExplorer? explorer = SelectExplorer (explorers, arch); - if (explorer == null) { - return; - } - - foreach (var f in explorer.AssembliesByName) { - Console.WriteLine ($"DEBUG!\tKey:{f.Key}"); - } + foreach (var f in explorer.AssembliesByName) { + Console.WriteLine ($"DEBUG!\tKey:{f.Key}"); + } - if (explorer.AssembliesByName.Count != 0) { - existingFiles.AddRange (explorer.AssembliesByName.Keys); + if (explorer.AssembliesByName.Count != 0) { + existingFiles.AddRange (explorer.AssembliesByName.Keys); - // We need to fake config and debug files since they have no named entries in the storeReader - foreach (string file in configFiles) { - AssemblyStoreItem asm = GetStoreAssembly (file); - if (asm == null) { - continue; - } + // We need to fake config and debug files since they have no named entries in the storeReader + foreach (string file in configFiles) { + AssemblyStoreItem asm = GetStoreAssembly (explorer, file); + if (asm == null) { + continue; + } - if (asm.ConfigOffset > 0) { - existingFiles.Add (file); + if (asm.ConfigOffset > 0) { + existingFiles.Add (file); + } } - } - foreach (string file in debugFiles) { - AssemblyStoreItem asm = GetStoreAssembly (file); - if (asm == null) { - continue; - } + foreach (string file in debugFiles) { + AssemblyStoreItem asm = GetStoreAssembly (explorer, file); + if (asm == null) { + continue; + } - if (asm.DebugOffset > 0) { - existingFiles.Add (file); + if (asm.DebugOffset > 0) { + existingFiles.Add (file); + } } } } @@ -443,8 +515,9 @@ void StoreContains (string[] fileNames, out List existingFiles, out List } additionalFiles = existingFiles.Where (x => !fileNames.Contains (x)).ToList (); + ListFiles (existingFiles, missingFiles, additionalFiles); - AssemblyStoreItem GetStoreAssembly (string file) + AssemblyStoreItem GetStoreAssembly (AssemblyStoreExplorer explorer, string file) { string assemblyName = Path.GetFileNameWithoutExtension (file); if (!explorer.AssembliesByName.TryGetValue (assemblyName, out AssemblyStoreItem asm) || asm == null) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 25f7dfca7a9..6e3b7363a12 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -4,215 +4,215 @@ "AndroidManifest.xml": { "Size": 3572 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 2102 + "assemblies/arm64-v8a/_Microsoft.Android.Resource.Designer.dll": { + "Size": 2100 }, - "assemblies/FormsViewGroup.dll": { - "Size": 7112 + "assemblies/arm64-v8a/FormsViewGroup.dll": { + "Size": 7105 }, - "assemblies/Java.Interop.dll": { - "Size": 69472 + "assemblies/arm64-v8a/Java.Interop.dll": { + "Size": 69537 }, - "assemblies/Mono.Android.dll": { - "Size": 447387 + "assemblies/arm64-v8a/Mono.Android.dll": { + "Size": 445422 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5514 + "assemblies/arm64-v8a/Mono.Android.Runtime.dll": { + "Size": 5516 }, - "assemblies/mscorlib.dll": { - "Size": 3852 + "assemblies/arm64-v8a/mscorlib.dll": { + "Size": 3845 }, - "assemblies/netstandard.dll": { - "Size": 5565 + "assemblies/arm64-v8a/netstandard.dll": { + "Size": 5554 }, - "assemblies/rc.bin": { - "Size": 1512 - }, - "assemblies/System.Collections.Concurrent.dll": { - "Size": 11502 + "assemblies/arm64-v8a/System.Collections.Concurrent.dll": { + "Size": 11486 }, - "assemblies/System.Collections.dll": { - "Size": 15398 + "assemblies/arm64-v8a/System.Collections.dll": { + "Size": 15377 }, - "assemblies/System.Collections.NonGeneric.dll": { - "Size": 7442 + "assemblies/arm64-v8a/System.Collections.NonGeneric.dll": { + "Size": 7432 }, - "assemblies/System.ComponentModel.dll": { - "Size": 1925 + "assemblies/arm64-v8a/System.ComponentModel.dll": { + "Size": 1921 }, - "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2538 + "assemblies/arm64-v8a/System.ComponentModel.Primitives.dll": { + "Size": 2536 }, - "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6020 + "assemblies/arm64-v8a/System.ComponentModel.TypeConverter.dll": { + "Size": 6016 }, - "assemblies/System.Console.dll": { - "Size": 6565 + "assemblies/arm64-v8a/System.Console.dll": { + "Size": 6561 }, - "assemblies/System.Core.dll": { - "Size": 1977 + "assemblies/arm64-v8a/System.Core.dll": { + "Size": 1975 }, - "assemblies/System.Diagnostics.DiagnosticSource.dll": { - "Size": 9057 + "assemblies/arm64-v8a/System.Diagnostics.DiagnosticSource.dll": { + "Size": 9050 }, - "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6540 + "assemblies/arm64-v8a/System.Diagnostics.TraceSource.dll": { + "Size": 6534 }, - "assemblies/System.dll": { - "Size": 2332 + "assemblies/arm64-v8a/System.dll": { + "Size": 2328 }, - "assemblies/System.Drawing.dll": { - "Size": 1923 + "assemblies/arm64-v8a/System.Drawing.dll": { + "Size": 1919 }, - "assemblies/System.Drawing.Primitives.dll": { - "Size": 11967 + "assemblies/arm64-v8a/System.Drawing.Primitives.dll": { + "Size": 11953 }, - "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11187 + "assemblies/arm64-v8a/System.IO.Compression.Brotli.dll": { + "Size": 11178 }, - "assemblies/System.IO.Compression.dll": { - "Size": 15856 + "assemblies/arm64-v8a/System.IO.Compression.dll": { + "Size": 15837 }, - "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9862 + "assemblies/arm64-v8a/System.IO.IsolatedStorage.dll": { + "Size": 9854 }, - "assemblies/System.Linq.dll": { - "Size": 19555 + "assemblies/arm64-v8a/System.Linq.dll": { + "Size": 19529 }, - "assemblies/System.Linq.Expressions.dll": { - "Size": 165111 + "assemblies/arm64-v8a/System.Linq.Expressions.dll": { + "Size": 164627 }, - "assemblies/System.Net.Http.dll": { - "Size": 67637 + "assemblies/arm64-v8a/System.Net.Http.dll": { + "Size": 67517 }, - "assemblies/System.Net.Primitives.dll": { - "Size": 22426 + "assemblies/arm64-v8a/System.Net.Primitives.dll": { + "Size": 22408 }, - "assemblies/System.Net.Requests.dll": { - "Size": 3589 + "assemblies/arm64-v8a/System.Net.Requests.dll": { + "Size": 3578 }, - "assemblies/System.ObjectModel.dll": { - "Size": 8106 + "assemblies/arm64-v8a/System.ObjectModel.dll": { + "Size": 8098 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 845653 + "assemblies/arm64-v8a/System.Private.CoreLib.dll": { + "Size": 843222 }, - "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 193645 + "assemblies/arm64-v8a/System.Private.DataContractSerialization.dll": { + "Size": 193094 }, - "assemblies/System.Private.Uri.dll": { - "Size": 42849 + "assemblies/arm64-v8a/System.Private.Uri.dll": { + "Size": 42787 }, - "assemblies/System.Private.Xml.dll": { - "Size": 215960 + "assemblies/arm64-v8a/System.Private.Xml.dll": { + "Size": 215307 }, - "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16631 + "assemblies/arm64-v8a/System.Private.Xml.Linq.dll": { + "Size": 16610 }, - "assemblies/System.Runtime.dll": { - "Size": 2740 + "assemblies/arm64-v8a/System.Runtime.dll": { + "Size": 2735 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 3851 + "assemblies/arm64-v8a/System.Runtime.InteropServices.dll": { + "Size": 3843 }, - "assemblies/System.Runtime.Serialization.dll": { - "Size": 1850 + "assemblies/arm64-v8a/System.Runtime.Serialization.dll": { + "Size": 1845 }, - "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2469 + "assemblies/arm64-v8a/System.Runtime.Serialization.Formatters.dll": { + "Size": 2467 }, - "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3747 + "assemblies/arm64-v8a/System.Runtime.Serialization.Primitives.dll": { + "Size": 3739 }, - "assemblies/System.Security.Cryptography.dll": { - "Size": 8089 + "assemblies/arm64-v8a/System.Security.Cryptography.dll": { + "Size": 8081 }, - "assemblies/System.Text.RegularExpressions.dll": { - "Size": 158950 + "assemblies/arm64-v8a/System.Text.RegularExpressions.dll": { + "Size": 158631 }, - "assemblies/System.Xml.dll": { - "Size": 1741 + "assemblies/arm64-v8a/System.Xml.dll": { + "Size": 1738 }, - "assemblies/System.Xml.Linq.dll": { - "Size": 1761 + "assemblies/arm64-v8a/System.Xml.Linq.dll": { + "Size": 1756 }, - "assemblies/UnnamedProject.dll": { - "Size": 4987 + "assemblies/arm64-v8a/UnnamedProject.dll": { + "Size": 4980 }, - "assemblies/Xamarin.AndroidX.Activity.dll": { - "Size": 5942 + "assemblies/arm64-v8a/Xamarin.AndroidX.Activity.dll": { + "Size": 5934 }, - "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6033 + "assemblies/arm64-v8a/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { + "Size": 6023 }, - "assemblies/Xamarin.AndroidX.AppCompat.dll": { - "Size": 119847 + "assemblies/arm64-v8a/Xamarin.AndroidX.AppCompat.dll": { + "Size": 119193 }, - "assemblies/Xamarin.AndroidX.CardView.dll": { - "Size": 6799 + "assemblies/arm64-v8a/Xamarin.AndroidX.CardView.dll": { + "Size": 6782 }, - "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { - "Size": 17257 + "assemblies/arm64-v8a/Xamarin.AndroidX.CoordinatorLayout.dll": { + "Size": 17219 }, - "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 100666 + "assemblies/arm64-v8a/Xamarin.AndroidX.Core.dll": { + "Size": 100217 }, - "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 14631 + "assemblies/arm64-v8a/Xamarin.AndroidX.DrawerLayout.dll": { + "Size": 14602 }, - "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 41733 + "assemblies/arm64-v8a/Xamarin.AndroidX.Fragment.dll": { + "Size": 41545 }, - "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { - "Size": 6080 + "assemblies/arm64-v8a/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { + "Size": 6072 }, - "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { - "Size": 6469 + "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.Common.dll": { + "Size": 6462 }, - "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { - "Size": 6615 + "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { + "Size": 6604 }, - "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { - "Size": 3068 + "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { + "Size": 3062 }, - "assemblies/Xamarin.AndroidX.Loader.dll": { - "Size": 12923 + "assemblies/arm64-v8a/Xamarin.AndroidX.Loader.dll": { + "Size": 12904 }, - "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 89997 + "assemblies/arm64-v8a/Xamarin.AndroidX.RecyclerView.dll": { + "Size": 89550 }, - "assemblies/Xamarin.AndroidX.SavedState.dll": { - "Size": 4906 + "assemblies/arm64-v8a/Xamarin.AndroidX.SavedState.dll": { + "Size": 4902 }, - "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 10572 + "assemblies/arm64-v8a/Xamarin.AndroidX.SwipeRefreshLayout.dll": { + "Size": 10555 }, - "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 18593 + "assemblies/arm64-v8a/Xamarin.AndroidX.ViewPager.dll": { + "Size": 18551 }, - "assemblies/Xamarin.Forms.Core.dll": { - "Size": 528450 + "assemblies/arm64-v8a/Xamarin.Forms.Core.dll": { + "Size": 526687 }, - "assemblies/Xamarin.Forms.Platform.Android.dll": { - "Size": 337925 + "assemblies/arm64-v8a/Xamarin.Forms.Platform.Android.dll": { + "Size": 336895 }, - "assemblies/Xamarin.Forms.Platform.dll": { - "Size": 11080 + "assemblies/arm64-v8a/Xamarin.Forms.Platform.dll": { + "Size": 11073 }, - "assemblies/Xamarin.Forms.Xaml.dll": { - "Size": 60774 + "assemblies/arm64-v8a/Xamarin.Forms.Xaml.dll": { + "Size": 60656 }, - "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 42282 + "assemblies/arm64-v8a/Xamarin.Google.Android.Material.dll": { + "Size": 42134 }, "classes.dex": { "Size": 3514468 }, + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 + }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 97080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 334784 + "Size": 334712 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3193200 @@ -227,7 +227,7 @@ "Size": 154904 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 102888 + "Size": 103128 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -341,13 +341,13 @@ "Size": 1213 }, "META-INF/BNDLTOOL.SF": { - "Size": 77024 + "Size": 77700 }, "META-INF/com.google.android.material_material.version": { "Size": 10 }, "META-INF/MANIFEST.MF": { - "Size": 76897 + "Size": 77573 }, "META-INF/proguard/androidx-annotations.pro": { "Size": 339 @@ -1916,5 +1916,5 @@ "Size": 325240 } }, - "PackageSize": 7949326 + "PackageSize": 7941801 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs index 82891c8360c..3e737019d26 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Utilities/ProjectExtensions.cs @@ -1,5 +1,9 @@ using System; using System.Collections.Generic; +using System.Linq; + +using Xamarin.Android.Tasks; +using Xamarin.Android.Tools; namespace Xamarin.ProjectTools { @@ -17,7 +21,7 @@ public static void SetAndroidSupportedAbis (this IShortFormProject project, para /// /// Sets $(AndroidSupportedAbis) or $(RuntimeIdentifiers) depending if running under dotnet /// - /// A semi-colon-delimited list of ABIs + /// A semi-colon-delimited list of ABIs [Obsolete ("SetAndroidSupportedAbis is deprecated, please use SetRuntimeIdentifiers instead.")] public static void SetAndroidSupportedAbis (this IShortFormProject project, string abis) { @@ -29,7 +33,7 @@ public static void SetRuntimeIdentifier (this IShortFormProject project, string project.SetProperty (KnownProperties.RuntimeIdentifier, AbiUtils.AbiToRuntimeIdentifier (androidAbi)); } - public static void SetRuntimeIdentifiers (this IShortFormProject project, string [] androidAbis) + public static void SetRuntimeIdentifiers (this IShortFormProject project, IEnumerable androidAbis) { var abis = new List (); foreach (var androidAbi in androidAbis) { @@ -37,5 +41,14 @@ public static void SetRuntimeIdentifiers (this IShortFormProject project, string } project.SetProperty (KnownProperties.RuntimeIdentifiers, string.Join (";", abis)); } + + public static void SetRuntimeIdentifiers (this IShortFormProject project, params AndroidTargetArch[] targetArches) + { + if (targetArches == null || targetArches.Length == 0) { + throw new ArgumentException ("must not be null or empty", nameof (targetArches)); + } + + project.SetProperty (KnownProperties.RuntimeIdentifiers, String.Join (";", targetArches.Select (arch => MonoAndroidHelper.ArchToRid (arch)))); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs index 5e2c0c32a10..7806c51b1d4 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs @@ -63,6 +63,13 @@ public static class RuntimeIdentifier { RuntimeIdentifier.X64, AndroidAbi.X64 }, }; + static readonly Dictionary RidToArchMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { RuntimeIdentifier.Arm32, AndroidTargetArch.Arm }, + { RuntimeIdentifier.Arm64, AndroidTargetArch.Arm64 }, + { RuntimeIdentifier.X86, AndroidTargetArch.X86 }, + { RuntimeIdentifier.X64, AndroidTargetArch.X86_64 }, + }; + static readonly Dictionary ArchToRidMap = new Dictionary { { AndroidTargetArch.Arm, RuntimeIdentifier.Arm32 }, { AndroidTargetArch.Arm64, RuntimeIdentifier.Arm64 }, @@ -104,6 +111,15 @@ public static string RidToAbi (string rid) return abi; } + public static AndroidTargetArch RidToArch (string rid) + { + if (!RidToArchMap.TryGetValue (rid, out AndroidTargetArch arch)) { + throw new NotSupportedException ($"Internal error: unsupported Runtime Identifier '{rid}'"); + }; + + return arch; + } + public static string ArchToRid (AndroidTargetArch arch) { if (!ArchToRidMap.TryGetValue (arch, out string rid)) { From 5a5960ba2e785b095c2e526c8f73d4803ff15b8f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 23 Nov 2023 18:33:39 +0100 Subject: [PATCH 029/143] Begin to fix typemaps, JCW and marshal methods generation All of this requires some refactoring of GenerateJavaStubs as the first step. --- .../Tasks/GenerateJavaStubs.cs | 110 +++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 475c86516fb..34cfbdf8e0c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -102,6 +102,8 @@ public override bool RunTask () { try { bool useMarshalMethods = !Debug && EnableMarshalMethods; + Run (useMarshalMethods); + // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { @@ -141,6 +143,112 @@ XAAssemblyResolver MakeResolver (bool useMarshalMethods) return res; } + void Run (bool useMarshalMethods) + { + // We will process each architecture completely separately as both type maps and marshal methods are strictly per-architecture and + // the assemblies should be processed strictly per architecture. Generation of JCWs, and the manifest are ABI-agnostic. + // We will generate them only for the first architecture, whichever it is. + Dictionary> allAssembliesPerArch = GetPerArchAssemblies (ResolvedAssemblies); + + Dictionary? firstArchAssemblies = null; + AndroidTargetArch firstArch = AndroidTargetArch.None; + foreach (var kvp in allAssembliesPerArch) { + if (firstArchAssemblies == null) { + firstArchAssemblies = kvp.Value; + firstArch = kvp.Key; + continue; + } + + EnsureDictionariesHaveTheSameEntries (firstArchAssemblies, kvp.Value, kvp.Key); + } + + // Should "never" happen... + if (firstArch == AndroidTargetArch.None) { + throw new InvalidOperationException ("Internal error: no per-architecture assemblies found?"); + } + + // ...just as this should "never" happen... + if (allAssembliesPerArch.Count != SupportedAbis.Length) { + throw new InvalidOperationException ($"Internal error: number of architectures ({allAssembliesPerArch.Count}) must equal the number of target ABIs ({SupportedAbis.Length})"); + } + + // ...or this... + foreach (string abi in SupportedAbis) { + AndroidTargetArch arch = MonoAndroidHelper.AbiToTargetArch (abi); + if (!allAssembliesPerArch.ContainsKey (arch)) { + throw new InvalidOperationException ($"Internal error: no assemblies for architecture '{arch}', which corresponds to target abi '{abi}'"); + } + } + + // ...as well as this + Dictionary> userAssembliesPerArch = GetPerArchAssemblies (ResolvedUserAssemblies); + foreach (var kvp in userAssembliesPerArch) { + if (!allAssembliesPerArch.TryGetValue (kvp.Key, out Dictionary allAssemblies)) { + throw new InvalidOperationException ($"Internal error: found user assemblies for architecture '{kvp.Key}' which isn't found in ResolvedAssemblies"); + } + + foreach (var asmKvp in kvp.Value) { + if (!allAssemblies.ContainsKey (asmKvp.Key)) { + throw new InvalidOperationException ($"Internal error: user assembly '{asmKvp.Value}' not found in ResolvedAssemblies"); + } + } + } + + // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture + var cache = new TypeDefinitionCache (); + (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (cache, firstArchAssemblies, userAssembliesPerArch[firstArch], useMarshalMethods); + + void EnsureDictionariesHaveTheSameEntries (Dictionary template, Dictionary dict, AndroidTargetArch arch) + { + if (dict.Count != template.Count) { + throw new InvalidOperationException ($"Internal error: architecture '{arch}' should have {template.Count} assemblies, however it has {dict.Count}"); + } + + foreach (var kvp in template) { + if (!dict.ContainsKey (kvp.Key)) { + throw new InvalidOperationException ($"Internal error: architecture '{arch}' does not have assembly '{kvp.Key}'"); + } + } + } + + Dictionary> GetPerArchAssemblies (ITaskItem[] input) + { + var assembliesPerArch = new Dictionary> (); + foreach (ITaskItem assembly in input) { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!assembliesPerArch.TryGetValue (arch, out Dictionary assemblies)) { + assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + assembliesPerArch.Add (arch, assemblies); + } + assemblies.Add (Path.GetFileName (assembly.ItemSpec), assembly); + } + + return assembliesPerArch; + } + } + + (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) + { + XAAssemblyResolver res = MakeResolver (useMarshalMethods); + var scanner = new XAJavaTypeScanner (Log, cache) { + ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, + }; + List allJavaTypes = scanner.GetJavaTypes (assemblies.Values, res); + var javaTypesForJCW = new List (); + + foreach (JavaType jt in allJavaTypes) { + // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during + // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android + // build and stored in a jar file. + if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + continue; + } + javaTypesForJCW.Add (jt); + } + + return (allJavaTypes, javaTypesForJCW); + } + void Run (XAAssemblyResolver res, bool useMarshalMethods) { PackageNamingPolicy pnp; @@ -218,7 +326,7 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) var javaTypes = new List (); foreach (JavaType jt in allJavaTypes) { - // Whem marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during + // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { From 41a3b515ab4ed71ecb03e17db96badd5b75e9edb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 24 Nov 2023 19:21:17 +0100 Subject: [PATCH 030/143] And the struggle against GenerateJavaStubs continues --- .../Tasks/GenerateJavaStubs.cs | 140 +++++++++++------- .../Tasks/GeneratePackageManagerJava.cs | 1 + .../Utilities/MarshalMethodsMirrorHelper.cs | 84 +++++++++++ .../MarshalMethodsNativeAssemblyGenerator.cs | 4 + .../Utilities/MarshalMethodsState.cs | 2 + .../Utilities/MonoAndroidHelper.cs | 61 ++++++++ 6 files changed, 242 insertions(+), 50 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 34cfbdf8e0c..79809dab621 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -102,7 +102,7 @@ public override bool RunTask () { try { bool useMarshalMethods = !Debug && EnableMarshalMethods; - Run (useMarshalMethods); + // Run (useMarshalMethods); // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them @@ -145,29 +145,15 @@ XAAssemblyResolver MakeResolver (bool useMarshalMethods) void Run (bool useMarshalMethods) { + PackageNamingPolicy pnp; + JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; + // We will process each architecture completely separately as both type maps and marshal methods are strictly per-architecture and // the assemblies should be processed strictly per architecture. Generation of JCWs, and the manifest are ABI-agnostic. // We will generate them only for the first architecture, whichever it is. - Dictionary> allAssembliesPerArch = GetPerArchAssemblies (ResolvedAssemblies); - - Dictionary? firstArchAssemblies = null; - AndroidTargetArch firstArch = AndroidTargetArch.None; - foreach (var kvp in allAssembliesPerArch) { - if (firstArchAssemblies == null) { - firstArchAssemblies = kvp.Value; - firstArch = kvp.Key; - continue; - } - - EnsureDictionariesHaveTheSameEntries (firstArchAssemblies, kvp.Value, kvp.Key); - } + Dictionary> allAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, validate: true); // Should "never" happen... - if (firstArch == AndroidTargetArch.None) { - throw new InvalidOperationException ("Internal error: no per-architecture assemblies found?"); - } - - // ...just as this should "never" happen... if (allAssembliesPerArch.Count != SupportedAbis.Length) { throw new InvalidOperationException ($"Internal error: number of architectures ({allAssembliesPerArch.Count}) must equal the number of target ABIs ({SupportedAbis.Length})"); } @@ -181,7 +167,7 @@ void Run (bool useMarshalMethods) } // ...as well as this - Dictionary> userAssembliesPerArch = GetPerArchAssemblies (ResolvedUserAssemblies); + Dictionary> userAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedUserAssemblies, validate: true); foreach (var kvp in userAssembliesPerArch) { if (!allAssembliesPerArch.TryGetValue (kvp.Key, out Dictionary allAssemblies)) { throw new InvalidOperationException ($"Internal error: found user assemblies for architecture '{kvp.Key}' which isn't found in ResolvedAssemblies"); @@ -195,41 +181,34 @@ void Run (bool useMarshalMethods) } // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture - var cache = new TypeDefinitionCache (); - (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (cache, firstArchAssemblies, userAssembliesPerArch[firstArch], useMarshalMethods); - - void EnsureDictionariesHaveTheSameEntries (Dictionary template, Dictionary dict, AndroidTargetArch arch) - { - if (dict.Count != template.Count) { - throw new InvalidOperationException ($"Internal error: architecture '{arch}' should have {template.Count} assemblies, however it has {dict.Count}"); - } - - foreach (var kvp in template) { - if (!dict.ContainsKey (kvp.Key)) { - throw new InvalidOperationException ($"Internal error: architecture '{arch}' does not have assembly '{kvp.Key}'"); - } - } - } + MarshalMethodsClassifier? classifier = null; + AndroidTargetArch firstArch = AndroidTargetArch.None; + bool archAgnosticDone = false; - Dictionary> GetPerArchAssemblies (ITaskItem[] input) - { - var assembliesPerArch = new Dictionary> (); - foreach (ITaskItem assembly in input) { - AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); - if (!assembliesPerArch.TryGetValue (arch, out Dictionary assemblies)) { - assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - assembliesPerArch.Add (arch, assemblies); + foreach (var kvp in allAssembliesPerArch) { + AndroidTargetArch arch = kvp.Key; + Dictionary archAssemblies = kvp.Value; + + if (!archAgnosticDone) { + // TODO: resolver must be modified to search ONLY the arch-specific dirs. The directories need to come from @(ResolvedAssemblies), because we + // mustn't use FrameworkDirectories when linking is enabled. @(ResolvedAssemblies) will contain **all** the relevant assembly paths, linked + // or not, so that's where we should look for references. This **has to** be done on per-arch basis. + XAAssemblyResolver res = MakeResolver (useMarshalMethods); + var cache = new TypeDefinitionCache (); + (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (res, cache, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods); + + if (!GenerateJavaSourcesAndMaybeClassifyMarshalMethods (res, javaTypesForJCW, cache, useMarshalMethods, out classifier)) { + return; } - assemblies.Add (Path.GetFileName (assembly.ItemSpec), assembly); + firstArch = arch; + archAgnosticDone = true; } - - return assembliesPerArch; + RewriteMarshalMethods (classifier, firstArch, allAssembliesPerArch); } } - (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) + (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) { - XAAssemblyResolver res = MakeResolver (useMarshalMethods); var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; @@ -249,6 +228,37 @@ Dictionary> GetPerArchAssemblie return (allJavaTypes, javaTypesForJCW); } + void RewriteMarshalMethods (MarshalMethodsClassifier? classifier, AndroidTargetArch classifiedArch, Dictionary> allAssembliesPerArch) + { + if (classifier == null) { + return; + } + + var mirrorHelper = new MarshalMethodsMirrorHelper (classifier, classifiedArch, allAssembliesPerArch, Log); + IDictionary perArchMarshalMethods = mirrorHelper.Reflect (); + + // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed + // in order to properly generate wrapper methods in the marshal methods assembly rewriter. + // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. + var environmentParser = new EnvironmentFilesParser (); + + //Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); + + // var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); + // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + } + + bool GenerateJavaSourcesAndMaybeClassifyMarshalMethods (XAAssemblyResolver res, List javaTypesForJCW, TypeDefinitionCache cache, bool useMarshalMethods, out MarshalMethodsClassifier? classifier) + { + if (useMarshalMethods) { + classifier = new MarshalMethodsClassifier (cache, res, Log); + } else { + classifier = null; + } + + return CreateJavaSources (javaTypesForJCW, cache, classifier, useMarshalMethods); + } + void Run (XAAssemblyResolver res, bool useMarshalMethods) { PackageNamingPolicy pnp; @@ -322,19 +332,39 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; + + Console.WriteLine ("Will scan for Java types in the following assemblies:"); + foreach (var asm in allTypemapAssemblies.Values) { + Console.WriteLine ($" {asm}"); + } List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - var javaTypes = new List (); + Console.WriteLine (); + Console.WriteLine ("All Java types:"); + foreach (JavaType jt in allJavaTypes) { + Console.WriteLine ($" {jt.Type.FullName}"); + } + var javaTypes = new List (); + Console.WriteLine (); + Console.WriteLine ($"Checking for types to reject (useMarshalMethods == {useMarshalMethods})"); foreach (JavaType jt in allJavaTypes) { // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + Console.WriteLine ($" rejecting: {jt.Type.FullName} ({jt.Type.Module.Assembly} => {jt.Type.Module.Assembly.Name.Name}"); + Console.WriteLine ($" if statement: {!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)} || {JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)}"); continue; } javaTypes.Add (jt); } + Console.WriteLine (); + Console.WriteLine ("Post-processed Java types:"); + foreach (JavaType jt in javaTypes) { + Console.WriteLine ($" {jt.Type.FullName}"); + } + MarshalMethodsClassifier classifier = null; if (useMarshalMethods) { classifier = new MarshalMethodsClassifier (cache, res, Log); @@ -346,6 +376,8 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) return; if (useMarshalMethods) { + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods:", classifier.MarshalMethods); + // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed // in order to properly generate wrapper methods in the marshal methods assembly rewriter. // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. @@ -353,8 +385,12 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after AddMethodsFromAbiSpecificAssemblies:", classifier.MarshalMethods); + var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (from classifier.MarshalMethods)", classifier.MarshalMethods); } // Step 3 - Generate type maps @@ -639,7 +675,11 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache } if (useMarshalMethods) { - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), new MarshalMethodsState (classifier.MarshalMethods), RegisteredTaskObjectLifetime.Build); + var state = new MarshalMethodsState (classifier.MarshalMethods); + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (CreateJavaSources/state)", state.MarshalMethods); + BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), state, RegisteredTaskObjectLifetime.Build); + var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after state registered and retrieved (CreateJavaSources/state)", state.MarshalMethods); } return ok; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 3731bc46eae..f0bcbd2b8c7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -389,6 +389,7 @@ void AddEnvironment () MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; if (enableMarshalMethods) { + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods in GeneratePackageManagerJava (from registered state)", marshalMethodsState.MarshalMethods!); marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( assemblyCount, uniqueAssemblyNames, diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs new file mode 100644 index 00000000000..a13aec2303c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +sealed class ArchitectureMarshalMethods +{ + public readonly IDictionary> MarshalMethods; + public readonly ICollection Assemblies; + + public ArchitectureMarshalMethods (IDictionary> marshalMethods, ICollection assemblies) + { + MarshalMethods = marshalMethods; + Assemblies = assemblies; + } +} + +/// +/// +/// Classifier contains types from assemblies that were in the architecture passed to the type scanner, so we can take a small shortcut in their case and use the types +/// as they are. For the other architectures we will have to look up types by name, using the scanned arch types as template, because the +/// assemblies are in different locations and may differ as far as type and method identifiers are concerned. They **have to**, however, contain all +/// the same types and methods. We'll error out if we find any discrepancies. +/// +/// +/// This class performs the task of "mirroring" the template architecture assemblies in other architectures. Performance might be worse, but we can't avoid it. +/// +/// +class MarshalMethodsMirrorHelper +{ + MarshalMethodsClassifier classifier; + AndroidTargetArch templateArch; + Dictionary> allAssembliesPerArch; + TaskLoggingHelper log; + + public MarshalMethodsMirrorHelper (MarshalMethodsClassifier classifier, AndroidTargetArch templateArch, Dictionary> allAssembliesPerArch, TaskLoggingHelper log) + { + this.classifier = classifier; + this.templateArch = templateArch; + this.allAssembliesPerArch = allAssembliesPerArch; + this.log = log; + } + + public IDictionary Reflect () + { + var ret = new Dictionary (); + + foreach (var kvp in allAssembliesPerArch) { + AndroidTargetArch arch = kvp.Key; + IDictionary assemblies = kvp.Value; + + if (arch == templateArch) { + ret.Add (arch, new ArchitectureMarshalMethods (classifier.MarshalMethods, classifier.Assemblies)); + continue; + } + + ret.Add (arch, Reflect (arch, assemblies)); + } + + return ret; + } + + ArchitectureMarshalMethods Reflect (AndroidTargetArch arch, IDictionary archAssemblies) + { + var marshalMethods = new Dictionary> (StringComparer.Ordinal); + var assemblyDefinitions = new List (); + + Console.WriteLine (); + Console.WriteLine ($"Reflecting marshal methods for architecture {arch}"); + foreach (var kvp in classifier.MarshalMethods) { + Console.WriteLine ($"Marshal methods in: {kvp.Key}:"); + foreach (MarshalMethodEntry mme in kvp.Value) { + Console.WriteLine ($" {mme.DeclaringType.FullName}.{mme.NativeCallback}"); + } + } + + return new ArchitectureMarshalMethods (marshalMethods, assemblyDefinitions); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index b399bfb6f1a..3e3a2f1b9b0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -256,6 +256,8 @@ public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, IColl /// public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, ICollection uniqueAssemblyNames, IDictionary> marshalMethods, TaskLoggingHelper logger) { + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods in MarshalMethodsNativeAssemblyGenerator ctor", marshalMethods); + this.numberOfAssembliesInApk = numberOfAssembliesInApk; this.uniqueAssemblyNames = uniqueAssemblyNames ?? throw new ArgumentNullException (nameof (uniqueAssemblyNames)); this.marshalMethods = marshalMethods; @@ -305,6 +307,7 @@ void Init () foreach (IList entryList in marshalMethods.Values) { bool useFullNativeSignature = entryList.Count > 1; foreach (MarshalMethodEntry entry in entryList) { + logger.LogDebugMessage ($"MM: processing {entry.DeclaringType.FullName} {entry.NativeCallback.FullName}"); ProcessAndAddMethod (allMethods, entry, useFullNativeSignature, seenClasses, overloadedNativeSymbolNames); } } @@ -654,6 +657,7 @@ void AddMarshalMethods (LlvmIrModule module, AssemblyCacheState acs, LlvmIrVaria void AddMarshalMethod (LlvmIrModule module, MarshalMethodInfo method, ulong asmId, MarshalMethodsWriteState writeState) { + logger.LogDebugMessage ($"MM: generating code for {method.Method.DeclaringType.FullName} {method.Method.NativeCallback.FullName}"); CecilMethodDefinition nativeCallback = method.Method.NativeCallback; string backingFieldName = $"native_cb_{method.Method.JniMethodName}_{asmId}_{method.ClassCacheIndex}_{nativeCallback.MetadataToken.ToUInt32():x}"; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs index 66bf0fdba2e..03668cc2240 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsState.cs @@ -9,6 +9,8 @@ sealed class MarshalMethodsState public MarshalMethodsState (IDictionary> marshalMethods) { + MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified ethods in MarshalMethodsState ctor", marshalMethods); + MarshalMethods = marshalMethods ?? throw new ArgumentNullException (nameof (marshalMethods)); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 2593ac422e0..88480ca24e8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -583,5 +583,66 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec string relPath = GetToolsRootDirectoryRelativePath (androidBinUtilsDirectory); return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } + + public static Dictionary> GetPerArchAssemblies (IEnumerable input, bool validate) + { + var assembliesPerArch = new Dictionary> (); + foreach (ITaskItem assembly in input) { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!assembliesPerArch.TryGetValue (arch, out Dictionary assemblies)) { + assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + assembliesPerArch.Add (arch, assemblies); + } + assemblies.Add (Path.GetFileName (assembly.ItemSpec), assembly); + } + + if (!validate) { + return assembliesPerArch; + } + + Dictionary? firstArchAssemblies = null; + AndroidTargetArch firstArch = AndroidTargetArch.None; + foreach (var kvp in assembliesPerArch) { + if (firstArchAssemblies == null) { + firstArchAssemblies = kvp.Value; + firstArch = kvp.Key; + continue; + } + + EnsureDictionariesHaveTheSameEntries (firstArchAssemblies, kvp.Value, kvp.Key); + } + + // Should "never" happen... + if (firstArch == AndroidTargetArch.None) { + throw new InvalidOperationException ("Internal error: no per-architecture assemblies found?"); + } + + return assembliesPerArch; + + void EnsureDictionariesHaveTheSameEntries (Dictionary template, Dictionary dict, AndroidTargetArch arch) + { + if (dict.Count != template.Count) { + throw new InvalidOperationException ($"Internal error: architecture '{arch}' should have {template.Count} assemblies, however it has {dict.Count}"); + } + + foreach (var kvp in template) { + if (!dict.ContainsKey (kvp.Key)) { + throw new InvalidOperationException ($"Internal error: architecture '{arch}' does not have assembly '{kvp.Key}'"); + } + } + } + } + + internal static void DumpMarshalMethodsToConsole (string heading, IDictionary> marshalMethods) + { + Console.WriteLine (); + Console.WriteLine ($"{heading}:"); + foreach (var kvp in marshalMethods) { + Console.WriteLine ($" {kvp.Key}"); + foreach (var method in kvp.Value) { + Console.WriteLine ($" {method.DeclaringType.FullName} {method.NativeCallback.FullName}"); + } + } + } } } From 333f3a7606a96f0bcb4278ccc6a85e318c82215f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 28 Nov 2023 21:17:46 +0100 Subject: [PATCH 031/143] Might need to rethink it... tomorrow --- .../Tasks/GenerateJavaStubs.cs | 38 +++- .../Utilities/IAssemblyResolverExtensions.cs | 11 ++ .../Utilities/MarshalMethodsClassifier.cs | 4 +- .../Utilities/MarshalMethodsMirrorHelper.cs | 72 +++++-- .../Utilities/XAAssemblyResolver.cs | 184 +++++++++++++++++- .../Utilities/XAJavaTypeScanner.cs | 22 +++ 6 files changed, 305 insertions(+), 26 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 79809dab621..7694c2bfeac 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -102,7 +102,7 @@ public override bool RunTask () { try { bool useMarshalMethods = !Debug && EnableMarshalMethods; - // Run (useMarshalMethods); + Run (useMarshalMethods); // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them @@ -143,6 +143,32 @@ XAAssemblyResolver MakeResolver (bool useMarshalMethods) return res; } + XAAssemblyResolverNew MakeResolver (bool useMarshalMethods, AndroidTargetArch targetArch, Dictionary assemblies) + { + var readerParams = new ReaderParameters(); + if (useMarshalMethods) { + readerParams.ReadWrite = true; + readerParams.InMemory = true; + } + + var res = new XAAssemblyResolverNew (targetArch, Log, loadDebugSymbols: true, loadReaderParameters: readerParams); + var uniqueDirs = new HashSet (StringComparer.OrdinalIgnoreCase); + + Log.LogDebugMessage ($"Adding search directories to new architecture {targetArch} resolver:"); + foreach (var kvp in assemblies) { + string assemblyDir = Path.GetDirectoryName (kvp.Value.ItemSpec); + if (uniqueDirs.Contains (assemblyDir)) { + continue; + } + + uniqueDirs.Add (assemblyDir); + res.SearchDirectories.Add (assemblyDir); + Log.LogDebugMessage ($" {assemblyDir}"); + } + + return res; + } + void Run (bool useMarshalMethods) { PackageNamingPolicy pnp; @@ -189,11 +215,8 @@ void Run (bool useMarshalMethods) AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; + XAAssemblyResolverNew res = MakeResolver (useMarshalMethods, arch, archAssemblies); if (!archAgnosticDone) { - // TODO: resolver must be modified to search ONLY the arch-specific dirs. The directories need to come from @(ResolvedAssemblies), because we - // mustn't use FrameworkDirectories when linking is enabled. @(ResolvedAssemblies) will contain **all** the relevant assembly paths, linked - // or not, so that's where we should look for references. This **has to** be done on per-arch basis. - XAAssemblyResolver res = MakeResolver (useMarshalMethods); var cache = new TypeDefinitionCache (); (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (res, cache, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods); @@ -203,11 +226,12 @@ void Run (bool useMarshalMethods) firstArch = arch; archAgnosticDone = true; } + RewriteMarshalMethods (classifier, firstArch, allAssembliesPerArch); } } - (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) + (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) { var scanner = new XAJavaTypeScanner (Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, @@ -248,7 +272,7 @@ void RewriteMarshalMethods (MarshalMethodsClassifier? classifier, AndroidTargetA // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); } - bool GenerateJavaSourcesAndMaybeClassifyMarshalMethods (XAAssemblyResolver res, List javaTypesForJCW, TypeDefinitionCache cache, bool useMarshalMethods, out MarshalMethodsClassifier? classifier) + bool GenerateJavaSourcesAndMaybeClassifyMarshalMethods (XAAssemblyResolverNew res, List javaTypesForJCW, TypeDefinitionCache cache, bool useMarshalMethods, out MarshalMethodsClassifier? classifier) { if (useMarshalMethods) { classifier = new MarshalMethodsClassifier (cache, res, Log); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs new file mode 100644 index 00000000000..e35950739a8 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/IAssemblyResolverExtensions.cs @@ -0,0 +1,11 @@ +using Mono.Cecil; + +namespace Xamarin.Android.Tasks; + +static class AssebmlyResolverExtensions +{ + public static AssemblyDefinition? Resolve (this IAssemblyResolver resolver, string fullName, ReaderParameters? parameters = null) + { + return resolver?.Resolve (AssemblyNameReference.Parse (fullName), parameters); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs index e7418e430a1..092a99c679a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs @@ -224,7 +224,7 @@ public bool Matches (MethodDefinition method) } TypeDefinitionCache tdCache; - XAAssemblyResolver resolver; + IAssemblyResolver resolver; Dictionary> marshalMethods; HashSet assemblies; TaskLoggingHelper log; @@ -237,7 +237,7 @@ public bool Matches (MethodDefinition method) public ulong RejectedMethodCount => rejectedMethodCount; public ulong WrappedMethodCount => wrappedMethodCount; - public MarshalMethodsClassifier (TypeDefinitionCache tdCache, XAAssemblyResolver res, TaskLoggingHelper log) + public MarshalMethodsClassifier (TypeDefinitionCache tdCache, IAssemblyResolver res, TaskLoggingHelper log) { this.log = log ?? throw new ArgumentNullException (nameof (log)); this.tdCache = tdCache ?? throw new ArgumentNullException (nameof (tdCache)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs index a13aec2303c..b243f00abd8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Mono.Cecil; @@ -18,6 +19,12 @@ public ArchitectureMarshalMethods (IDictionary MarshalMethods = marshalMethods; Assemblies = assemblies; } + + public ArchitectureMarshalMethods () + { + MarshalMethods = new Dictionary> (StringComparer.OrdinalIgnoreCase); + Assemblies = new List (); + } } /// @@ -33,10 +40,10 @@ public ArchitectureMarshalMethods (IDictionary /// class MarshalMethodsMirrorHelper { - MarshalMethodsClassifier classifier; - AndroidTargetArch templateArch; - Dictionary> allAssembliesPerArch; - TaskLoggingHelper log; + readonly MarshalMethodsClassifier classifier; + readonly AndroidTargetArch templateArch; + readonly Dictionary> allAssembliesPerArch; + readonly TaskLoggingHelper log; public MarshalMethodsMirrorHelper (MarshalMethodsClassifier classifier, AndroidTargetArch templateArch, Dictionary> allAssembliesPerArch, TaskLoggingHelper log) { @@ -67,18 +74,59 @@ public IDictionary Reflect () ArchitectureMarshalMethods Reflect (AndroidTargetArch arch, IDictionary archAssemblies) { - var marshalMethods = new Dictionary> (StringComparer.Ordinal); - var assemblyDefinitions = new List (); + var ret = new ArchitectureMarshalMethods (); + var cache = new Dictionary (StringComparer.OrdinalIgnoreCase); - Console.WriteLine (); - Console.WriteLine ($"Reflecting marshal methods for architecture {arch}"); + log.LogDebugMessage ($"Reflecting marshal methods for architecture {arch}"); foreach (var kvp in classifier.MarshalMethods) { - Console.WriteLine ($"Marshal methods in: {kvp.Key}:"); - foreach (MarshalMethodEntry mme in kvp.Value) { - Console.WriteLine ($" {mme.DeclaringType.FullName}.{mme.NativeCallback}"); + foreach (MarshalMethodEntry templateMethod in kvp.Value) { + ReflectMethod (arch, templateMethod, archAssemblies, ret, cache); } } - return new ArchitectureMarshalMethods (marshalMethods, assemblyDefinitions); + return ret; + } + + void ReflectMethod (AndroidTargetArch arch, MarshalMethodEntry templateMethod, IDictionary archAssemblies, ArchitectureMarshalMethods archMarshalMethods, Dictionary cache) + { + string? assemblyName = templateMethod.NativeCallback.DeclaringType.Module?.Assembly?.Name?.Name; + if (String.IsNullOrEmpty (assemblyName)) { + throw new InvalidOperationException ($"Unable to obtain assembly name for method {templateMethod}"); + } + + if (!cache.TryGetValue (assemblyName, out AssemblyDefinition assembly)) { + assembly = LoadAssembly (arch, assemblyName, archAssemblies, cache); + cache.Add (assemblyName, assembly); + } + + throw new NotImplementedException (); + } + + AssemblyDefinition LoadAssembly (AndroidTargetArch arch, string assemblyName, IDictionary archAssemblies, Dictionary cache) + { + if (!archAssemblies.TryGetValue (assemblyName, out ITaskItem assemblyItem)) { + throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' not found for architecture '{arch}'"); + } + + throw new NotImplementedException (); + } + + void ReflectType (AndroidTargetArch arch, string fullTypeName, IList templateMethods, IDictionary archAssemblies, ArchitectureMarshalMethods archMarshalMethods) + { + log.LogDebugMessage ($" Marshal methods in: {fullTypeName}:"); + string[] parts = fullTypeName.Split (','); + if (parts.Length != 2) { + throw new InvalidOperationException ($"Internal error: invalid full type name '{fullTypeName}'"); + } + + string typeName = parts[0].Trim (); + string assemblyName = parts[1].Trim (); + if (!archAssemblies.TryGetValue (assemblyName, out ITaskItem assemblyItem)) { + throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' not found for architecture '{arch}'"); + } + + foreach (MarshalMethodEntry mme in templateMethods) { + log.LogDebugMessage ($" {mme.DeclaringType.FullName}.{mme.NativeCallback}"); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs index 746f45802e7..9126016c759 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -9,6 +9,185 @@ namespace Xamarin.Android.Tasks; +class XAAssemblyResolverNew : IAssemblyResolver +{ + readonly List viewStreams = new List (); + readonly Dictionary cache; + bool disposed; + TaskLoggingHelper log; + bool loadDebugSymbols; + ReaderParameters readerParameters; + AndroidTargetArch targetArch; + + /// + /// **MUST** point to directories which contain assemblies for single ABI **only**. + /// One special case is when linking isn't enabled, in which instance directories + /// containing ABI-agnostic assemblies can we used as well. + public ICollection SearchDirectories { get; } = new List (); + + public XAAssemblyResolverNew (AndroidTargetArch targetArch, TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) + { + this.targetArch = targetArch; + this.log = log; + this.loadDebugSymbols = loadDebugSymbols; + this.readerParameters = loadReaderParameters ?? new ReaderParameters(); + + cache = new Dictionary (StringComparer.OrdinalIgnoreCase); + } + + public AssemblyDefinition? Resolve (AssemblyNameReference name) + { + return Resolve (name, null); + } + + public AssemblyDefinition? Resolve (AssemblyNameReference name, ReaderParameters? parameters) + { + string shortName = name.Name; + if (cache.TryGetValue (shortName, out AssemblyDefinition? assembly)) { + return assembly; + } + + return FindAndLoadFromDirectories (name, parameters); + } + + AssemblyDefinition? FindAndLoadFromDirectories (AssemblyNameReference name, ReaderParameters? parameters) + { + string? assemblyFile; + foreach (string dir in SearchDirectories) { + if ((assemblyFile = SearchDirectory (name.Name, dir)) != null) { + return Load (assemblyFile, parameters); + } + } + + return null; + } + + static string? SearchDirectory (string name, string directory) + { + if (Path.IsPathRooted (name) && File.Exists (name)) { + return name; + } + + var file = Path.Combine (directory, $"{name}.dll"); + if (File.Exists (file)) { + return file; + } + + return null; + } + + public virtual AssemblyDefinition? Load (string filePath, ReaderParameters? readerParameters = null) + { + string name = Path.GetFileNameWithoutExtension (filePath); + + if (cache.TryGetValue (name, out AssemblyDefinition? assembly)) { + if (assembly != null) { + return assembly; + } + } + + try { + assembly = ReadAssembly (filePath, readerParameters); + } catch (Exception e) when (e is FileNotFoundException || e is DirectoryNotFoundException) { + // These are ok, we can return null + return null; + } + + if (!cache.ContainsKey (name)) { + cache.Add (name, assembly); + } + + return assembly; + } + + AssemblyDefinition ReadAssembly (string filePath, ReaderParameters? readerParametersOverride = null) + { + ReaderParameters templateParameters = readerParametersOverride ?? this.readerParameters; + bool haveDebugSymbols = loadDebugSymbols && File.Exists (Path.ChangeExtension (filePath, ".pdb")); + var loadReaderParams = new ReaderParameters () { + ApplyWindowsRuntimeProjections = templateParameters.ApplyWindowsRuntimeProjections, + AssemblyResolver = this, + MetadataImporterProvider = templateParameters.MetadataImporterProvider, + InMemory = templateParameters.InMemory, + MetadataResolver = templateParameters.MetadataResolver, + ReadingMode = templateParameters.ReadingMode, + ReadSymbols = haveDebugSymbols, + ReadWrite = templateParameters.ReadWrite, + ReflectionImporterProvider = templateParameters.ReflectionImporterProvider, + SymbolReaderProvider = templateParameters.SymbolReaderProvider, + SymbolStream = templateParameters.SymbolStream, + }; + try { + return LoadFromMemoryMappedFile (filePath, loadReaderParams); + } catch (Exception ex) { + log.LogWarning ($"Failed to read '{filePath}' with debugging symbols for target architecture '{targetArch}'. Retrying to load it without it. Error details are logged below."); + log.LogWarning ($"{ex.ToString ()}"); + loadReaderParams.ReadSymbols = false; + return LoadFromMemoryMappedFile (filePath, loadReaderParams); + } + } + + AssemblyDefinition LoadFromMemoryMappedFile (string file, ReaderParameters options) + { + // We can't use MemoryMappedFile when ReadWrite is true + if (options.ReadWrite) { + return AssemblyDefinition.ReadAssembly (file, options); + } + + bool origReadSymbols = options.ReadSymbols; + MemoryMappedViewStream? viewStream = null; + try { + // We must disable reading of symbols, even if they were present, because Cecil is unable to find the symbols file when + // assembly file name is unknown, and this is precisely the case when reading module from a stream. + // Until this issue is resolved, skipping symbol read saves time because reading exception isn't thrown and we don't + // retry the load. + options.ReadSymbols = false; + + // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict + using var fileStream = new FileStream (file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); + using var mappedFile = MemoryMappedFile.CreateFromFile (fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); + viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); + + AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, options).Assembly; + viewStreams.Add (viewStream); + + // We transferred the ownership of the viewStream to the collection. + viewStream = null; + + return result; + } finally { + options.ReadSymbols = origReadSymbols; + viewStream?.Dispose (); + } + } + + public void Dispose () + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose (disposing: true); + GC.SuppressFinalize (this); + } + + protected virtual void Dispose (bool disposing) + { + if (disposed || !disposing) { + return; + } + + foreach (var kvp in cache) { + kvp.Value?.Dispose (); + } + cache.Clear (); + + foreach (MemoryMappedViewStream viewStream in viewStreams) { + viewStream.Dispose (); + } + viewStreams.Clear (); + + disposed = true; + } +} + class XAAssemblyResolver : IAssemblyResolver { sealed class CacheEntry : IDisposable @@ -107,11 +286,6 @@ public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderP cache = new Dictionary (StringComparer.OrdinalIgnoreCase); } - public AssemblyDefinition? Resolve (string fullName, ReaderParameters? parameters = null) - { - return Resolve (AssemblyNameReference.Parse (fullName), parameters); - } - public AssemblyDefinition? Resolve (AssemblyNameReference name) { return Resolve (name, null); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 4640251e521..8c6677fb17f 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -47,6 +47,28 @@ public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) this.cache = cache; } + public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolverNew resolver) + { + var types = new Dictionary (StringComparer.Ordinal); + foreach (ITaskItem asmItem in inputAssemblies) { + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); + AssemblyDefinition asmdef = resolver.Load (asmItem.ItemSpec); + + foreach (ModuleDefinition md in asmdef.Modules) { + foreach (TypeDefinition td in md.Types) { + AddJavaType (td, types, arch); + } + } + } + + var ret = new List (); + foreach (var kvp in types) { + ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.PerAbi)); + } + + return ret; + } + public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) { var types = new Dictionary (StringComparer.Ordinal); From c3ae6733323f78bcf62aeb18cda98b630339fc17 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 29 Nov 2023 18:18:28 +0100 Subject: [PATCH 032/143] Bad direction, too complex... Trying to perform type scanning only once and then "mirror" assemblies from all other supported ABIs based on the first scan appears to be too fragile and overly complicated. It appears the best course of action is to scan for Java types over each assemblies for each architecture and then compare the sets (based on type names). JCWs can still be generated only once, of course. --- .../Tasks/GenerateJavaStubs.cs | 31 ++- .../Utilities/MarshalMethodsClassifier.cs | 17 +- .../Utilities/MarshalMethodsMirrorHelper.cs | 251 +++++++++++++++--- .../Utilities/XAAssemblyResolver.cs | 9 +- 4 files changed, 248 insertions(+), 60 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 7694c2bfeac..5ff64d07008 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -208,26 +208,35 @@ void Run (bool useMarshalMethods) // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture MarshalMethodsClassifier? classifier = null; - AndroidTargetArch firstArch = AndroidTargetArch.None; - bool archAgnosticDone = false; + MarshalMethodsMirrorHelperState? mirrorHelperState = null; + bool abiAgnosticCode = false; foreach (var kvp in allAssembliesPerArch) { AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; XAAssemblyResolverNew res = MakeResolver (useMarshalMethods, arch, archAssemblies); - if (!archAgnosticDone) { + if (!abiAgnosticCode) { var cache = new TypeDefinitionCache (); (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (res, cache, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods); if (!GenerateJavaSourcesAndMaybeClassifyMarshalMethods (res, javaTypesForJCW, cache, useMarshalMethods, out classifier)) { return; } - firstArch = arch; - archAgnosticDone = true; + abiAgnosticCode = true; + + if (useMarshalMethods) { + mirrorHelperState = new MarshalMethodsMirrorHelperState (arch, archAssemblies, classifier); + } + } + + if (mirrorHelperState != null) { + mirrorHelperState.CurrentArch = arch; + mirrorHelperState.CurrentArchResolver = res; + mirrorHelperState.CurrentArchAssemblies = archAssemblies; } - RewriteMarshalMethods (classifier, firstArch, allAssembliesPerArch); + RewriteMarshalMethods (mirrorHelperState); } } @@ -252,14 +261,14 @@ void Run (bool useMarshalMethods) return (allJavaTypes, javaTypesForJCW); } - void RewriteMarshalMethods (MarshalMethodsClassifier? classifier, AndroidTargetArch classifiedArch, Dictionary> allAssembliesPerArch) + void RewriteMarshalMethods (MarshalMethodsMirrorHelperState? mirrorHelperState) { - if (classifier == null) { + if (mirrorHelperState == null) { return; } - var mirrorHelper = new MarshalMethodsMirrorHelper (classifier, classifiedArch, allAssembliesPerArch, Log); - IDictionary perArchMarshalMethods = mirrorHelper.Reflect (); + var mirrorHelper = new MarshalMethodsMirrorHelper (mirrorHelperState, Log); + ArchitectureMarshalMethods perArchMarshalMethods = mirrorHelper.Reflect (); // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed // in order to properly generate wrapper methods in the marshal methods assembly rewriter. @@ -888,7 +897,7 @@ void FindMatchingMethodInType (MarshalMethodEntry methodEntry, TypeDefinition ty } Log.LogDebugMessage ($"Found match for '{typeNativeCallbackMethod.FullName}' in {type.Module.FileName}"); - string methodKey = classifier.GetStoreMethodKey (methodEntry); + string methodKey = methodEntry.GetStoreMethodKey (classifier.TypeDefinitionCache); classifier.MarshalMethods[methodKey].Add (new MarshalMethodEntry (methodEntry, typeNativeCallbackMethod)); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs index 092a99c679a..77144121d01 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs @@ -80,6 +80,13 @@ string EnsureNonEmpty (string s, string argName) return s; } + + public string GetStoreMethodKey (TypeDefinitionCache tdCache) + { + MethodDefinition registeredMethod = RegisteredMethod; + string typeName = registeredMethod.DeclaringType.FullName.Replace ('/', '+'); + return $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; + } } class MarshalMethodsClassifier : JavaCallableMethodClassifier @@ -236,6 +243,7 @@ public bool Matches (MethodDefinition method) public ICollection Assemblies => assemblies; public ulong RejectedMethodCount => rejectedMethodCount; public ulong WrappedMethodCount => wrappedMethodCount; + public TypeDefinitionCache TypeDefinitionCache => tdCache; public MarshalMethodsClassifier (TypeDefinitionCache tdCache, IAssemblyResolver res, TaskLoggingHelper log) { @@ -688,16 +696,9 @@ FieldDefinition FindField (TypeDefinition type, string fieldName, bool lookForIn return FindField (tdCache.Resolve (type.BaseType), fieldName, lookForInherited); } - public string GetStoreMethodKey (MarshalMethodEntry methodEntry) - { - MethodDefinition registeredMethod = methodEntry.RegisteredMethod; - string typeName = registeredMethod.DeclaringType.FullName.Replace ('/', '+'); - return $"{typeName}, {registeredMethod.DeclaringType.GetPartialAssemblyName (tdCache)}\t{registeredMethod.Name}"; - } - void StoreMethod (MarshalMethodEntry entry) { - string key = GetStoreMethodKey (entry); + string key = entry.GetStoreMethodKey (tdCache); // Several classes can override the same method, we need to generate the marshal method only once, at the same time // keeping track of overloads diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs index b243f00abd8..43bdafb837e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; +using Java.Interop.Tools.Cecil; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -11,19 +12,37 @@ namespace Xamarin.Android.Tasks; sealed class ArchitectureMarshalMethods { - public readonly IDictionary> MarshalMethods; + public readonly IDictionary> Methods; public readonly ICollection Assemblies; public ArchitectureMarshalMethods (IDictionary> marshalMethods, ICollection assemblies) { - MarshalMethods = marshalMethods; + Methods = marshalMethods; Assemblies = assemblies; } public ArchitectureMarshalMethods () { - MarshalMethods = new Dictionary> (StringComparer.OrdinalIgnoreCase); - Assemblies = new List (); + Methods = new Dictionary> (StringComparer.OrdinalIgnoreCase); + Assemblies = new HashSet (); + } +} + +sealed class MarshalMethodsMirrorHelperState +{ + public readonly AndroidTargetArch TemplateArch; + public readonly Dictionary TemplateArchAssemblies; + public readonly MarshalMethodsClassifier Classifier; + + public AndroidTargetArch CurrentArch { get; set; } = AndroidTargetArch.None; + public XAAssemblyResolverNew? CurrentArchResolver { get; set; } + public Dictionary? CurrentArchAssemblies { get; set; } + + public MarshalMethodsMirrorHelperState (AndroidTargetArch classifiedArch, Dictionary classifiedArchAssemblies, MarshalMethodsClassifier classifier) + { + TemplateArch = classifiedArch; + TemplateArchAssemblies = classifiedArchAssemblies; + Classifier = classifier; } } @@ -40,75 +59,229 @@ public ArchitectureMarshalMethods () /// class MarshalMethodsMirrorHelper { - readonly MarshalMethodsClassifier classifier; - readonly AndroidTargetArch templateArch; - readonly Dictionary> allAssembliesPerArch; + readonly MarshalMethodsMirrorHelperState state; readonly TaskLoggingHelper log; - public MarshalMethodsMirrorHelper (MarshalMethodsClassifier classifier, AndroidTargetArch templateArch, Dictionary> allAssembliesPerArch, TaskLoggingHelper log) + public MarshalMethodsMirrorHelper (MarshalMethodsMirrorHelperState state, TaskLoggingHelper log) { - this.classifier = classifier; - this.templateArch = templateArch; - this.allAssembliesPerArch = allAssembliesPerArch; + this.state = state; this.log = log; } - public IDictionary Reflect () + public ArchitectureMarshalMethods Reflect () { - var ret = new Dictionary (); + if (state.CurrentArch == state.TemplateArch) { + log.LogDebugMessage ($"Not reflecting marshal methods for architecture '{state.CurrentArch}' since it's the template architecture"); + return new ArchitectureMarshalMethods (state.Classifier.MarshalMethods, state.Classifier.Assemblies); + } - foreach (var kvp in allAssembliesPerArch) { - AndroidTargetArch arch = kvp.Key; - IDictionary assemblies = kvp.Value; + var ret = new ArchitectureMarshalMethods (); + var assemblyCache = new Dictionary (StringComparer.OrdinalIgnoreCase); + var typeCache = new TypeDefinitionCache (); - if (arch == templateArch) { - ret.Add (arch, new ArchitectureMarshalMethods (classifier.MarshalMethods, classifier.Assemblies)); - continue; + log.LogDebugMessage ($"Reflecting marshal methods for architecture {state.CurrentArch}"); + foreach (var kvp in state.Classifier.MarshalMethods) { + foreach (MarshalMethodEntry templateMethod in kvp.Value) { + ReflectMethod (templateMethod, ret, assemblyCache, typeCache); } + } - ret.Add (arch, Reflect (arch, assemblies)); + if (ret.Assemblies.Count != state.TemplateArchAssemblies.Count) { + throw new InvalidOperationException ($"Internal error: expected to found {state.TemplateArchAssemblies.Count} assemblies for architecture '{state.CurrentArch}', but found {ret.Assemblies.Count} instead"); + } + + foreach (AssemblyDefinition templateAssembly in state.Classifier.Assemblies) { + bool found = false; + + foreach (AssemblyDefinition assembly in ret.Assemblies) { + if (String.Compare (templateAssembly.FullName, assembly.FullName, StringComparison.Ordinal) != 0) { + continue; + } + found = true; + break; + } + + if (!found) { + throw new InvalidOperationException ($"Internal error: assembly '{templateAssembly.FullName}' not found in assembly set for architecture '{state.CurrentArch}'"); + } } return ret; } - ArchitectureMarshalMethods Reflect (AndroidTargetArch arch, IDictionary archAssemblies) + void ReflectMethod (MarshalMethodEntry templateMethod, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) { - var ret = new ArchitectureMarshalMethods (); - var cache = new Dictionary (StringComparer.OrdinalIgnoreCase); + TypeDefinition matchingType = GetMatchingType (templateMethod.NativeCallback, archMethods, assemblyCache, typeCache); + MethodDefinition nativeCallback = FindMatchingMethod (templateMethod.NativeCallback, matchingType, "native callback"); - log.LogDebugMessage ($"Reflecting marshal methods for architecture {arch}"); - foreach (var kvp in classifier.MarshalMethods) { - foreach (MarshalMethodEntry templateMethod in kvp.Value) { - ReflectMethod (arch, templateMethod, archAssemblies, ret, cache); + MarshalMethodEntry? archMethod = null; + if (templateMethod.IsSpecial) { + // All we need is the native callback in this case + archMethod = new MarshalMethodEntry ( + matchingType, + nativeCallback, + templateMethod.JniTypeName, + templateMethod.JniMethodName, + templateMethod.JniMethodSignature + ); + + AddMethod (archMethod); + return; + } + + // This marshal method must have **all** the associated methods present + matchingType = GetMatchingType (templateMethod.Connector, archMethods, assemblyCache, typeCache); + MethodDefinition connector = FindMatchingMethod (templateMethod.Connector, matchingType, "connector"); + + matchingType = GetMatchingType (templateMethod.RegisteredMethod, archMethods, assemblyCache, typeCache); + MethodDefinition registered = FindMatchingMethod (templateMethod.RegisteredMethod, matchingType, "registered"); + + matchingType = GetMatchingType (templateMethod.ImplementedMethod, archMethods, assemblyCache, typeCache); + MethodDefinition implemented = FindMatchingMethod (templateMethod.ImplementedMethod, matchingType, "implemented"); + + TypeDefinition? fieldMatchingType = GetMatchingType (templateMethod.CallbackField, archMethods, assemblyCache, typeCache); + FieldDefinition? callbackField = null; + if (fieldMatchingType != null) {// callback field is optional + callbackField = FindMatchingField (templateMethod.CallbackField, fieldMatchingType, "callback"); + } + + archMethod = new MarshalMethodEntry ( + matchingType, + nativeCallback, + connector, + registered, + implemented, + callbackField, + templateMethod.JniTypeName, + templateMethod.JniMethodName, + templateMethod.JniMethodSignature, + templateMethod.NeedsBlittableWorkaround + ); + AddMethod (archMethod); + + void AddMethod (MarshalMethodEntry method) + { + string methodKey = method.GetStoreMethodKey (typeCache); + if (!archMethods.Methods.TryGetValue (methodKey, out IList methodList)) { + methodList = new List (); + archMethods.Methods.Add (methodKey, methodList); } + + methodList.Add (method); } + } - return ret; + TypeDefinition GetMatchingType (MethodDefinition? templateMethod, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) + { + if (templateMethod == null) { + throw new ArgumentNullException (nameof (templateMethod)); + } + + return GetMatchingType (templateMethod.DeclaringType, archMethods, assemblyCache, typeCache); } - void ReflectMethod (AndroidTargetArch arch, MarshalMethodEntry templateMethod, IDictionary archAssemblies, ArchitectureMarshalMethods archMarshalMethods, Dictionary cache) + TypeDefinition? GetMatchingType (FieldDefinition? templateField, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) { - string? assemblyName = templateMethod.NativeCallback.DeclaringType.Module?.Assembly?.Name?.Name; + if (templateField == null) { + return null; + } + + return GetMatchingType (templateField.DeclaringType, archMethods, assemblyCache, typeCache); + } + + TypeDefinition GetMatchingType (TypeDefinition templateDeclaringType, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) + { + string? assemblyName = templateDeclaringType.Module?.Assembly?.Name?.Name; if (String.IsNullOrEmpty (assemblyName)) { - throw new InvalidOperationException ($"Unable to obtain assembly name for method {templateMethod}"); + throw new InvalidOperationException ($"Unable to obtain assembly name"); + } + assemblyName = $"{assemblyName}.dll"; + + if (!assemblyCache.TryGetValue (assemblyName, out AssemblyDefinition assembly)) { + assembly = LoadAssembly (assemblyName, assemblyCache); + assemblyCache.Add (assemblyName, assembly); + } + + if (!archMethods.Assemblies.Contains (assembly)) { + archMethods.Assemblies.Add (assembly); } - if (!cache.TryGetValue (assemblyName, out AssemblyDefinition assembly)) { - assembly = LoadAssembly (arch, assemblyName, archAssemblies, cache); - cache.Add (assemblyName, assembly); + string templateTypeName = templateDeclaringType.FullName; + log.LogDebugMessage ($" looking for type '{templateTypeName}' ('{templateDeclaringType.Name}')"); + + TypeDefinition? matchingType = typeCache.Resolve (templateDeclaringType); + if (matchingType == null) { + throw new InvalidOperationException ($"Unable to find type '{templateTypeName}'"); } - throw new NotImplementedException (); + if (matchingType == null) { + throw new InvalidOperationException ($"Unable to locate type '{templateDeclaringType.FullName}' in assembly '{assembly.FullName}'"); + } + log.LogDebugMessage (" type found"); + + return matchingType; } - AssemblyDefinition LoadAssembly (AndroidTargetArch arch, string assemblyName, IDictionary archAssemblies, Dictionary cache) + MethodDefinition FindMatchingMethod (MethodDefinition? templateMethod, TypeDefinition type, string methodDescription) { - if (!archAssemblies.TryGetValue (assemblyName, out ITaskItem assemblyItem)) { - throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' not found for architecture '{arch}'"); + if (templateMethod == null) { + throw new ArgumentNullException (nameof (templateMethod)); + } + + string templateMethodName = templateMethod.FullName; + log.LogDebugMessage ($" looking for method '{templateMethodName}'"); + foreach (MethodDefinition method in type.Methods) { + if (String.Compare (method.FullName, templateMethodName, StringComparison.Ordinal) != 0) { + continue; + } + + log.LogDebugMessage (" found"); + return method; + } + + throw new InvalidOperationException ($"Unable to locate {methodDescription} method '{templateMethod.FullName}' in '{type.FullName}'"); + } + + FieldDefinition? FindMatchingField (FieldDefinition? templateField, TypeDefinition type, string fieldDescription) + { + if (templateField == null) { + return null; + } + + string templateFieldName = templateField.FullName; + log.LogDebugMessage ($" looking for field '{templateFieldName}'"); + foreach (FieldDefinition field in type.Fields) { + if (String.Compare (field.FullName, templateFieldName, StringComparison.Ordinal) != 0) { + continue; + } + + log.LogDebugMessage (" found"); + return field; + } + + return null; + } + + AssemblyDefinition LoadAssembly (string assemblyName, Dictionary cache) + { + if (state.CurrentArchResolver == null) { + throw new InvalidOperationException ($"Internal error: resolver for architecture '{state.CurrentArch}' not set"); + } + + if (state.CurrentArchResolver.TargetArch != state.CurrentArch) { + throw new InvalidOperationException ($"Internal error: resolver should target architecture '{state.CurrentArch}', but it targets '{state.CurrentArchResolver.TargetArch}' instead"); + } + + if (!state.CurrentArchAssemblies.TryGetValue (assemblyName, out ITaskItem assemblyItem)) { + throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' not found for architecture '{state.CurrentArch}'"); + } + + AssemblyDefinition? assembly = state.CurrentArchResolver.Resolve (assemblyName); + if (assembly == null) { + throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' cannot be resolved for architecture '{state.CurrentArch}'"); } - throw new NotImplementedException (); + return assembly; } void ReflectType (AndroidTargetArch arch, string fullTypeName, IList templateMethods, IDictionary archAssemblies, ArchitectureMarshalMethods archMarshalMethods) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs index 9126016c759..f501bb3e41e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -17,13 +17,14 @@ class XAAssemblyResolverNew : IAssemblyResolver TaskLoggingHelper log; bool loadDebugSymbols; ReaderParameters readerParameters; - AndroidTargetArch targetArch; + readonly AndroidTargetArch targetArch; /// /// **MUST** point to directories which contain assemblies for single ABI **only**. /// One special case is when linking isn't enabled, in which instance directories /// containing ABI-agnostic assemblies can we used as well. public ICollection SearchDirectories { get; } = new List (); + public AndroidTargetArch TargetArch => targetArch; public XAAssemblyResolverNew (AndroidTargetArch targetArch, TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) { @@ -68,7 +69,11 @@ public XAAssemblyResolverNew (AndroidTargetArch targetArch, TaskLoggingHelper lo return name; } - var file = Path.Combine (directory, $"{name}.dll"); + if (!name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { + name = "${name}.dll"; + } + + var file = Path.Combine (directory, name); if (File.Exists (file)) { return file; } From 91aa1f87c50f50fc7840dfdcf3b417aef85b897a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 30 Nov 2023 21:12:22 +0100 Subject: [PATCH 033/143] New direction. All the architectures will be scanned for Java types separately. This includes marshal method classification. Only the first architecture will generate JCWs. Code to do all that has been moved to a separate class. --- .../Tasks/GenerateJavaStubs.cs | 93 +++--- .../Utilities/JCWGenerator.cs | 213 ++++++++++++ .../Utilities/JavaStubsState.cs | 29 ++ .../Utilities/MarshalMethodsMirrorHelper.cs | 305 ------------------ 4 files changed, 287 insertions(+), 353 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 5ff64d07008..00f9cec77b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -207,37 +207,45 @@ void Run (bool useMarshalMethods) } // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture - MarshalMethodsClassifier? classifier = null; - MarshalMethodsMirrorHelperState? mirrorHelperState = null; - bool abiAgnosticCode = false; - + bool generateJavaCode = true; foreach (var kvp in allAssembliesPerArch) { AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; + (bool success, JavaStubsState? stubsState) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods, generateJavaCode); - XAAssemblyResolverNew res = MakeResolver (useMarshalMethods, arch, archAssemblies); - if (!abiAgnosticCode) { - var cache = new TypeDefinitionCache (); - (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (res, cache, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods); - - if (!GenerateJavaSourcesAndMaybeClassifyMarshalMethods (res, javaTypesForJCW, cache, useMarshalMethods, out classifier)) { - return; - } - abiAgnosticCode = true; - - if (useMarshalMethods) { - mirrorHelperState = new MarshalMethodsMirrorHelperState (arch, archAssemblies, classifier); - } + if (!success) { + return; } - if (mirrorHelperState != null) { - mirrorHelperState.CurrentArch = arch; - mirrorHelperState.CurrentArchResolver = res; - mirrorHelperState.CurrentArchAssemblies = archAssemblies; + if (generateJavaCode) { + generateJavaCode = false; } - RewriteMarshalMethods (mirrorHelperState); +// RewriteMarshalMethods (mirrorHelperState); + } + } + + (bool success, JavaStubsState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) + { + XAAssemblyResolverNew resolver = MakeResolver (useMarshalMethods, arch, assemblies); + var tdCache = new TypeDefinitionCache (); + (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); + + var jcwContext = new JCWGeneratorContext (arch, resolver, assemblies.Values, javaTypesForJCW, tdCache, useMarshalMethods); + var jcwGenerator = new JCWGenerator (Log, jcwContext); + bool success; + + if (generateJavaCode) { + success = jcwGenerator.GenerateAndClassify (AndroidSdkPlatform, outputPath: Path.Combine (OutputDirectory, "src"), ApplicationJavaClass); + } else { + success = jcwGenerator.Classify (AndroidSdkPlatform); + } + + if (!success) { + return (false, null); } + + return (true, new JavaStubsState (allJavaTypes, jcwGenerator.Classifier)); } (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) @@ -261,36 +269,25 @@ void Run (bool useMarshalMethods) return (allJavaTypes, javaTypesForJCW); } - void RewriteMarshalMethods (MarshalMethodsMirrorHelperState? mirrorHelperState) - { - if (mirrorHelperState == null) { - return; - } - - var mirrorHelper = new MarshalMethodsMirrorHelper (mirrorHelperState, Log); - ArchitectureMarshalMethods perArchMarshalMethods = mirrorHelper.Reflect (); - - // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed - // in order to properly generate wrapper methods in the marshal methods assembly rewriter. - // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. - var environmentParser = new EnvironmentFilesParser (); + // void RewriteMarshalMethods (MarshalMethodsMirrorHelperState? mirrorHelperState) + // { + // if (mirrorHelperState == null) { + // return; + // } - //Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); + // var mirrorHelper = new MarshalMethodsMirrorHelper (mirrorHelperState, Log); + // ArchitectureMarshalMethods perArchMarshalMethods = mirrorHelper.Reflect (); - // var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); - } + // // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed + // // in order to properly generate wrapper methods in the marshal methods assembly rewriter. + // // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. + // var environmentParser = new EnvironmentFilesParser (); - bool GenerateJavaSourcesAndMaybeClassifyMarshalMethods (XAAssemblyResolverNew res, List javaTypesForJCW, TypeDefinitionCache cache, bool useMarshalMethods, out MarshalMethodsClassifier? classifier) - { - if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); - } else { - classifier = null; - } + // //Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - return CreateJavaSources (javaTypesForJCW, cache, classifier, useMarshalMethods); - } + // // var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); + // // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + // } void Run (XAAssemblyResolver res, bool useMarshalMethods) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs new file mode 100644 index 00000000000..db1c734c841 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -0,0 +1,213 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.Diagnostics; +using Java.Interop.Tools.JavaCallableWrappers; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class JCWGeneratorContext +{ + public bool UseMarshalMethods { get; } + public AndroidTargetArch Arch { get; } + public TypeDefinitionCache TypeDefinitionCache { get; } + public XAAssemblyResolverNew Resolver { get; } + public IList JavaTypes { get; } + public ICollection ResolvedAssemblies { get; } + + public JCWGeneratorContext (AndroidTargetArch arch, XAAssemblyResolverNew res, ICollection resolvedAssemblies, List javaTypesForJCW, TypeDefinitionCache tdCache, bool useMarshalMethods) + { + Arch = arch; + Resolver = res; + ResolvedAssemblies = resolvedAssemblies; + JavaTypes = javaTypesForJCW.AsReadOnly (); + TypeDefinitionCache = tdCache; + UseMarshalMethods = useMarshalMethods; + } +} + +class JCWGenerator +{ + readonly TaskLoggingHelper log; + readonly JCWGeneratorContext context; + + public MarshalMethodsClassifier? Classifier { get; private set; } + + public JCWGenerator (TaskLoggingHelper log, JCWGeneratorContext context) + { + this.log = log; + this.context = context; + } + + /// + /// Performs marshal method classification, if marshal methods are used, but does not generate any code. + /// If marshal methods are used, this method will set the property to a valid + /// classifier instance on return. If marshal methods are disabled, this call is a no-op but it will + /// return true. + /// + public bool Classify (string androidSdkPlatform) + { + if (!context.UseMarshalMethods) { + return true; + } + + Classifier = new MarshalMethodsClassifier (context.TypeDefinitionCache, context.Resolver, log); + return ProcessTypes ( + generateCode: false, + androidSdkPlatform, + Classifier, + outputPath: null, + applicationJavaClass: null + ); + } + + public bool GenerateAndClassify (string androidSdkPlatform, string outputPath, string applicationJavaClass) + { + if (context.UseMarshalMethods) { + Classifier = new MarshalMethodsClassifier (context.TypeDefinitionCache, context.Resolver, log); + } + + return ProcessTypes ( + generateCode: true, + androidSdkPlatform, + Classifier, + outputPath, + applicationJavaClass + ); + } + + bool ProcessTypes (bool generateCode, string androidSdkPlatform, MarshalMethodsClassifier? classifier, string? outputPath, string? applicationJavaClass) + { + if (generateCode && String.IsNullOrEmpty (outputPath)) { + throw new ArgumentException ("must not be null or empty", nameof (outputPath)); + } + + string monoInit = GetMonoInitSource (androidSdkPlatform); + bool hasExportReference = context.ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); + bool ok = true; + + foreach (JavaType jt in context.JavaTypes) { + TypeDefinition type = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids + if (type.IsInterface) { + // Interfaces are in typemap but they shouldn't have JCW generated for them + continue; + } + + JavaCallableWrapperGenerator generator = CreateGenerator (type, classifier, monoInit, hasExportReference, applicationJavaClass); + if (!generateCode) { + continue; + } + + if (!GenerateCode (generator, type, outputPath, hasExportReference, classifier)) { + ok = false; + } + } + + return ok; + } + + bool GenerateCode (JavaCallableWrapperGenerator generator, TypeDefinition type, string outputPath, bool hasExportReference, MarshalMethodsClassifier? classifier) + { + bool ok = true; + using var writer = MemoryStreamPool.Shared.CreateStreamWriter (); + + try { + generator.Generate (writer); + if (context.UseMarshalMethods) { + if (classifier.FoundDynamicallyRegisteredMethods (type)) { + log.LogWarning ($"Type '{type.GetAssemblyQualifiedName (context.TypeDefinitionCache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); + } + } + writer.Flush (); + + string path = generator.GetDestinationPath (outputPath); + Files.CopyIfStreamChanged (writer.BaseStream, path); + if (generator.HasExport && !hasExportReference) { + Diagnostic.Error (4210, Properties.Resources.XA4210); + } + } catch (XamarinAndroidException xae) { + ok = false; + log.LogError ( + subcategory: "", + errorCode: "XA" + xae.Code, + helpKeyword: string.Empty, + file: xae.SourceFile, + lineNumber: xae.SourceLine, + columnNumber: 0, + endLineNumber: 0, + endColumnNumber: 0, + message: xae.MessageWithoutCode, + messageArgs: Array.Empty () + ); + } catch (DirectoryNotFoundException ex) { + ok = false; + if (OS.IsWindows) { + Diagnostic.Error (5301, Properties.Resources.XA5301, type.FullName, ex); + } else { + Diagnostic.Error (4209, Properties.Resources.XA4209, type.FullName, ex); + } + } catch (Exception ex) { + ok = false; + Diagnostic.Error (4209, Properties.Resources.XA4209, type.FullName, ex); + } + + return ok; + } + + JavaCallableWrapperGenerator CreateGenerator (TypeDefinition type, MarshalMethodsClassifier? classifier, string monoInit, bool hasExportReference, string? applicationJavaClass) + { + return new JavaCallableWrapperGenerator (type, log.LogWarning, context.TypeDefinitionCache, classifier) { + GenerateOnCreateOverrides = false, // this was used only when targetting Android API <= 10, which is no longer supported + ApplicationJavaClass = applicationJavaClass, + MonoRuntimeInitialization = monoInit, + }; + } + + static string GetMonoInitSource (string androidSdkPlatform) + { + if (String.IsNullOrEmpty (androidSdkPlatform)) { + throw new ArgumentException ("must not be null or empty", nameof (androidSdkPlatform)); + } + + // Lookup the mono init section from MonoRuntimeProvider: + // Mono Runtime Initialization {{{ + // }}} + var builder = new StringBuilder (); + var runtime = "Bundled"; + var api = ""; + if (int.TryParse (androidSdkPlatform, out int apiLevel) && apiLevel < 21) { + api = ".20"; + } + + var assembly = Assembly.GetExecutingAssembly (); + using var s = assembly.GetManifestResourceStream ($"MonoRuntimeProvider.{runtime}{api}.java"); + using var reader = new StreamReader (s); + bool copy = false; + string? line; + while ((line = reader.ReadLine ()) != null) { + if (string.CompareOrdinal ("\t\t// Mono Runtime Initialization {{{", line) == 0) { + copy = true; + } + + if (copy) { + builder.AppendLine (line); + } + + if (string.CompareOrdinal ("\t\t// }}}", line) == 0) { + break; + } + } + + return builder.ToString (); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs new file mode 100644 index 00000000000..1dbc1c199ac --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.Diagnostics; +using Java.Interop.Tools.JavaCallableWrappers; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +class JavaStubsState +{ + public MarshalMethodsClassifier? Classifier { get; } + public List AllJavaTypes { get; } + + public JavaStubsState (List allJavaTypes, MarshalMethodsClassifier? classifier) + { + AllJavaTypes = allJavaTypes; + Classifier = classifier; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs deleted file mode 100644 index 43bdafb837e..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsMirrorHelper.cs +++ /dev/null @@ -1,305 +0,0 @@ -using System; -using System.Collections.Generic; - -using Java.Interop.Tools.Cecil; -using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Mono.Cecil; -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks; - -sealed class ArchitectureMarshalMethods -{ - public readonly IDictionary> Methods; - public readonly ICollection Assemblies; - - public ArchitectureMarshalMethods (IDictionary> marshalMethods, ICollection assemblies) - { - Methods = marshalMethods; - Assemblies = assemblies; - } - - public ArchitectureMarshalMethods () - { - Methods = new Dictionary> (StringComparer.OrdinalIgnoreCase); - Assemblies = new HashSet (); - } -} - -sealed class MarshalMethodsMirrorHelperState -{ - public readonly AndroidTargetArch TemplateArch; - public readonly Dictionary TemplateArchAssemblies; - public readonly MarshalMethodsClassifier Classifier; - - public AndroidTargetArch CurrentArch { get; set; } = AndroidTargetArch.None; - public XAAssemblyResolverNew? CurrentArchResolver { get; set; } - public Dictionary? CurrentArchAssemblies { get; set; } - - public MarshalMethodsMirrorHelperState (AndroidTargetArch classifiedArch, Dictionary classifiedArchAssemblies, MarshalMethodsClassifier classifier) - { - TemplateArch = classifiedArch; - TemplateArchAssemblies = classifiedArchAssemblies; - Classifier = classifier; - } -} - -/// -/// -/// Classifier contains types from assemblies that were in the architecture passed to the type scanner, so we can take a small shortcut in their case and use the types -/// as they are. For the other architectures we will have to look up types by name, using the scanned arch types as template, because the -/// assemblies are in different locations and may differ as far as type and method identifiers are concerned. They **have to**, however, contain all -/// the same types and methods. We'll error out if we find any discrepancies. -/// -/// -/// This class performs the task of "mirroring" the template architecture assemblies in other architectures. Performance might be worse, but we can't avoid it. -/// -/// -class MarshalMethodsMirrorHelper -{ - readonly MarshalMethodsMirrorHelperState state; - readonly TaskLoggingHelper log; - - public MarshalMethodsMirrorHelper (MarshalMethodsMirrorHelperState state, TaskLoggingHelper log) - { - this.state = state; - this.log = log; - } - - public ArchitectureMarshalMethods Reflect () - { - if (state.CurrentArch == state.TemplateArch) { - log.LogDebugMessage ($"Not reflecting marshal methods for architecture '{state.CurrentArch}' since it's the template architecture"); - return new ArchitectureMarshalMethods (state.Classifier.MarshalMethods, state.Classifier.Assemblies); - } - - var ret = new ArchitectureMarshalMethods (); - var assemblyCache = new Dictionary (StringComparer.OrdinalIgnoreCase); - var typeCache = new TypeDefinitionCache (); - - log.LogDebugMessage ($"Reflecting marshal methods for architecture {state.CurrentArch}"); - foreach (var kvp in state.Classifier.MarshalMethods) { - foreach (MarshalMethodEntry templateMethod in kvp.Value) { - ReflectMethod (templateMethod, ret, assemblyCache, typeCache); - } - } - - if (ret.Assemblies.Count != state.TemplateArchAssemblies.Count) { - throw new InvalidOperationException ($"Internal error: expected to found {state.TemplateArchAssemblies.Count} assemblies for architecture '{state.CurrentArch}', but found {ret.Assemblies.Count} instead"); - } - - foreach (AssemblyDefinition templateAssembly in state.Classifier.Assemblies) { - bool found = false; - - foreach (AssemblyDefinition assembly in ret.Assemblies) { - if (String.Compare (templateAssembly.FullName, assembly.FullName, StringComparison.Ordinal) != 0) { - continue; - } - found = true; - break; - } - - if (!found) { - throw new InvalidOperationException ($"Internal error: assembly '{templateAssembly.FullName}' not found in assembly set for architecture '{state.CurrentArch}'"); - } - } - - return ret; - } - - void ReflectMethod (MarshalMethodEntry templateMethod, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) - { - TypeDefinition matchingType = GetMatchingType (templateMethod.NativeCallback, archMethods, assemblyCache, typeCache); - MethodDefinition nativeCallback = FindMatchingMethod (templateMethod.NativeCallback, matchingType, "native callback"); - - MarshalMethodEntry? archMethod = null; - if (templateMethod.IsSpecial) { - // All we need is the native callback in this case - archMethod = new MarshalMethodEntry ( - matchingType, - nativeCallback, - templateMethod.JniTypeName, - templateMethod.JniMethodName, - templateMethod.JniMethodSignature - ); - - AddMethod (archMethod); - return; - } - - // This marshal method must have **all** the associated methods present - matchingType = GetMatchingType (templateMethod.Connector, archMethods, assemblyCache, typeCache); - MethodDefinition connector = FindMatchingMethod (templateMethod.Connector, matchingType, "connector"); - - matchingType = GetMatchingType (templateMethod.RegisteredMethod, archMethods, assemblyCache, typeCache); - MethodDefinition registered = FindMatchingMethod (templateMethod.RegisteredMethod, matchingType, "registered"); - - matchingType = GetMatchingType (templateMethod.ImplementedMethod, archMethods, assemblyCache, typeCache); - MethodDefinition implemented = FindMatchingMethod (templateMethod.ImplementedMethod, matchingType, "implemented"); - - TypeDefinition? fieldMatchingType = GetMatchingType (templateMethod.CallbackField, archMethods, assemblyCache, typeCache); - FieldDefinition? callbackField = null; - if (fieldMatchingType != null) {// callback field is optional - callbackField = FindMatchingField (templateMethod.CallbackField, fieldMatchingType, "callback"); - } - - archMethod = new MarshalMethodEntry ( - matchingType, - nativeCallback, - connector, - registered, - implemented, - callbackField, - templateMethod.JniTypeName, - templateMethod.JniMethodName, - templateMethod.JniMethodSignature, - templateMethod.NeedsBlittableWorkaround - ); - AddMethod (archMethod); - - void AddMethod (MarshalMethodEntry method) - { - string methodKey = method.GetStoreMethodKey (typeCache); - if (!archMethods.Methods.TryGetValue (methodKey, out IList methodList)) { - methodList = new List (); - archMethods.Methods.Add (methodKey, methodList); - } - - methodList.Add (method); - } - } - - TypeDefinition GetMatchingType (MethodDefinition? templateMethod, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) - { - if (templateMethod == null) { - throw new ArgumentNullException (nameof (templateMethod)); - } - - return GetMatchingType (templateMethod.DeclaringType, archMethods, assemblyCache, typeCache); - } - - TypeDefinition? GetMatchingType (FieldDefinition? templateField, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) - { - if (templateField == null) { - return null; - } - - return GetMatchingType (templateField.DeclaringType, archMethods, assemblyCache, typeCache); - } - - TypeDefinition GetMatchingType (TypeDefinition templateDeclaringType, ArchitectureMarshalMethods archMethods, Dictionary assemblyCache, TypeDefinitionCache typeCache) - { - string? assemblyName = templateDeclaringType.Module?.Assembly?.Name?.Name; - if (String.IsNullOrEmpty (assemblyName)) { - throw new InvalidOperationException ($"Unable to obtain assembly name"); - } - assemblyName = $"{assemblyName}.dll"; - - if (!assemblyCache.TryGetValue (assemblyName, out AssemblyDefinition assembly)) { - assembly = LoadAssembly (assemblyName, assemblyCache); - assemblyCache.Add (assemblyName, assembly); - } - - if (!archMethods.Assemblies.Contains (assembly)) { - archMethods.Assemblies.Add (assembly); - } - - string templateTypeName = templateDeclaringType.FullName; - log.LogDebugMessage ($" looking for type '{templateTypeName}' ('{templateDeclaringType.Name}')"); - - TypeDefinition? matchingType = typeCache.Resolve (templateDeclaringType); - if (matchingType == null) { - throw new InvalidOperationException ($"Unable to find type '{templateTypeName}'"); - } - - if (matchingType == null) { - throw new InvalidOperationException ($"Unable to locate type '{templateDeclaringType.FullName}' in assembly '{assembly.FullName}'"); - } - log.LogDebugMessage (" type found"); - - return matchingType; - } - - MethodDefinition FindMatchingMethod (MethodDefinition? templateMethod, TypeDefinition type, string methodDescription) - { - if (templateMethod == null) { - throw new ArgumentNullException (nameof (templateMethod)); - } - - string templateMethodName = templateMethod.FullName; - log.LogDebugMessage ($" looking for method '{templateMethodName}'"); - foreach (MethodDefinition method in type.Methods) { - if (String.Compare (method.FullName, templateMethodName, StringComparison.Ordinal) != 0) { - continue; - } - - log.LogDebugMessage (" found"); - return method; - } - - throw new InvalidOperationException ($"Unable to locate {methodDescription} method '{templateMethod.FullName}' in '{type.FullName}'"); - } - - FieldDefinition? FindMatchingField (FieldDefinition? templateField, TypeDefinition type, string fieldDescription) - { - if (templateField == null) { - return null; - } - - string templateFieldName = templateField.FullName; - log.LogDebugMessage ($" looking for field '{templateFieldName}'"); - foreach (FieldDefinition field in type.Fields) { - if (String.Compare (field.FullName, templateFieldName, StringComparison.Ordinal) != 0) { - continue; - } - - log.LogDebugMessage (" found"); - return field; - } - - return null; - } - - AssemblyDefinition LoadAssembly (string assemblyName, Dictionary cache) - { - if (state.CurrentArchResolver == null) { - throw new InvalidOperationException ($"Internal error: resolver for architecture '{state.CurrentArch}' not set"); - } - - if (state.CurrentArchResolver.TargetArch != state.CurrentArch) { - throw new InvalidOperationException ($"Internal error: resolver should target architecture '{state.CurrentArch}', but it targets '{state.CurrentArchResolver.TargetArch}' instead"); - } - - if (!state.CurrentArchAssemblies.TryGetValue (assemblyName, out ITaskItem assemblyItem)) { - throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' not found for architecture '{state.CurrentArch}'"); - } - - AssemblyDefinition? assembly = state.CurrentArchResolver.Resolve (assemblyName); - if (assembly == null) { - throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' cannot be resolved for architecture '{state.CurrentArch}'"); - } - - return assembly; - } - - void ReflectType (AndroidTargetArch arch, string fullTypeName, IList templateMethods, IDictionary archAssemblies, ArchitectureMarshalMethods archMarshalMethods) - { - log.LogDebugMessage ($" Marshal methods in: {fullTypeName}:"); - string[] parts = fullTypeName.Split (','); - if (parts.Length != 2) { - throw new InvalidOperationException ($"Internal error: invalid full type name '{fullTypeName}'"); - } - - string typeName = parts[0].Trim (); - string assemblyName = parts[1].Trim (); - if (!archAssemblies.TryGetValue (assemblyName, out ITaskItem assemblyItem)) { - throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' not found for architecture '{arch}'"); - } - - foreach (MarshalMethodEntry mme in templateMethods) { - log.LogDebugMessage ($" {mme.DeclaringType.FullName}.{mme.NativeCallback}"); - } - } -} From 9cdd0c4c2fbb6a11c5df01b86f361e629f0ef70e Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 1 Dec 2023 17:38:29 +0100 Subject: [PATCH 034/143] This should be the right direction --- .../Tasks/GenerateJavaStubs.cs | 8 +- .../Utilities/JCWGenerator.cs | 88 +++++++++++++++++++ .../Utilities/JavaStubsState.cs | 34 ++++--- 3 files changed, 114 insertions(+), 16 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 00f9cec77b7..8215b8d480b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -207,11 +207,12 @@ void Run (bool useMarshalMethods) } // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture + var javaStubStates = new Dictionary (); bool generateJavaCode = true; foreach (var kvp in allAssembliesPerArch) { AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; - (bool success, JavaStubsState? stubsState) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods, generateJavaCode); + (bool success, JavaStubsState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods, generateJavaCode); if (!success) { return; @@ -221,8 +222,11 @@ void Run (bool useMarshalMethods) generateJavaCode = false; } + javaStubStates.Add (arch, state); // RewriteMarshalMethods (mirrorHelperState); } + + JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, javaStubStates); } (bool success, JavaStubsState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) @@ -245,7 +249,7 @@ void Run (bool useMarshalMethods) return (false, null); } - return (true, new JavaStubsState (allJavaTypes, jcwGenerator.Classifier)); + return (true, new JavaStubsState (arch, allJavaTypes, jcwGenerator.Classifier)); } (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs index db1c734c841..ca92442cb05 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -210,4 +210,92 @@ static string GetMonoInitSource (string androidSdkPlatform) return builder.ToString (); } + + public static void EnsureAllArchitecturesAreIdentical (TaskLoggingHelper logger, Dictionary javaStubStates) + { + if (javaStubStates.Count <= 1) { + return; + } + + // An expensive process, but we must be sure that all the architectures have the same data + JavaStubsState? templateStub = null; + foreach (var kvp in javaStubStates) { + JavaStubsState state = kvp.Value; + + if (templateStub == null) { + templateStub = state; + continue; + } + + EnsureIdenticalCollections (logger, templateStub, state); + } + } + + static void EnsureIdenticalCollections (TaskLoggingHelper logger, JavaStubsState templateState, JavaStubsState state) + { + if (templateState.Classifier == null) { + if (state.Classifier != null) { + throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES NOT have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); + } + } else { + if (state.Classifier == null) { + throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); + } + + EnsureClassifiersMatch (templateState, state); + } + + List templateTypes = templateState.AllJavaTypes; + List types = state.AllJavaTypes; + + if (types.Count != templateTypes.Count) { + throw new InvalidOperationException ($"Internal error: architecture '{state.TargetArch}' has a different number of types ({types.Count}) than the template architecture '{templateState.TargetArch}' ({templateTypes.Count})"); + } + + var matchedTemplateTypes = new HashSet (); + var mismatchedTypes = new List (); + + foreach (JavaType type in types) { + TypeDefinition? matchedType = null; + + foreach (JavaType templateType in templateTypes) { + if (matchedTemplateTypes.Contains (templateType.Type) || !CheckWhetherTypesMatch (templateType.Type, type.Type)) { + continue; + } + + matchedTemplateTypes.Add (templateType.Type); + matchedType = templateType.Type; + break; + } + + if (matchedType == null) { + mismatchedTypes.Add (type.Type); + } + } + + if (mismatchedTypes.Count > 0) { + logger.LogError ($"Architecture '{state.TargetArch}' has Java types which have no counterparts in template architecture '{templateState.TargetArch}':"); + foreach (TypeDefinition td in mismatchedTypes) { + logger.LogError ($" {td.FullName}"); + } + } + } + + static void EnsureClassifiersMatch (JavaStubsState templateState, JavaStubsState state) + { + MarshalMethodsClassifier templateClassifier = templateState.Classifier; + MarshalMethodsClassifier classifier = state.Classifier; + + if (templateClassifier.MarshalMethods.Count != classifier.MarshalMethods.Count) { + throw new InvalidOperationException ( + $"Internal error: classifier for template architecture '{templateState.TargetArch}' contains {templateClassifier.MarshalMethods.Count} marshal methods, but the one for architecture '{state.TargetArch}' has {classifier.MarshalMethods.Count}" + ); + } + } + + static bool CheckWhetherTypesMatch (TypeDefinition templateType, TypeDefinition type) + { + // TODO: should we compare individual methods, fields, properties? + return String.Compare (templateType.FullName, type.FullName, StringComparison.Ordinal) == 0; + } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs index 1dbc1c199ac..4fc21527b68 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs @@ -1,28 +1,34 @@ -using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; -using Java.Interop.Tools.Cecil; -using Java.Interop.Tools.Diagnostics; -using Java.Interop.Tools.JavaCallableWrappers; -using Microsoft.Android.Build.Tasks; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Mono.Cecil; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; +/// +/// Holds state for typemap and marshal methods generators. A single instance of this +/// class is created for each enabled target architecture. +/// class JavaStubsState { + /// + /// Target architecture for which this instance was created. + /// + public AndroidTargetArch TargetArch { get; } + + /// + /// Classifier used when scanning for Java types in the target architecture's + /// assemblies. Will be **null** if marshal methods are disabled. + /// public MarshalMethodsClassifier? Classifier { get; } - public List AllJavaTypes { get; } - public JavaStubsState (List allJavaTypes, MarshalMethodsClassifier? classifier) + /// + /// All the Java types discovered in the target architecture's assemblies. + /// + public List AllJavaTypes { get; } + + public JavaStubsState (AndroidTargetArch arch, List allJavaTypes, MarshalMethodsClassifier? classifier) { + TargetArch = arch; AllJavaTypes = allJavaTypes; Classifier = classifier; } From ecfbee88a978b6a08234a7a31f4350212886829a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 4 Dec 2023 16:55:34 +0100 Subject: [PATCH 035/143] Bump LLVM version to 17.0.6 Changes: https://discourse.llvm.org/t/llvm-17-0-2-released/73840 Changes: https://discourse.llvm.org/t/llvm-17-0-3-released/74172 Changes: https://discourse.llvm.org/t/llvm-17-0-4-released/74548 Changes: https://discourse.llvm.org/t/llvm-17-0-5-released/74906 Changes: https://discourse.llvm.org/t/llvm-17-0-6-released/75281 LLVM 17.0.6 has just been released. For full set of changes in this release please see the links above. There are no changes in these releases that are particularly relevant to Xamarin.Android. This is merely a sync release so that we continue tracking the latest LLVM stable release in our toolchain. --- build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs index 4fe4f25c879..2d7289696c2 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.cs @@ -15,7 +15,7 @@ namespace Xamarin.Android.Prepare // partial class Configurables { - const string BinutilsVersion = "L_17.0.1-7.0.0"; + const string BinutilsVersion = "L_17.0.6-7.1.0"; const string MicrosoftOpenJDK17Version = "17.0.8"; const string MicrosoftOpenJDK17Release = "17.0.8.7"; From 38b0f6418ebc5e9f7ebae6503dae474462c5c86a Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 4 Dec 2023 21:21:18 +0100 Subject: [PATCH 036/143] Trudging on --- .../Tasks/GenerateJavaStubs.cs | 44 +++-- .../Utilities/JCWGenerator.cs | 170 +++++++++++++++--- .../MarshalMethodsAssemblyRewriter.cs | 5 + 3 files changed, 179 insertions(+), 40 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 8215b8d480b..0858d39b336 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -223,10 +223,23 @@ void Run (bool useMarshalMethods) } javaStubStates.Add (arch, state); -// RewriteMarshalMethods (mirrorHelperState); } - JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, javaStubStates); + + if (useMarshalMethods) { + // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed + // in order to properly generate wrapper methods in the marshal methods assembly rewriter. + // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. + var environmentParser = new EnvironmentFilesParser (); + bool brokenExceptionTransitionsEnabled = environmentParser.AreBrokenExceptionTransitionsEnabled (Environments); + + foreach (var kvp in javaStubStates) { + RewriteMarshalMethods (kvp.Value, brokenExceptionTransitionsEnabled); + } + } + + // TODO: manifest + // TODO: typemaps } (bool success, JavaStubsState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) @@ -273,25 +286,18 @@ void Run (bool useMarshalMethods) return (allJavaTypes, javaTypesForJCW); } - // void RewriteMarshalMethods (MarshalMethodsMirrorHelperState? mirrorHelperState) - // { - // if (mirrorHelperState == null) { - // return; - // } - - // var mirrorHelper = new MarshalMethodsMirrorHelper (mirrorHelperState, Log); - // ArchitectureMarshalMethods perArchMarshalMethods = mirrorHelper.Reflect (); - - // // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed - // // in order to properly generate wrapper methods in the marshal methods assembly rewriter. - // // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. - // var environmentParser = new EnvironmentFilesParser (); + void RewriteMarshalMethods (JavaStubsState state, bool brokenExceptionTransitionsEnabled) + { + if (state.Classifier == null) { + return; + } - // //Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); + var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier); + //Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - // // var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - // // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); - // } + // var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); + // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + } void Run (XAAssemblyResolver res, bool useMarshalMethods) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs index ca92442cb05..63e168a061b 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -218,32 +218,23 @@ public static void EnsureAllArchitecturesAreIdentical (TaskLoggingHelper logger, } // An expensive process, but we must be sure that all the architectures have the same data - JavaStubsState? templateStub = null; + JavaStubsState? templateState = null; foreach (var kvp in javaStubStates) { JavaStubsState state = kvp.Value; - if (templateStub == null) { - templateStub = state; + if (templateState == null) { + templateState = state; continue; } - EnsureIdenticalCollections (logger, templateStub, state); + EnsureIdenticalCollections (logger, templateState, state); + EnsureClassifiersMatch (logger, templateState, state); } } static void EnsureIdenticalCollections (TaskLoggingHelper logger, JavaStubsState templateState, JavaStubsState state) { - if (templateState.Classifier == null) { - if (state.Classifier != null) { - throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES NOT have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); - } - } else { - if (state.Classifier == null) { - throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); - } - - EnsureClassifiersMatch (templateState, state); - } + logger.LogDebugMessage ($"Ensuring Java type collection in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); List templateTypes = templateState.AllJavaTypes; List types = state.AllJavaTypes; @@ -281,21 +272,158 @@ static void EnsureIdenticalCollections (TaskLoggingHelper logger, JavaStubsState } } - static void EnsureClassifiersMatch (JavaStubsState templateState, JavaStubsState state) + static bool CheckWhetherTypesMatch (TypeDefinition templateType, TypeDefinition type) { - MarshalMethodsClassifier templateClassifier = templateState.Classifier; - MarshalMethodsClassifier classifier = state.Classifier; + // TODO: should we compare individual methods, fields, properties? + return String.Compare (templateType.FullName, type.FullName, StringComparison.Ordinal) == 0; + } + + static void EnsureClassifiersMatch (TaskLoggingHelper logger, JavaStubsState templateState, JavaStubsState state) + { + logger.LogDebugMessage ($"Ensuring marshal method classifier in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); + + MarshalMethodsClassifier? templateClassifier = templateState.Classifier; + MarshalMethodsClassifier? classifier = state.Classifier; + + if (templateClassifier == null) { + if (classifier != null) { + throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES NOT have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); + } + return; + } + + if (classifier == null) { + throw new InvalidOperationException ($"Internal error: architecture '{templateState.TargetArch}' DOES have a marshal methods classifier, unlike architecture '{state.TargetArch}'"); + } if (templateClassifier.MarshalMethods.Count != classifier.MarshalMethods.Count) { throw new InvalidOperationException ( $"Internal error: classifier for template architecture '{templateState.TargetArch}' contains {templateClassifier.MarshalMethods.Count} marshal methods, but the one for architecture '{state.TargetArch}' has {classifier.MarshalMethods.Count}" ); } + + var matchedTemplateMethods = new HashSet (); + var mismatchedMethods = new List (); + bool foundMismatches = false; + + foreach (var kvp in classifier.MarshalMethods) { + string key = kvp.Key; + IList methods = kvp.Value; + + logger.LogDebugMessage ($"Comparing marshal method '{key}' in architecture '{templateState.TargetArch}', with {methods.Count} overloads, against architecture '{state.TargetArch}'"); + + if (!templateClassifier.MarshalMethods.TryGetValue (key, out IList templateMethods)) { + logger.LogDebugMessage ($"Architecture '{state.TargetArch}' has marshal method '{key}' which does not exist in architecture '{templateState.TargetArch}'"); + foundMismatches = true; + continue; + } + + if (methods.Count != templateMethods.Count) { + logger.LogDebugMessage ($"Architecture '{state.TargetArch}' has an incorrect number of marshal method '{key}' overloads. Expected {templateMethods.Count}, but found {methods.Count}"); + continue; + } + + foreach (MarshalMethodEntry templateMethod in templateMethods) { + MarshalMethodEntry? match = null; + + foreach (MarshalMethodEntry method in methods) { + if (CheckWhetherMethodsMatch (logger, templateMethod, templateState.TargetArch, method, state.TargetArch)) { + match = method; + break; + } + } + + if (match == null) { + foundMismatches = true; + } + } + } + + if (!foundMismatches) { + return; + } + + logger.LogError ($"Architecture '{state.TargetArch}' doesn't match all marshal methods in architecture '{templateState.TargetArch}'. Please see detailed MSBuild logs for more information."); } - static bool CheckWhetherTypesMatch (TypeDefinition templateType, TypeDefinition type) + static bool CheckWhetherMethodsMatch (TaskLoggingHelper logger, MarshalMethodEntry templateMethod, AndroidTargetArch templateArch, MarshalMethodEntry method, AndroidTargetArch arch) { - // TODO: should we compare individual methods, fields, properties? - return String.Compare (templateType.FullName, type.FullName, StringComparison.Ordinal) == 0; + bool success = true; + string methodName = templateMethod.NativeCallback.FullName; + + if (!CheckWhetherTypesMatch (templateMethod.DeclaringType, method.DeclaringType)) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' should be declared in type '{templateMethod.DeclaringType.FullName}', but instead was declared in '{method.DeclaringType.FullName}'"); + success = false; + } + + bool skipJniCheck = false; + if (!CheckWhetherMembersMatch (logger, methodName, "native callback", templateMethod.NativeCallback, templateArch, method.NativeCallback, arch)) { + success = false; + + // This takes care of overloads for the same methods, and avoids false negatives below + skipJniCheck = true; + } + + if (!skipJniCheck) { + if (String.Compare (templateMethod.JniMethodName, method.JniMethodName, StringComparison.Ordinal) != 0) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' has a different JNI method name than architecture '{templateArch}':"); + logger.LogDebugMessage ($" Expected: '{templateMethod.JniMethodName}', found: '{method.JniMethodName}'"); + success = false; + } + + if (String.Compare (templateMethod.JniMethodSignature, method.JniMethodSignature, StringComparison.Ordinal) != 0) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' has a different JNI method signature than architecture '{templateArch}':"); + logger.LogDebugMessage ($" Expected: '{templateMethod.JniMethodSignature}', found: '{method.JniMethodSignature}'"); + success = false; + } + + if (String.Compare (templateMethod.JniTypeName, method.JniTypeName, StringComparison.Ordinal) != 0) { + logger.LogDebugMessage ($"Marshal method '{methodName}' for architecture '{arch}' has a different JNI type name than architecture '{templateArch}':"); + logger.LogDebugMessage ($" Expected: '{templateMethod.JniTypeName}', found: '{method.JniTypeName}'"); + success = false; + } + } + + if (templateMethod.IsSpecial) { + // Other method definitions will be `null`, so we can skip them + if (method.IsSpecial) { + return success; + } + + logger.LogDebugMessage ($"Marshal method '{templateMethod.NativeCallback.FullName}' is marked as special in architecture '{templateArch}', but not in architecture '{arch}'"); + return false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "connector", templateMethod.Connector, templateArch, method.Connector, arch)) { + success = false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "implemented", templateMethod.ImplementedMethod, templateArch, method.ImplementedMethod, arch)) { + success = false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "registered", templateMethod.RegisteredMethod, templateArch, method.RegisteredMethod, arch)) { + success = false; + } + + if (!CheckWhetherMembersMatch (logger, methodName, "callback backing field", templateMethod.CallbackField, templateArch, method.CallbackField, arch)) { + success = false; + } + + return success; + } + + static bool CheckWhetherMembersMatch (TaskLoggingHelper logger, string marshalMethodName, string description, MemberReference? templateMethod, AndroidTargetArch templateArch, MemberReference? method, AndroidTargetArch arch) + { + if (templateMethod == null) { + if (method == null) { + return true; + } + + logger.LogDebugMessage ($"Marshal method '{marshalMethodName}' component '{description}' is null in architecture '{templateArch}', but not null in architecture '{arch}'"); + return false; + } + + return String.Compare (templateMethod.FullName, method.FullName, StringComparison.Ordinal) == 0; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 96063bf2126..2f3f45a065e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -7,6 +7,7 @@ using Microsoft.Build.Utilities; using Mono.Cecil; using Mono.Cecil.Cil; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -26,6 +27,10 @@ sealed class AssemblyImports IDictionary assemblyPaths; TaskLoggingHelper log; + public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch targetArch, MarshalMethodsClassifier classifier) + { + } + public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) { this.assemblyPaths = assemblyPaths; From 54a48ebb7b42c498aa1446dc9b73ad44ea6a80a1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Dec 2023 14:42:53 +0100 Subject: [PATCH 037/143] Marshal methods classification and rewriting done --- .../Tasks/GenerateJavaStubs.cs | 90 +++--- .../Utilities/JCWGenerator.cs | 6 +- .../Utilities/JavaStubsState.cs | 5 +- .../MarshalMethodsAssemblyRewriter.cs | 282 ++++++++++++++---- .../Utilities/MarshalMethodsClassifier.cs | 43 ++- 5 files changed, 315 insertions(+), 111 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 0858d39b336..5a85b540ba0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -106,9 +106,9 @@ public override bool RunTask () // We're going to do 3 steps here instead of separate tasks so // we can share the list of JLO TypeDefinitions between them - using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { - Run (res, useMarshalMethods); - } + // using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { + // Run (res, useMarshalMethods); + // } } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); if (MonoAndroidHelper.LogInternalExceptions) @@ -234,12 +234,26 @@ void Run (bool useMarshalMethods) bool brokenExceptionTransitionsEnabled = environmentParser.AreBrokenExceptionTransitionsEnabled (Environments); foreach (var kvp in javaStubStates) { - RewriteMarshalMethods (kvp.Value, brokenExceptionTransitionsEnabled); + JavaStubsState state = kvp.Value; + RewriteMarshalMethods (state, brokenExceptionTransitionsEnabled); + state.Classifier.AddSpecialCaseMethods (); + + Log.LogDebugMessage ($"[{state.TargetArch}] Number of generated marshal methods: {state.Classifier.MarshalMethods.Count}"); + if (state.Classifier.RejectedMethodCount > 0) { + Log.LogWarning ($"[{state.TargetArch}] Number of methods in the project that will be registered dynamically: {state.Classifier.RejectedMethodCount}"); + } + + if (state.Classifier.WrappedMethodCount > 0) { + // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers + Log.LogDebugMessage ($"[{state.TargetArch}] Number of methods in the project that need marshal method wrappers: {state.Classifier.WrappedMethodCount}"); + } } } - // TODO: manifest // TODO: typemaps + // TODO: ACW maps + // TODO: manifest + // TODO: additional java sources } (bool success, JavaStubsState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) @@ -262,7 +276,7 @@ void Run (bool useMarshalMethods) return (false, null); } - return (true, new JavaStubsState (arch, allJavaTypes, jcwGenerator.Classifier)); + return (true, new JavaStubsState (arch, resolver, allJavaTypes, jcwGenerator.Classifier)); } (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) @@ -292,11 +306,8 @@ void RewriteMarshalMethods (JavaStubsState state, bool brokenExceptionTransition return; } - var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier); - //Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - - // var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - // rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); + var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier, state.Resolver); + rewriter.Rewrite (brokenExceptionTransitionsEnabled); } void Run (XAAssemblyResolver res, bool useMarshalMethods) @@ -406,31 +417,24 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } MarshalMethodsClassifier classifier = null; - if (useMarshalMethods) { - classifier = new MarshalMethodsClassifier (cache, res, Log); - } + // if (useMarshalMethods) { + // classifier = new MarshalMethodsClassifier (cache, res, Log); + // } // Step 2 - Generate Java stub code - var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - if (!success) - return; + bool success = true; + // var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); + // if (!success) + // return; if (useMarshalMethods) { - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods:", classifier.MarshalMethods); - // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed // in order to properly generate wrapper methods in the marshal methods assembly rewriter. // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. var environmentParser = new EnvironmentFilesParser (); - - Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); - - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after AddMethodsFromAbiSpecificAssemblies:", classifier.MarshalMethods); - - var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); - rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); - - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (from classifier.MarshalMethods)", classifier.MarshalMethods); +// Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); +// var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); +// rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); } // Step 3 - Generate type maps @@ -649,9 +653,9 @@ AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = nul bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) { - if (useMarshalMethods && classifier == null) { - throw new ArgumentNullException (nameof (classifier)); - } + // if (useMarshalMethods && classifier == null) { + // throw new ArgumentNullException (nameof (classifier)); + // } string outputPath = Path.Combine (OutputDirectory, "src"); string monoInit = GetMonoInitSource (AndroidSdkPlatform); @@ -675,11 +679,11 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache }; jti.Generate (writer); - if (useMarshalMethods) { - if (classifier.FoundDynamicallyRegisteredMethods (t)) { - Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName (cache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); - } - } + // if (useMarshalMethods) { + // if (classifier.FoundDynamicallyRegisteredMethods (t)) { + // Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName (cache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); + // } + // } writer.Flush (); var path = jti.GetDestinationPath (outputPath); @@ -714,13 +718,13 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache } } - if (useMarshalMethods) { - var state = new MarshalMethodsState (classifier.MarshalMethods); - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (CreateJavaSources/state)", state.MarshalMethods); - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), state, RegisteredTaskObjectLifetime.Build); - var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after state registered and retrieved (CreateJavaSources/state)", state.MarshalMethods); - } + // if (useMarshalMethods) { + // var state = new MarshalMethodsState (classifier.MarshalMethods); + // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (CreateJavaSources/state)", state.MarshalMethods); + // BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), state, RegisteredTaskObjectLifetime.Build); + // var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); + // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after state registered and retrieved (CreateJavaSources/state)", state.MarshalMethods); + // } return ok; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs index 63e168a061b..a14fe4352d1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -61,7 +61,7 @@ public bool Classify (string androidSdkPlatform) return true; } - Classifier = new MarshalMethodsClassifier (context.TypeDefinitionCache, context.Resolver, log); + Classifier = MakeClassifier (); return ProcessTypes ( generateCode: false, androidSdkPlatform, @@ -74,7 +74,7 @@ public bool Classify (string androidSdkPlatform) public bool GenerateAndClassify (string androidSdkPlatform, string outputPath, string applicationJavaClass) { if (context.UseMarshalMethods) { - Classifier = new MarshalMethodsClassifier (context.TypeDefinitionCache, context.Resolver, log); + Classifier = MakeClassifier (); } return ProcessTypes ( @@ -86,6 +86,8 @@ public bool GenerateAndClassify (string androidSdkPlatform, string outputPath, s ); } + MarshalMethodsClassifier MakeClassifier () => new MarshalMethodsClassifier (context.Arch, context.TypeDefinitionCache, context.Resolver, log); + bool ProcessTypes (bool generateCode, string androidSdkPlatform, MarshalMethodsClassifier? classifier, string? outputPath, string? applicationJavaClass) { if (generateCode && String.IsNullOrEmpty (outputPath)) { diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs index 4fc21527b68..d538db6662a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs @@ -26,9 +26,12 @@ class JavaStubsState /// public List AllJavaTypes { get; } - public JavaStubsState (AndroidTargetArch arch, List allJavaTypes, MarshalMethodsClassifier? classifier) + public XAAssemblyResolverNew Resolver { get; } + + public JavaStubsState (AndroidTargetArch arch, XAAssemblyResolverNew resolver, List allJavaTypes, MarshalMethodsClassifier? classifier) { TargetArch = arch; + Resolver = resolver; AllJavaTypes = allJavaTypes; Classifier = classifier; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 2f3f45a065e..397d056da7e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -22,33 +22,38 @@ sealed class AssemblyImports public MethodReference WaitForBridgeProcessingMethod; } - IDictionary> methods; - ICollection uniqueAssemblies; - IDictionary assemblyPaths; - TaskLoggingHelper log; - - public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch targetArch, MarshalMethodsClassifier classifier) + // IDictionary> methods; + // ICollection uniqueAssemblies; + // IDictionary assemblyPaths; + readonly TaskLoggingHelper log; + readonly MarshalMethodsClassifier classifier; + readonly XAAssemblyResolverNew resolver; + readonly AndroidTargetArch targetArch; + + public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch targetArch, MarshalMethodsClassifier classifier, XAAssemblyResolverNew resolver) { - } - - public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) - { - this.assemblyPaths = assemblyPaths; - this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); - this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.targetArch = targetArch; + this.classifier = classifier ?? throw new ArgumentNullException (nameof (classifier));; + this.resolver = resolver ?? throw new ArgumentNullException (nameof (resolver));; } + // public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) + // { + // this.assemblyPaths = assemblyPaths; + // this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); + // this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); + // this.log = log ?? throw new ArgumentNullException (nameof (log)); + // } + // TODO: do away with broken exception transitions, there's no point in supporting them - public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransitions) + public void Rewrite (bool brokenExceptionTransitions) { - if (resolver == null) { - throw new ArgumentNullException (nameof (resolver)); - } + Console.WriteLine ("[new] Rewriting assemblies"); AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); if (monoAndroidRuntime == null) { - throw new InvalidOperationException ($"Internal error: unable to load the Mono.Android.Runtime assembly"); + throw new InvalidOperationException ($"[{targetArch}] Internal error: unable to load the Mono.Android.Runtime assembly"); } TypeDefinition runtime = FindType (monoAndroidRuntime, "Android.Runtime.AndroidRuntimeInternal", required: true)!; @@ -64,8 +69,11 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition TypeDefinition systemException = FindType (corlib, "System.Exception", required: true); MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver); + var assemblyImports = new Dictionary (); - foreach (AssemblyDefinition asm in uniqueAssemblies) { + Console.WriteLine (" populating assembly imports dict"); + foreach (AssemblyDefinition asm in classifier.Assemblies) { + Console.WriteLine ($" assembly path: {asm.MainModule.FileName}"); var imports = new AssemblyImports { MonoUnhandledExceptionMethod = asm.MainModule.ImportReference (monoUnhandledExceptionMethod), SystemException = asm.MainModule.ImportReference (systemException), @@ -77,10 +85,10 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition assemblyImports.Add (asm, imports); } - log.LogDebugMessage ("Rewriting assemblies for marshal methods support"); + log.LogDebugMessage ($"[{targetArch}] Rewriting assemblies for marshal methods support"); var processedMethods = new Dictionary (StringComparer.Ordinal); - foreach (IList methodList in methods.Values) { + foreach (IList methodList in classifier.MarshalMethods.Values) { foreach (MarshalMethodEntry method in methodList) { string fullNativeCallbackName = method.NativeCallback.FullName; if (processedMethods.TryGetValue (fullNativeCallbackName, out MethodDefinition nativeCallbackWrapper)) { @@ -91,19 +99,19 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition method.NativeCallbackWrapper = GenerateWrapper (method, assemblyImports, brokenExceptionTransitions); if (method.Connector != null) { if (method.Connector.IsStatic && method.Connector.IsPrivate) { - log.LogDebugMessage ($"Removing connector method {method.Connector.FullName}"); + log.LogDebugMessage ($"[{targetArch}] Removing connector method {method.Connector.FullName}"); method.Connector.DeclaringType?.Methods?.Remove (method.Connector); } else { - log.LogWarning ($"NOT removing connector method {method.Connector.FullName} because it's either not static or not private"); + log.LogWarning ($"[{targetArch}] NOT removing connector method {method.Connector.FullName} because it's either not static or not private"); } } if (method.CallbackField != null) { if (method.CallbackField.IsStatic && method.CallbackField.IsPrivate) { - log.LogDebugMessage ($"Removing callback delegate backing field {method.CallbackField.FullName}"); + log.LogDebugMessage ($"[{targetArch}] Removing callback delegate backing field {method.CallbackField.FullName}"); method.CallbackField.DeclaringType?.Fields?.Remove (method.CallbackField); } else { - log.LogWarning ($"NOT removing callback field {method.CallbackField.FullName} because it's either not static or not private"); + log.LogWarning ($"[{targetArch}] NOT removing callback field {method.CallbackField.FullName} because it's either not static or not private"); } } @@ -111,8 +119,12 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition } } - foreach (AssemblyDefinition asm in uniqueAssemblies) { - string path = GetAssemblyPath (asm); + foreach (AssemblyDefinition asm in classifier.Assemblies) { + string? path = asm.MainModule.FileName; + if (String.IsNullOrEmpty (path)) { + throw new InvalidOperationException ($"[{targetArch}] Internal error: assembly '{asm}' does not specify path to its file"); + } + string pathPdb = Path.ChangeExtension (path, ".pdb"); bool havePdb = File.Exists (pathPdb); @@ -123,7 +135,7 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition string directory = Path.Combine (Path.GetDirectoryName (path), "new"); Directory.CreateDirectory (directory); string output = Path.Combine (directory, Path.GetFileName (path)); - log.LogDebugMessage ($"Writing new version of '{path}' assembly: {output}"); + log.LogDebugMessage ($"[{targetArch}] Writing new version of '{path}' assembly: {output}"); // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated // since Cecil doesn't update the MVID in the already loaded types @@ -144,7 +156,7 @@ public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransition void CopyFile (string source, string target) { - log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); + log.LogDebugMessage ($"[{targetArch}] Copying rewritten assembly: {source} -> {target}"); string targetBackup = $"{target}.bak"; if (File.Exists (target)) { @@ -159,8 +171,8 @@ void CopyFile (string source, string target) File.Delete (targetBackup); } catch (Exception ex) { // On Windows the deletion may fail, depending on lock state of the original `target` file before the move. - log.LogDebugMessage ($"While trying to delete '{targetBackup}', exception was thrown: {ex}"); - log.LogDebugMessage ($"Failed to delete backup file '{targetBackup}', ignoring."); + log.LogDebugMessage ($"[{targetArch}] While trying to delete '{targetBackup}', exception was thrown: {ex}"); + log.LogDebugMessage ($"[{targetArch}] Failed to delete backup file '{targetBackup}', ignoring."); } } } @@ -172,15 +184,157 @@ void RemoveFile (string? path) } try { - log.LogDebugMessage ($"Deleting: {path}"); + log.LogDebugMessage ($"[{targetArch}] Deleting: {path}"); File.Delete (path); } catch (Exception ex) { - log.LogWarning ($"Unable to delete source file '{path}'"); - log.LogDebugMessage (ex.ToString ()); + log.LogWarning ($"[{targetArch}] Unable to delete source file '{path}'"); + log.LogDebugMessage ($"[{targetArch}] {ex.ToString ()}"); } } } + // TODO: do away with broken exception transitions, there's no point in supporting them + // public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransitions) + // { + // if (resolver == null) { + // throw new ArgumentNullException (nameof (resolver)); + // } + + // AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); + // if (monoAndroidRuntime == null) { + // throw new InvalidOperationException ($"Internal error: unable to load the Mono.Android.Runtime assembly"); + // } + + // TypeDefinition runtime = FindType (monoAndroidRuntime, "Android.Runtime.AndroidRuntimeInternal", required: true)!; + // MethodDefinition waitForBridgeProcessingMethod = FindMethod (runtime, "WaitForBridgeProcessing", required: true)!; + + // TypeDefinition androidEnvironment = FindType (monoAndroidRuntime, "Android.Runtime.AndroidEnvironmentInternal", required: true)!; + // MethodDefinition unhandledExceptionMethod = FindMethod (androidEnvironment, "UnhandledException", required: true)!; + + // TypeDefinition runtimeNativeMethods = FindType (monoAndroidRuntime, "Android.Runtime.RuntimeNativeMethods", required: true); + // MethodDefinition monoUnhandledExceptionMethod = FindMethod (runtimeNativeMethods, "monodroid_debugger_unhandled_exception", required: true); + + // AssemblyDefinition corlib = resolver.Resolve ("System.Private.CoreLib"); + // TypeDefinition systemException = FindType (corlib, "System.Exception", required: true); + + // MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver); + // var assemblyImports = new Dictionary (); + // foreach (AssemblyDefinition asm in uniqueAssemblies) { + // var imports = new AssemblyImports { + // MonoUnhandledExceptionMethod = asm.MainModule.ImportReference (monoUnhandledExceptionMethod), + // SystemException = asm.MainModule.ImportReference (systemException), + // UnhandledExceptionMethod = asm.MainModule.ImportReference (unhandledExceptionMethod), + // UnmanagedCallersOnlyAttribute = CreateImportedUnmanagedCallersOnlyAttribute (asm, unmanagedCallersOnlyAttributeCtor), + // WaitForBridgeProcessingMethod = asm.MainModule.ImportReference (waitForBridgeProcessingMethod), + // }; + + // assemblyImports.Add (asm, imports); + // } + + // log.LogDebugMessage ("Rewriting assemblies for marshal methods support"); + + // var processedMethods = new Dictionary (StringComparer.Ordinal); + // foreach (IList methodList in methods.Values) { + // foreach (MarshalMethodEntry method in methodList) { + // string fullNativeCallbackName = method.NativeCallback.FullName; + // if (processedMethods.TryGetValue (fullNativeCallbackName, out MethodDefinition nativeCallbackWrapper)) { + // method.NativeCallbackWrapper = nativeCallbackWrapper; + // continue; + // } + + // method.NativeCallbackWrapper = GenerateWrapper (method, assemblyImports, brokenExceptionTransitions); + // if (method.Connector != null) { + // if (method.Connector.IsStatic && method.Connector.IsPrivate) { + // log.LogDebugMessage ($"Removing connector method {method.Connector.FullName}"); + // method.Connector.DeclaringType?.Methods?.Remove (method.Connector); + // } else { + // log.LogWarning ($"NOT removing connector method {method.Connector.FullName} because it's either not static or not private"); + // } + // } + + // if (method.CallbackField != null) { + // if (method.CallbackField.IsStatic && method.CallbackField.IsPrivate) { + // log.LogDebugMessage ($"Removing callback delegate backing field {method.CallbackField.FullName}"); + // method.CallbackField.DeclaringType?.Fields?.Remove (method.CallbackField); + // } else { + // log.LogWarning ($"NOT removing callback field {method.CallbackField.FullName} because it's either not static or not private"); + // } + // } + + // processedMethods.Add (fullNativeCallbackName, method.NativeCallback); + // } + // } + + // foreach (AssemblyDefinition asm in uniqueAssemblies) { + // string path = GetAssemblyPath (asm); + // string pathPdb = Path.ChangeExtension (path, ".pdb"); + // bool havePdb = File.Exists (pathPdb); + + // var writerParams = new WriterParameters { + // WriteSymbols = havePdb, + // }; + + // string directory = Path.Combine (Path.GetDirectoryName (path), "new"); + // Directory.CreateDirectory (directory); + // string output = Path.Combine (directory, Path.GetFileName (path)); + // log.LogDebugMessage ($"Writing new version of '{path}' assembly: {output}"); + + // // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated + // // since Cecil doesn't update the MVID in the already loaded types + // //asm.MainModule.Mvid = Guid.NewGuid (); + // asm.Write (output, writerParams); + + // CopyFile (output, path); + // RemoveFile (output); + + // if (havePdb) { + // string outputPdb = Path.ChangeExtension (output, ".pdb"); + // if (File.Exists (outputPdb)) { + // CopyFile (outputPdb, pathPdb); + // } + // RemoveFile (outputPdb); + // } + // } + + // void CopyFile (string source, string target) + // { + // log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); + + // string targetBackup = $"{target}.bak"; + // if (File.Exists (target)) { + // // Try to avoid sharing violations by first renaming the target + // File.Move (target, targetBackup); + // } + + // File.Copy (source, target, true); + + // if (File.Exists (targetBackup)) { + // try { + // File.Delete (targetBackup); + // } catch (Exception ex) { + // // On Windows the deletion may fail, depending on lock state of the original `target` file before the move. + // log.LogDebugMessage ($"While trying to delete '{targetBackup}', exception was thrown: {ex}"); + // log.LogDebugMessage ($"Failed to delete backup file '{targetBackup}', ignoring."); + // } + // } + // } + + // void RemoveFile (string? path) + // { + // if (String.IsNullOrEmpty (path) || !File.Exists (path)) { + // return; + // } + + // try { + // log.LogDebugMessage ($"Deleting: {path}"); + // File.Delete (path); + // } catch (Exception ex) { + // log.LogWarning ($"Unable to delete source file '{path}'"); + // log.LogDebugMessage (ex.ToString ()); + // } + // } + // } + MethodDefinition GenerateWrapper (MarshalMethodEntry method, Dictionary assemblyImports, bool brokenExceptionTransitions) { MethodDefinition callback = method.NativeCallback; @@ -328,7 +482,7 @@ bool IsBooleanConversion (TypeReference sourceType, TypeReference targetType) { if (String.Compare ("System.Boolean", sourceType.FullName, StringComparison.Ordinal) == 0) { if (String.Compare ("System.Byte", targetType.FullName, StringComparison.Ordinal) != 0) { - throw new InvalidOperationException ($"Unexpected conversion from '{sourceType.FullName}' to '{targetType.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unexpected conversion from '{sourceType.FullName}' to '{targetType.FullName}'"); } return true; @@ -339,7 +493,7 @@ bool IsBooleanConversion (TypeReference sourceType, TypeReference targetType) void ThrowUnsupportedType (TypeReference type) { - throw new InvalidOperationException ($"Unsupported non-blittable type '{type.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unsupported non-blittable type '{type.FullName}'"); } } @@ -389,7 +543,7 @@ void AddSetDefaultValueInstructions (MethodBody body, TypeReference type, Variab return; } - throw new InvalidOperationException ($"Unsupported type: '{type.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unsupported type: '{type.FullName}'"); } @@ -441,31 +595,31 @@ TypeReference MapToBlittableTypeIfNecessary (TypeReference type, out bool typeMa return ReturnValid (typeof(byte)); } - throw new NotSupportedException ($"Cannot map unsupported blittable type '{type.FullName}'"); + throw new NotSupportedException ($"[{targetArch}] Cannot map unsupported blittable type '{type.FullName}'"); TypeReference ReturnValid (Type typeToLookUp) { TypeReference? mappedType = type.Module.Assembly.MainModule.ImportReference (typeToLookUp); if (mappedType == null) { - throw new InvalidOperationException ($"Unable to obtain reference to type '{typeToLookUp.FullName}'"); + throw new InvalidOperationException ($"[{targetArch}] Unable to obtain reference to type '{typeToLookUp.FullName}'"); } return mappedType; } } - string GetAssemblyPath (AssemblyDefinition asm) - { - string filePath = asm.MainModule.FileName; - if (!String.IsNullOrEmpty (filePath)) { - return filePath; - } + // string GetAssemblyPath (AssemblyDefinition asm) + // { + // string filePath = asm.MainModule.FileName; + // if (!String.IsNullOrEmpty (filePath)) { + // return filePath; + // } - // No checking on purpose - the assembly **must** be there if its MainModule.FileName property returns a null or empty string - return assemblyPaths[asm]; - } + // // No checking on purpose - the assembly **must** be there if its MainModule.FileName property returns a null or empty string + // return assemblyPaths[asm]; + // } - MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver resolver) + MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (IAssemblyResolver resolver) { AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices"); TypeDefinition unmanagedCallersOnlyAttribute = null; @@ -485,7 +639,7 @@ MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver } if (unmanagedCallersOnlyAttribute == null) { - throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type"); + throw new InvalidOperationException ("[{targetArch}] Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type"); } foreach (MethodDefinition md in unmanagedCallersOnlyAttribute.Methods) { @@ -496,7 +650,7 @@ MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (XAAssemblyResolver return md; } - throw new InvalidOperationException ("Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor"); + throw new InvalidOperationException ("[{targetArch}] Unable to find the System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute type constructor"); } CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition targetAssembly, MethodDefinition unmanagedCallersOnlyAtributeCtor) @@ -506,17 +660,17 @@ CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition MethodDefinition? FindMethod (TypeDefinition type, string methodName, bool required) { - log.LogDebugMessage ($"Looking for method '{methodName}' in type {type}"); + log.LogDebugMessage ($"[{targetArch}] Looking for method '{methodName}' in type {type}"); foreach (MethodDefinition method in type.Methods) { - log.LogDebugMessage ($" method: {method.Name}"); + log.LogDebugMessage ($"[{targetArch}] method: {method.Name}"); if (String.Compare (methodName, method.Name, StringComparison.Ordinal) == 0) { - log.LogDebugMessage (" match!"); + log.LogDebugMessage ($"[{targetArch}] match!"); return method; } } if (required) { - throw new InvalidOperationException ($"Internal error: required method '{methodName}' in type {type} not found"); + throw new InvalidOperationException ($"[{targetArch}] Internal error: required method '{methodName}' in type {type} not found"); } return null; @@ -524,20 +678,30 @@ CustomAttribute CreateImportedUnmanagedCallersOnlyAttribute (AssemblyDefinition TypeDefinition? FindType (AssemblyDefinition asm, string typeName, bool required) { - log.LogDebugMessage ($"Looking for type '{typeName}' in assembly '{asm}'"); + log.LogDebugMessage ($"[{targetArch}] Looking for type '{typeName}' in assembly '{asm}' ({GetAssemblyPathInfo (asm)})"); foreach (TypeDefinition t in asm.MainModule.Types) { - log.LogDebugMessage ($" checking {t.FullName}"); + log.LogDebugMessage ($"[{targetArch}] checking {t.FullName}"); if (String.Compare (typeName, t.FullName, StringComparison.Ordinal) == 0) { - log.LogDebugMessage ($" match!"); + log.LogDebugMessage ($"[{targetArch}] match!"); return t; } } if (required) { - throw new InvalidOperationException ($"Internal error: required type '{typeName}' in assembly {asm} not found"); + throw new InvalidOperationException ($"[{targetArch}] Internal error: required type '{typeName}' in assembly {asm} not found"); } return null; } + + static string GetAssemblyPathInfo (AssemblyDefinition asm) + { + string? path = asm.MainModule.FileName; + if (String.IsNullOrEmpty (path)) { + return "no assembly path"; + } + + return path; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs index 77144121d01..922134a70d1 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsClassifier.cs @@ -9,6 +9,7 @@ using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; using Mono.Cecil; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -238,6 +239,7 @@ public bool Matches (MethodDefinition method) HashSet typesWithDynamicallyRegisteredMethods; ulong rejectedMethodCount = 0; ulong wrappedMethodCount = 0; + readonly AndroidTargetArch targetArch; public IDictionary> MarshalMethods => marshalMethods; public ICollection Assemblies => assemblies; @@ -245,6 +247,17 @@ public bool Matches (MethodDefinition method) public ulong WrappedMethodCount => wrappedMethodCount; public TypeDefinitionCache TypeDefinitionCache => tdCache; + public MarshalMethodsClassifier (AndroidTargetArch targetArch, TypeDefinitionCache tdCache, IAssemblyResolver res, TaskLoggingHelper log) + { + this.targetArch = targetArch; + this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.tdCache = tdCache ?? throw new ArgumentNullException (nameof (tdCache)); + resolver = res ?? throw new ArgumentNullException (nameof (tdCache)); + marshalMethods = new Dictionary> (StringComparer.Ordinal); + assemblies = new HashSet (); + typesWithDynamicallyRegisteredMethods = new HashSet (); + } + public MarshalMethodsClassifier (TypeDefinitionCache tdCache, IAssemblyResolver res, TaskLoggingHelper log) { this.log = log ?? throw new ArgumentNullException (nameof (log)); @@ -410,6 +423,24 @@ public void AddSpecialCaseMethods () AddTypeManagerSpecialCaseMethods (); } + string GetAssemblyPathInfo (FieldDefinition? field) => GetAssemblyPathInfo (field?.DeclaringType); + string GetAssemblyPathInfo (MethodDefinition? method) => GetAssemblyPathInfo (method?.DeclaringType); + string GetAssemblyPathInfo (TypeDefinition? type) => GetAssemblyPathInfo (type?.Module?.Assembly); + + string GetAssemblyPathInfo (AssemblyDefinition? asmdef) + { + if (asmdef == null) { + return "[assembly definition missing]"; + } + + string? path = asmdef.MainModule.FileName; + if (String.IsNullOrEmpty (path)) { + path = "unknown"; + } + + return $"[Arch: {targetArch}; Assembly: {path}]"; + } + bool IsDynamicallyRegistered (TypeDefinition topType, MethodDefinition registeredMethod, MethodDefinition implementedMethod, CustomAttribute registerAttribute) { if (registerAttribute.ConstructorArguments.Count != 3) { @@ -423,7 +454,7 @@ bool IsDynamicallyRegistered (TypeDefinition topType, MethodDefinition registere return false; } - log.LogWarning ($"Method '{registeredMethod.FullName}' will be registered dynamically"); + log.LogWarning ($"Method '{registeredMethod.FullName}' will be registered dynamically {GetAssemblyPathInfo (registeredMethod)}"); rejectedMethodCount++; return true; } @@ -437,7 +468,7 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD if (connectorName.Length < HandlerNameStart.Length + HandlerNameEnd.Length + 1 || !connectorName.StartsWith (HandlerNameStart, StringComparison.Ordinal) || !connectorName.EndsWith (HandlerNameEnd, StringComparison.Ordinal)) { - log.LogWarning ($"\tConnector name '{connectorName}' must start with '{HandlerNameStart}', end with '{HandlerNameEnd}' and have at least one character between the two parts."); + log.LogWarning ($"Connector name '{connectorName}' must start with '{HandlerNameStart}', end with '{HandlerNameEnd}' and have at least one character between the two parts."); return false; } @@ -452,19 +483,19 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD MethodDefinition connectorMethod = FindMethod (connectorDeclaringType, connectorName); if (connectorMethod == null) { - log.LogWarning ($"\tConnector method '{connectorName}' not found in type '{connectorDeclaringType.FullName}'"); + log.LogWarning ($"Connector method '{connectorName}' not found in type '{connectorDeclaringType.FullName}' {GetAssemblyPathInfo (connectorDeclaringType)}"); return false; } if (String.Compare ("System.Delegate", connectorMethod.ReturnType.FullName, StringComparison.Ordinal) != 0) { - log.LogWarning ($"\tConnector '{connectorName}' in type '{connectorDeclaringType.FullName}' has invalid return type, expected 'System.Delegate', found '{connectorMethod.ReturnType.FullName}'"); + log.LogWarning ($"Connector '{connectorName}' in type '{connectorDeclaringType.FullName}' has invalid return type, expected 'System.Delegate', found '{connectorMethod.ReturnType.FullName}' {GetAssemblyPathInfo (connectorDeclaringType)}"); return false; } var ncbs = new NativeCallbackSignature (registeredMethod, log); MethodDefinition nativeCallbackMethod = FindMethod (connectorDeclaringType, nativeCallbackName, ncbs); if (nativeCallbackMethod == null) { - log.LogWarning ($"\tUnable to find native callback method '{nativeCallbackName}' in type '{connectorDeclaringType.FullName}', matching the '{registeredMethod.FullName}' signature (jniName: '{jniName}')"); + log.LogWarning ($"Unable to find native callback method '{nativeCallbackName}' in type '{connectorDeclaringType.FullName}', matching the '{registeredMethod.FullName}' signature (jniName: '{jniName}') {GetAssemblyPathInfo (connectorDeclaringType)}"); return false; } @@ -477,7 +508,7 @@ bool IsStandardHandler (TypeDefinition topType, ConnectorInfo connector, MethodD FieldDefinition delegateField = FindField (nativeCallbackMethod.DeclaringType, delegateFieldName); if (delegateField != null) { if (String.Compare ("System.Delegate", delegateField.FieldType.FullName, StringComparison.Ordinal) != 0) { - log.LogWarning ($"\tdelegate field '{delegateFieldName}' in type '{nativeCallbackMethod.DeclaringType.FullName}' has invalid type, expected 'System.Delegate', found '{delegateField.FieldType.FullName}'"); + log.LogWarning ($"delegate field '{delegateFieldName}' in type '{nativeCallbackMethod.DeclaringType.FullName}' has invalid type, expected 'System.Delegate', found '{delegateField.FieldType.FullName}' {GetAssemblyPathInfo (delegateField)}"); return false; } } From a763218d1c66b5c1fe9248bfe080bcedd4ecbf00 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 5 Dec 2023 22:11:09 +0100 Subject: [PATCH 038/143] Typemap generation works --- .../Tasks/GenerateJavaStubs.cs | 50 +++-- .../Tasks/GeneratePackageManagerJava.cs | 20 +- .../Utilities/JCWGenerator.cs | 10 +- ...avaStubsState.cs => NativeCodeGenState.cs} | 16 +- .../Utilities/TypeMapGenerator.cs | 209 ++++++------------ 5 files changed, 118 insertions(+), 187 deletions(-) rename src/Xamarin.Android.Build.Tasks/Utilities/{JavaStubsState.cs => NativeCodeGenState.cs} (53%) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 5a85b540ba0..de16616c269 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -207,12 +207,12 @@ void Run (bool useMarshalMethods) } // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture - var javaStubStates = new Dictionary (); + var nativeCodeGenStates = new Dictionary (); bool generateJavaCode = true; foreach (var kvp in allAssembliesPerArch) { AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; - (bool success, JavaStubsState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods, generateJavaCode); + (bool success, NativeCodeGenState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods, generateJavaCode); if (!success) { return; @@ -222,9 +222,9 @@ void Run (bool useMarshalMethods) generateJavaCode = false; } - javaStubStates.Add (arch, state); + nativeCodeGenStates.Add (arch, state); } - JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, javaStubStates); + JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, nativeCodeGenStates); if (useMarshalMethods) { // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed @@ -233,8 +233,8 @@ void Run (bool useMarshalMethods) var environmentParser = new EnvironmentFilesParser (); bool brokenExceptionTransitionsEnabled = environmentParser.AreBrokenExceptionTransitionsEnabled (Environments); - foreach (var kvp in javaStubStates) { - JavaStubsState state = kvp.Value; + foreach (var kvp in nativeCodeGenStates) { + NativeCodeGenState state = kvp.Value; RewriteMarshalMethods (state, brokenExceptionTransitionsEnabled); state.Classifier.AddSpecialCaseMethods (); @@ -250,13 +250,25 @@ void Run (bool useMarshalMethods) } } - // TODO: typemaps + bool typemapsAreAbiAgnostic = Debug && !GenerateNativeAssembly; + bool first = true; + foreach (var kvp in nativeCodeGenStates) { + if (!first && typemapsAreAbiAgnostic) { + Log.LogDebugMessage ("Typemaps: it's a debug build and type maps are ABI-agnostic, not processing more ABIs"); + break; + } + + NativeCodeGenState state = kvp.Value; + first = false; + WriteTypeMappings (state); + } + // TODO: ACW maps // TODO: manifest // TODO: additional java sources } - (bool success, JavaStubsState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) + (bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) { XAAssemblyResolverNew resolver = MakeResolver (useMarshalMethods, arch, assemblies); var tdCache = new TypeDefinitionCache (); @@ -276,7 +288,7 @@ void Run (bool useMarshalMethods) return (false, null); } - return (true, new JavaStubsState (arch, resolver, allJavaTypes, jcwGenerator.Classifier)); + return (true, new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, jcwGenerator.Classifier)); } (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) @@ -300,7 +312,7 @@ void Run (bool useMarshalMethods) return (allJavaTypes, javaTypesForJCW); } - void RewriteMarshalMethods (JavaStubsState state, bool brokenExceptionTransitionsEnabled) + void RewriteMarshalMethods (NativeCodeGenState state, bool brokenExceptionTransitionsEnabled) { if (state.Classifier == null) { return; @@ -771,14 +783,24 @@ void SaveResource (string resource, string filename, string destDir, Func types, TypeDefinitionCache cache) + void WriteTypeMappings (NativeCodeGenState state) { - var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); - if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { + Log.LogDebugMessage ($"Generating type maps for architecture '{state.TargetArch}'"); + var tmg = new TypeMapGenerator (Log, state); + if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, TypemapOutputDirectory, GenerateNativeAssembly)) { throw new XamarinAndroidException (4308, Properties.Resources.XA4308); } GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); + } + + void WriteTypeMappings (List types, TypeDefinitionCache cache) + { + // var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); + // if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { + // throw new XamarinAndroidException (4308, Properties.Resources.XA4308); + // } + // GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); + // BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); } /// diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index f0bcbd2b8c7..95f29ee9e2f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -145,25 +145,7 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) - { - switch (abi.Trim ()) { - case "armeabi-v7a": - return AndroidTargetArch.Arm; - - case "arm64-v8a": - return AndroidTargetArch.Arm64; - - case "x86": - return AndroidTargetArch.X86; - - case "x86_64": - return AndroidTargetArch.X86_64; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - } + static internal AndroidTargetArch GetAndroidTargetArchForAbi (string abi) => MonoAndroidHelper.AbiToTargetArch (abi); static readonly string[] defaultLogLevel = {"MONO_LOG_LEVEL", "info"}; static readonly string[] defaultMonoDebug = {"MONO_DEBUG", "gen-compact-seq-points"}; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs index a14fe4352d1..57d280940b6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -213,16 +213,16 @@ static string GetMonoInitSource (string androidSdkPlatform) return builder.ToString (); } - public static void EnsureAllArchitecturesAreIdentical (TaskLoggingHelper logger, Dictionary javaStubStates) + public static void EnsureAllArchitecturesAreIdentical (TaskLoggingHelper logger, Dictionary javaStubStates) { if (javaStubStates.Count <= 1) { return; } // An expensive process, but we must be sure that all the architectures have the same data - JavaStubsState? templateState = null; + NativeCodeGenState? templateState = null; foreach (var kvp in javaStubStates) { - JavaStubsState state = kvp.Value; + NativeCodeGenState state = kvp.Value; if (templateState == null) { templateState = state; @@ -234,7 +234,7 @@ public static void EnsureAllArchitecturesAreIdentical (TaskLoggingHelper logger, } } - static void EnsureIdenticalCollections (TaskLoggingHelper logger, JavaStubsState templateState, JavaStubsState state) + static void EnsureIdenticalCollections (TaskLoggingHelper logger, NativeCodeGenState templateState, NativeCodeGenState state) { logger.LogDebugMessage ($"Ensuring Java type collection in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); @@ -280,7 +280,7 @@ static bool CheckWhetherTypesMatch (TypeDefinition templateType, TypeDefinition return String.Compare (templateType.FullName, type.FullName, StringComparison.Ordinal) == 0; } - static void EnsureClassifiersMatch (TaskLoggingHelper logger, JavaStubsState templateState, JavaStubsState state) + static void EnsureClassifiersMatch (TaskLoggingHelper logger, NativeCodeGenState templateState, NativeCodeGenState state) { logger.LogDebugMessage ($"Ensuring marshal method classifier in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs similarity index 53% rename from src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs rename to src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs index d538db6662a..ec46b31e03e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JavaStubsState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; +using Java.Interop.Tools.Cecil; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; @@ -8,29 +9,32 @@ namespace Xamarin.Android.Tasks; /// Holds state for typemap and marshal methods generators. A single instance of this /// class is created for each enabled target architecture. /// -class JavaStubsState +class NativeCodeGenState { /// /// Target architecture for which this instance was created. /// - public AndroidTargetArch TargetArch { get; } + public AndroidTargetArch TargetArch { get; } /// /// Classifier used when scanning for Java types in the target architecture's /// assemblies. Will be **null** if marshal methods are disabled. /// - public MarshalMethodsClassifier? Classifier { get; } + public MarshalMethodsClassifier? Classifier { get; } /// /// All the Java types discovered in the target architecture's assemblies. /// - public List AllJavaTypes { get; } + public List AllJavaTypes { get; } - public XAAssemblyResolverNew Resolver { get; } + public XAAssemblyResolverNew Resolver { get; } + public TypeDefinitionCache TypeCache { get; } + public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } - public JavaStubsState (AndroidTargetArch arch, XAAssemblyResolverNew resolver, List allJavaTypes, MarshalMethodsClassifier? classifier) + public NativeCodeGenState (AndroidTargetArch arch, TypeDefinitionCache tdCache, XAAssemblyResolverNew resolver, List allJavaTypes, MarshalMethodsClassifier? classifier) { TargetArch = arch; + TypeCache = tdCache; Resolver = resolver; AllJavaTypes = allJavaTypes; Classifier = classifier; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 4a930cf9c5c..3c4687ff238 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -1,11 +1,11 @@ using System; using System.IO; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Linq; using System.Text; using Java.Interop.Tools.Cecil; +using Microsoft.Build.Utilities; using Mono.Cecil; using Microsoft.Android.Build.Tasks; using Xamarin.Android.Tools; @@ -96,27 +96,13 @@ sealed class ReleaseGenerationState public readonly Dictionary KnownAssemblies; public readonly Dictionary MvidCache; - public readonly IDictionary> TempModules; + public readonly Dictionary TempModules; - // Just a convenient way to access one of the temp modules dictionaries, to be used when dealing with ABI-agnostic - // types in ProcessReleaseType. - public readonly Dictionary TempModulesAbiAgnostic; - - public ReleaseGenerationState (string[] supportedAbis) + public ReleaseGenerationState () { KnownAssemblies = new Dictionary (StringComparer.Ordinal); MvidCache = new Dictionary (); - - var tempModules = new Dictionary> (); - foreach (string abi in supportedAbis) { - var dict = new Dictionary (); - if (TempModulesAbiAgnostic == null) { - TempModulesAbiAgnostic = dict; - } - tempModules.Add (MonoAndroidHelper.AbiToTargetArch (abi), dict); - } - - TempModules = new ReadOnlyDictionary> (tempModules); + TempModules = new Dictionary (); } public void AddKnownAssembly (TypeDefinition td) @@ -133,80 +119,70 @@ public void AddKnownAssembly (TypeDefinition td) public string GetAssemblyName (TypeDefinition td) => td.Module.Assembly.FullName; } - Action logger; - Encoding outputEncoding; - byte[] moduleMagicString; - byte[] typemapIndexMagicString; - string[] supportedAbis; + readonly Encoding outputEncoding; + readonly byte[] moduleMagicString; + readonly byte[] typemapIndexMagicString; + readonly TaskLoggingHelper log; + readonly NativeCodeGenState state; public IList GeneratedBinaryTypeMaps { get; } = new List (); - public TypeMapGenerator (Action logger, string[] supportedAbis) + public TypeMapGenerator (TaskLoggingHelper log, NativeCodeGenState state) { - this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); - if (supportedAbis == null) - throw new ArgumentNullException (nameof (supportedAbis)); - this.supportedAbis = supportedAbis; - + this.log = log ?? throw new ArgumentNullException (nameof (log)); + this.state = state ?? throw new ArgumentNullException (nameof (state)); outputEncoding = Files.UTF8withoutBOM; moduleMagicString = outputEncoding.GetBytes (TypeMapMagicString); typemapIndexMagicString = outputEncoding.GetBytes (TypeMapIndexMagicString); } - void UpdateApplicationConfig (TypeDefinition javaType, ApplicationConfigTaskState appConfState) + void UpdateApplicationConfig (TypeDefinition javaType) { - if (appConfState.JniAddNativeMethodRegistrationAttributePresent) - return; - if (!javaType.HasCustomAttributes) + if (state.JniAddNativeMethodRegistrationAttributePresent || !javaType.HasCustomAttributes) { return; + } foreach (CustomAttribute ca in javaType.CustomAttributes) { - if (!appConfState.JniAddNativeMethodRegistrationAttributePresent && String.Compare ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal) == 0) { - appConfState.JniAddNativeMethodRegistrationAttributePresent = true; + if (!state.JniAddNativeMethodRegistrationAttributePresent && String.Compare ("JniAddNativeMethodRegistrationAttribute", ca.AttributeType.Name, StringComparison.Ordinal) == 0) { + state.JniAddNativeMethodRegistrationAttributePresent = true; break; } } } - public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, out ApplicationConfigTaskState appConfState) + public bool Generate (bool debugBuild, bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory, bool generateNativeAssembly) { - if (String.IsNullOrEmpty (outputDirectory)) + if (String.IsNullOrEmpty (outputDirectory)) { throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); - - if (!Directory.Exists (outputDirectory)) - Directory.CreateDirectory (outputDirectory); - - appConfState = new ApplicationConfigTaskState { - JniAddNativeMethodRegistrationAttributePresent = skipJniAddNativeMethodRegistrationAttributeScan - }; + } + Directory.CreateDirectory (outputDirectory); string typemapsOutputDirectory = Path.Combine (outputDirectory, "typemaps"); - if (debugBuild) { - return GenerateDebug (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, generateNativeAssembly, appConfState); + return GenerateDebug (skipJniAddNativeMethodRegistrationAttributeScan, typemapsOutputDirectory, generateNativeAssembly); } - return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, typemapsOutputDirectory, appConfState); + return GenerateRelease (skipJniAddNativeMethodRegistrationAttributeScan, typemapsOutputDirectory); } - bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, bool generateNativeAssembly, ApplicationConfigTaskState appConfState) + bool GenerateDebug (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory, bool generateNativeAssembly) { if (generateNativeAssembly) { - return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); + return GenerateDebugNativeAssembly (skipJniAddNativeMethodRegistrationAttributeScan, outputDirectory); } - return GenerateDebugFiles (skipJniAddNativeMethodRegistrationAttributeScan, javaTypes, cache, outputDirectory, appConfState); + return GenerateDebugFiles (skipJniAddNativeMethodRegistrationAttributeScan, outputDirectory); } - bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory) { var modules = new Dictionary (StringComparer.Ordinal); int maxModuleFileNameWidth = 0; int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { + foreach (JavaType jt in state.AllJavaTypes) { TypeDefinition td = jt.Type; - UpdateApplicationConfig (td, appConfState); + UpdateApplicationConfig (td); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -232,8 +208,8 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L modules.Add (moduleName, module); } - TypeMapDebugEntry entry = GetDebugEntry (td, cache); - HandleDebugDuplicates (javaDuplicates, entry, td, cache); + TypeMapDebugEntry entry = GetDebugEntry (td, state.TypeCache); + HandleDebugDuplicates (javaDuplicates, entry, td, state.TypeCache); if (entry.JavaName.Length > module.JavaNameWidth) module.JavaNameWidth = (uint)entry.JavaName.Length + 1; @@ -263,18 +239,18 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, L return true; } - bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory) { var javaToManaged = new List (); var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in javaTypes) { + foreach (JavaType jt in state.AllJavaTypes) { TypeDefinition td = jt.Type; - UpdateApplicationConfig (td, appConfState); + UpdateApplicationConfig (td); - TypeMapDebugEntry entry = GetDebugEntry (td, cache); - HandleDebugDuplicates (javaDuplicates, entry, td, cache); + TypeMapDebugEntry entry = GetDebugEntry (td, state.TypeCache); + HandleDebugDuplicates (javaDuplicates, entry, td, state.TypeCache); javaToManaged.Add (entry); managedToJava.Add (entry); @@ -375,35 +351,21 @@ string GetManagedTypeName (TypeDefinition td) return $"{managedTypeName}, {td.Module.Assembly.Name.Name}"; } - void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, AndroidTargetArch typeArch, ApplicationConfigTaskState appConfState, TypeDefinitionCache cache) + void ProcessReleaseType (ReleaseGenerationState genState, TypeDefinition td) { - UpdateApplicationConfig (td, appConfState); - - state.AddKnownAssembly (td); + UpdateApplicationConfig (td); + genState.AddKnownAssembly (td); // We must NOT use Guid here! The reason is that Guid sort order is different than its corresponding // byte array representation and on the runtime we need the latter in order to be able to binary search // through the module array. byte[] moduleUUID; - if (!state.MvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { + if (!genState.MvidCache.TryGetValue (td.Module.Mvid, out moduleUUID)) { moduleUUID = td.Module.Mvid.ToByteArray (); - state.MvidCache.Add (td.Module.Mvid, moduleUUID); - } - - bool abiAgnosticType = typeArch == AndroidTargetArch.None; - Dictionary tempModules; - if (abiAgnosticType) { - tempModules = state.TempModulesAbiAgnostic; - } else { - // It will throw if `typeArch` isn't in the dictionary. This is intentional, since we must have no TypeDefinition entries for architectures not - // mentioned in `supportedAbis`. - try { - tempModules = state.TempModules[typeArch]; - } catch (KeyNotFoundException ex) { - throw new InvalidOperationException ($"Internal error: cannot process type specific to architecture '{typeArch}', since that architecture isn't mentioned in the set of supported ABIs", ex); - } + genState.MvidCache.Add (td.Module.Mvid, moduleUUID); } + Dictionary tempModules = genState.TempModules; if (!tempModules.TryGetValue (moduleUUID, out ModuleReleaseData moduleData)) { moduleData = new ModuleReleaseData { Mvid = td.Module.Mvid, @@ -414,18 +376,10 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi DuplicateTypes = new List (), }; - if (abiAgnosticType) { - // ABI-agnostic types must be added to all the ABIs - foreach (var kvp in state.TempModules) { - kvp.Value.Add (moduleUUID, moduleData); - } - } else { - // ABI-specific types are added only to their respective tempModules - tempModules.Add (moduleUUID, moduleData); - } + tempModules.Add (moduleUUID, moduleData); } - string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, cache); + string javaName = Java.Interop.Tools.TypeNameMappings.JavaNativeTypeManager.ToJniName (td, state.TypeCache); // We will ignore generic types and interfaces when generating the Java to Managed map, but we must not // omit them from the table we output - we need the same number of entries in both java-to-managed and // managed-to-java tables. `SkipInJavaToManaged` set to `true` will cause the native assembly generator @@ -435,7 +389,7 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi JavaName = javaName, ManagedTypeName = td.FullName, Token = td.MetadataToken.ToUInt32 (), - AssemblyNameIndex = state.KnownAssemblies [state.GetAssemblyName (td)], + AssemblyNameIndex = genState.KnownAssemblies [genState.GetAssemblyName (td)], SkipInJavaToManaged = ShouldSkipInJavaToManaged (td), }; @@ -450,37 +404,30 @@ void ProcessReleaseType (ReleaseGenerationState state, TypeDefinition td, Androi } } - bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, List javaTypes, TypeDefinitionCache cache, string outputDirectory, ApplicationConfigTaskState appConfState) + bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory) { - var state = new ReleaseGenerationState (supportedAbis); - - foreach (JavaType jt in javaTypes) { - foreach (var kvp in jt.PerAbiTypes) { - ProcessReleaseType (state, kvp.Value, kvp.Key, appConfState, cache); - } + var genState = new ReleaseGenerationState (); + foreach (JavaType jt in state.AllJavaTypes) { + ProcessReleaseType (genState, jt.Type); } - foreach (var kvp in state.TempModules) { - AndroidTargetArch arch = kvp.Key; - Dictionary tempModules = kvp.Value; - ModuleReleaseData[] modules = tempModules.Values.ToArray (); - Array.Sort (modules, new ModuleUUIDArrayComparer ()); - - foreach (ModuleReleaseData module in modules) { - if (module.TypesScratch.Count == 0) { - module.Types = Array.Empty (); - continue; - } + ModuleReleaseData[] modules = genState.TempModules.Values.ToArray (); + Array.Sort (modules, new ModuleUUIDArrayComparer ()); - // No need to sort here, the LLVM IR generator will compute hashes and sort - // the array on write. - module.Types = module.TypesScratch.Values.ToArray (); + foreach (ModuleReleaseData module in modules) { + if (module.TypesScratch.Count == 0) { + module.Types = Array.Empty (); + continue; } - var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); - GenerateNativeAssembly (arch, composer, composer.Construct (), outputDirectory); + // No need to sort here, the LLVM IR generator will compute hashes and sort + // the array on write. + module.Types = module.TypesScratch.Values.ToArray (); } + var composer = new TypeMappingReleaseNativeAssemblyGenerator (new NativeTypeMappingData (modules)); + GenerateNativeAssembly (composer, composer.Construct (), outputDirectory); + return true; } @@ -489,35 +436,22 @@ bool ShouldSkipInJavaToManaged (TypeDefinition td) return td.IsInterface || td.HasGenericParameters; } - string GetOutputFilePath (string baseFileName, string abi) => $"{baseFileName}.{abi}.ll"; + string GetOutputFilePath (string baseFileName, AndroidTargetArch arch) => $"{baseFileName}.{MonoAndroidHelper.ArchToAbi (arch)}.ll"; - void GenerateNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) + void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) { WriteNativeAssembly ( - arch, composer, typeMapModule, - GetOutputFilePath (baseFileName, ArchToAbi (arch)) + GetOutputFilePath (baseFileName, state.TargetArch) ); } - void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string baseFileName) - { - foreach (string abi in supportedAbis) { - WriteNativeAssembly ( - GeneratePackageManagerJava.GetAndroidTargetArchForAbi (abi), - composer, - typeMapModule, - GetOutputFilePath (baseFileName, abi) - ); - } - } - - void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) + void WriteNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) { using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { try { - composer.Generate (typeMapModule, arch, sw, outputFile); + composer.Generate (typeMapModule, state.TargetArch, sw, outputFile); } catch { throw; } finally { @@ -527,17 +461,6 @@ void WriteNativeAssembly (AndroidTargetArch arch, LLVMIR.LlvmIrComposer composer } } - static string ArchToAbi (AndroidTargetArch arch) - { - return arch switch { - AndroidTargetArch.Arm => "armeabi-v7a", - AndroidTargetArch.Arm64 => "arm64-v8a", - AndroidTargetArch.X86_64 => "x86_64", - AndroidTargetArch.X86 => "x86", - _ => throw new InvalidOperationException ($"Unknown architecture {arch}") - }; - } - // Binary index file format, all data is little-endian: // // [Magic string] # XATI From 0c42a46937e7dfe3080094e36563a4987c62235c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Dec 2023 15:50:54 +0100 Subject: [PATCH 039/143] All the code generation steps appear to work fine --- .../Tasks/GenerateJavaStubs.cs | 691 +++--------------- .../Utilities/ACWMapGenerator.cs | 103 +++ .../Utilities/NativeCodeGenState.cs | 4 +- .../Utilities/TypeMapGenerator.cs | 2 + 4 files changed, 200 insertions(+), 600 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index de16616c269..dc61884a20b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -103,12 +103,6 @@ public override bool RunTask () try { bool useMarshalMethods = !Debug && EnableMarshalMethods; Run (useMarshalMethods); - - // We're going to do 3 steps here instead of separate tasks so - // we can share the list of JLO TypeDefinitions between them - // using (XAAssemblyResolver res = MakeResolver (useMarshalMethods)) { - // Run (res, useMarshalMethods); - // } } catch (XamarinAndroidException e) { Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode); if (MonoAndroidHelper.LogInternalExceptions) @@ -125,24 +119,6 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - XAAssemblyResolver MakeResolver (bool useMarshalMethods) - { - var readerParams = new ReaderParameters(); - if (useMarshalMethods) { - readerParams.ReadWrite = true; - readerParams.InMemory = true; - } - - var res = new XAAssemblyResolver (Log, loadDebugSymbols: true, loadReaderParameters: readerParams); - foreach (var dir in FrameworkDirectories) { - if (Directory.Exists (dir.ItemSpec)) { - res.FrameworkSearchDirectories.Add (dir.ItemSpec); - } - } - - return res; - } - XAAssemblyResolverNew MakeResolver (bool useMarshalMethods, AndroidTargetArch targetArch, Dictionary assemblies) { var readerParams = new ReaderParameters(); @@ -209,6 +185,8 @@ void Run (bool useMarshalMethods) // Now that "never" never happened, we can proceed knowing that at least the assembly sets are the same for each architecture var nativeCodeGenStates = new Dictionary (); bool generateJavaCode = true; + NativeCodeGenState? templateCodeGenState = null; + foreach (var kvp in allAssembliesPerArch) { AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; @@ -218,12 +196,21 @@ void Run (bool useMarshalMethods) return; } + if (templateCodeGenState == null) { + templateCodeGenState = state; + } + if (generateJavaCode) { generateJavaCode = false; } nativeCodeGenStates.Add (arch, state); } + + if (templateCodeGenState == null) { + throw new InvalidOperationException ($"Internal error: no native code generator state defined"); + } + JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, nativeCodeGenStates); if (useMarshalMethods) { @@ -263,271 +250,73 @@ void Run (bool useMarshalMethods) WriteTypeMappings (state); } - // TODO: ACW maps - // TODO: manifest - // TODO: additional java sources - } - - (bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) - { - XAAssemblyResolverNew resolver = MakeResolver (useMarshalMethods, arch, assemblies); - var tdCache = new TypeDefinitionCache (); - (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); - - var jcwContext = new JCWGeneratorContext (arch, resolver, assemblies.Values, javaTypesForJCW, tdCache, useMarshalMethods); - var jcwGenerator = new JCWGenerator (Log, jcwContext); - bool success; - - if (generateJavaCode) { - success = jcwGenerator.GenerateAndClassify (AndroidSdkPlatform, outputPath: Path.Combine (OutputDirectory, "src"), ApplicationJavaClass); - } else { - success = jcwGenerator.Classify (AndroidSdkPlatform); - } - - if (!success) { - return (false, null); - } - - return (true, new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, jcwGenerator.Classifier)); - } - - (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) - { - var scanner = new XAJavaTypeScanner (Log, cache) { - ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, - }; - List allJavaTypes = scanner.GetJavaTypes (assemblies.Values, res); - var javaTypesForJCW = new List (); - - foreach (JavaType jt in allJavaTypes) { - // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during - // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android - // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { - continue; - } - javaTypesForJCW.Add (jt); - } - - return (allJavaTypes, javaTypesForJCW); - } - - void RewriteMarshalMethods (NativeCodeGenState state, bool brokenExceptionTransitionsEnabled) - { - if (state.Classifier == null) { + var acwMapGen = new ACWMapGenerator (Log); + if (!acwMapGen.Generate (templateCodeGenState, AcwMapFile)) { return; } - var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier, state.Resolver); - rewriter.Rewrite (brokenExceptionTransitionsEnabled); + IList additionalProviders = MergeManifest (templateCodeGenState, userAssembliesPerArch[templateCodeGenState.TargetArch]); + GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders); } - void Run (XAAssemblyResolver res, bool useMarshalMethods) + void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList additionalProviders) { - PackageNamingPolicy pnp; - JavaNativeTypeManager.PackageNamingPolicy = Enum.TryParse (PackageNamingPolicy, out pnp) ? pnp : PackageNamingPolicyEnum.LowercaseCrc64; + // Create additional runtime provider java sources. + string providerTemplateFile = "MonoRuntimeProvider.Bundled.java"; + string providerTemplate = GetResource (providerTemplateFile); - Dictionary>? abiSpecificAssembliesByPath = null; - if (useMarshalMethods) { - abiSpecificAssembliesByPath = new Dictionary> (StringComparer.Ordinal); + foreach (var provider in additionalProviders) { + var contents = providerTemplate.Replace ("MonoRuntimeProvider", provider); + var real_provider = Path.Combine (OutputDirectory, "src", "mono", provider + ".java"); + Files.CopyIfStringChanged (contents, real_provider); } - // Put every assembly we'll need in the resolver - var allTypemapAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - var userAssemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - AndroidTargetArch arch; - string? assemblyKey; - - foreach (var assembly in ResolvedAssemblies) { - bool value; - if (bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {assembly.ItemSpec}"); - continue; - } - - bool addAssembly = false; - string fileName = Path.GetFileName (assembly.ItemSpec); - if (String.Compare ("Mono.Android.Export.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - addAssembly = true; - } else if (String.Compare ("Mono.Android.dll", fileName, StringComparison.OrdinalIgnoreCase) == 0) { - addAssembly = true; - } else if (MonoAndroidHelper.FrameworkAssembliesToTreatAsUserAssemblies.Contains (fileName)) { - if (!bool.TryParse (assembly.GetMetadata (AndroidSkipJavaStubGeneration), out value) || !value) { - string name = Path.GetFileNameWithoutExtension (fileName); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, assembly.ItemSpec); - addAssembly = true; - } - } - - assemblyKey = GetAssemblyKey (assembly, out arch); - if (addAssembly) { - MaybeAddAbiSpecifcAssembly (assembly, fileName); - if (!allTypemapAssemblies.ContainsKey (assemblyKey)) { - allTypemapAssemblies.Add (assemblyKey, assembly); + // Create additional application java sources. + StringWriter regCallsWriter = new StringWriter (); + regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); + foreach (JavaType jt in codeGenState.JavaTypesForJCW) { + TypeDefinition type = jt.Type; + if (JavaNativeTypeManager.IsApplication (type, codeGenState.TypeCache) || JavaNativeTypeManager.IsInstrumentation (type, codeGenState.TypeCache)) { + if (codeGenState.Classifier != null && !codeGenState.Classifier.FoundDynamicallyRegisteredMethods (type)) { + continue; } - } - res.Load (arch, assembly.ItemSpec); - } - - // However we only want to look for JLO types in user code for Java stub code generation - foreach (var asm in ResolvedUserAssemblies) { - if (bool.TryParse (asm.GetMetadata (AndroidSkipJavaStubGeneration), out bool value) && value) { - Log.LogDebugMessage ($"Skipping Java Stub Generation for {asm.ItemSpec}"); - continue; - } - - assemblyKey = GetAssemblyKey (asm, out arch); - res.Load (arch, asm.ItemSpec); - MaybeAddAbiSpecifcAssembly (asm, Path.GetFileName (asm.ItemSpec)); - if (!allTypemapAssemblies.ContainsKey (assemblyKey)) { - allTypemapAssemblies.Add (assemblyKey, asm); - } - - string name = Path.GetFileNameWithoutExtension (asm.ItemSpec); - if (!userAssemblies.ContainsKey (name)) - userAssemblies.Add (name, asm.ItemSpec); - } - - // Step 1 - Find all the JLO types - var cache = new TypeDefinitionCache (); - var scanner = new XAJavaTypeScanner (Log, cache) { - ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, - }; - - Console.WriteLine ("Will scan for Java types in the following assemblies:"); - foreach (var asm in allTypemapAssemblies.Values) { - Console.WriteLine ($" {asm}"); - } - List allJavaTypes = scanner.GetJavaTypes (allTypemapAssemblies.Values, res); - Console.WriteLine (); - Console.WriteLine ("All Java types:"); - foreach (JavaType jt in allJavaTypes) { - Console.WriteLine ($" {jt.Type.FullName}"); - } - - var javaTypes = new List (); - Console.WriteLine (); - Console.WriteLine ($"Checking for types to reject (useMarshalMethods == {useMarshalMethods})"); - foreach (JavaType jt in allJavaTypes) { - // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during - // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android - // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { - Console.WriteLine ($" rejecting: {jt.Type.FullName} ({jt.Type.Module.Assembly} => {jt.Type.Module.Assembly.Name.Name}"); - Console.WriteLine ($" if statement: {!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)} || {JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)}"); - continue; + string javaKey = JavaNativeTypeManager.ToJniName (type, codeGenState.TypeCache).Replace ('/', '.'); + regCallsWriter.WriteLine ( + "\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", + type.GetAssemblyQualifiedName (codeGenState.TypeCache), + javaKey + ); } - javaTypes.Add (jt); } + regCallsWriter.Close (); - Console.WriteLine (); - Console.WriteLine ("Post-processed Java types:"); - foreach (JavaType jt in javaTypes) { - Console.WriteLine ($" {jt.Type.FullName}"); - } + var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); + string applicationTemplateFile = "ApplicationRegistration.java"; + SaveResource ( + applicationTemplateFile, + applicationTemplateFile, + real_app_dir, + template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ()) + ); - MarshalMethodsClassifier classifier = null; // if (useMarshalMethods) { - // classifier = new MarshalMethodsClassifier (cache, res, Log); + // var state = new MarshalMethodsState (classifier.MarshalMethods); + // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (CreateJavaSources/state)", state.MarshalMethods); + // BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), state, RegisteredTaskObjectLifetime.Build); + // var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); + // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after state registered and retrieved (CreateJavaSources/state)", state.MarshalMethods); // } + } - // Step 2 - Generate Java stub code - bool success = true; - // var success = CreateJavaSources (javaTypes, cache, classifier, useMarshalMethods); - // if (!success) - // return; - - if (useMarshalMethods) { - // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed - // in order to properly generate wrapper methods in the marshal methods assembly rewriter. - // We don't care about those generated by us, since they won't contain the `XA_BROKEN_EXCEPTION_TRANSITIONS` variable we look for. - var environmentParser = new EnvironmentFilesParser (); -// Dictionary assemblyPaths = AddMethodsFromAbiSpecificAssemblies (classifier, res, abiSpecificAssembliesByPath); -// var rewriter = new MarshalMethodsAssemblyRewriter (classifier.MarshalMethods, classifier.Assemblies, assemblyPaths, Log); -// rewriter.Rewrite (res, environmentParser.AreBrokenExceptionTransitionsEnabled (Environments)); - } - - // Step 3 - Generate type maps - // Type mappings need to use all the assemblies, always. - WriteTypeMappings (allJavaTypes, cache); - - // We need to save a map of .NET type -> ACW type for resource file fixups - var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); - - var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); - var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); - - using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter ()) { - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - string managedKey = type.FullName.Replace ('/', '.'); - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - - acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - TypeDefinition conflict; - bool hasConflict = false; - if (managed.TryGetValue (managedKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!managedConflicts.TryGetValue (managedKey, out var list)) - managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); - list.Add (type.GetPartialAssemblyName (cache)); - } - hasConflict = true; - } - if (java.TryGetValue (javaKey, out conflict)) { - if (!conflict.Module.Name.Equals (type.Module.Name)) { - if (!javaConflicts.TryGetValue (javaKey, out var list)) - javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); - list.Add (type.GetAssemblyQualifiedName (cache)); - success = false; - } - hasConflict = true; - } - if (!hasConflict) { - managed.Add (managedKey, type); - java.Add (javaKey, type); - - acw_map.Write (managedKey); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - - acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); - acw_map.Write (';'); - acw_map.Write (javaKey); - acw_map.WriteLine (); - } - } - - acw_map.Flush (); - Files.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); - } - - foreach (var kvp in managedConflicts) { - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); - Log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); - } - - foreach (var kvp in javaConflicts) { - Log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); - foreach (var typeName in kvp.Value) - Log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); - } - - // Step 3 - Merge [Activity] and friends into AndroidManifest.xml + IList MergeManifest (NativeCodeGenState codeGenState, Dictionary userAssemblies) + { var manifest = new ManifestDocument (ManifestTemplate) { PackageName = PackageName, VersionName = VersionName, ApplicationLabel = ApplicationLabel ?? PackageName, Placeholders = ManifestPlaceholders, - Resolver = res, + Resolver = codeGenState.Resolver, SdkDir = AndroidSdkDir, TargetSdkVersion = AndroidSdkPlatform, MinSdkVersion = MonoAndroidHelper.ConvertSupportedOSPlatformVersionToApiLevel (SupportedOSPlatformVersion).ToString (), @@ -542,7 +331,7 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) } else if (!string.IsNullOrEmpty (VersionCode)) { manifest.VersionCode = VersionCode; } - manifest.Assemblies.AddRange (userAssemblies.Values); + manifest.Assemblies.AddRange (userAssemblies.Values.Select (item => item.ItemSpec)); if (!String.IsNullOrWhiteSpace (CheckedBuild)) { // We don't validate CheckedBuild value here, this will be done in BuildApk. We just know that if it's @@ -551,222 +340,68 @@ void Run (XAAssemblyResolver res, bool useMarshalMethods) manifest.ForceExtractNativeLibs = true; } - var additionalProviders = manifest.Merge (Log, cache, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); + IList additionalProviders = manifest.Merge (Log, codeGenState.TypeCache, codeGenState.AllJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); // Only write the new manifest if it actually changed if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { Log.LogDebugMessage ($"Saving: {MergedAndroidManifestOutput}"); } - // Create additional runtime provider java sources. - string providerTemplateFile = "MonoRuntimeProvider.Bundled.java"; - string providerTemplate = GetResource (providerTemplateFile); - - foreach (var provider in additionalProviders) { - var contents = providerTemplate.Replace ("MonoRuntimeProvider", provider); - var real_provider = Path.Combine (OutputDirectory, "src", "mono", provider + ".java"); - Files.CopyIfStringChanged (contents, real_provider); - } - - // Create additional application java sources. - StringWriter regCallsWriter = new StringWriter (); - regCallsWriter.WriteLine ("\t\t// Application and Instrumentation ACWs must be registered first."); - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; - if (JavaNativeTypeManager.IsApplication (type, cache) || JavaNativeTypeManager.IsInstrumentation (type, cache)) { - if (classifier != null && !classifier.FoundDynamicallyRegisteredMethods (type)) { - continue; - } - - string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); - regCallsWriter.WriteLine ("\t\tmono.android.Runtime.register (\"{0}\", {1}.class, {1}.__md_methods);", - type.GetAssemblyQualifiedName (cache), javaKey); - } - } - regCallsWriter.Close (); - - var real_app_dir = Path.Combine (OutputDirectory, "src", "mono", "android", "app"); - string applicationTemplateFile = "ApplicationRegistration.java"; - SaveResource (applicationTemplateFile, applicationTemplateFile, real_app_dir, - template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); - - if (useMarshalMethods) { - classifier.AddSpecialCaseMethods (); + return additionalProviders; + } - Log.LogDebugMessage ($"Number of generated marshal methods: {classifier.MarshalMethods.Count}"); + (bool success, NativeCodeGenState? stubsState) GenerateJavaSourcesAndMaybeClassifyMarshalMethods (AndroidTargetArch arch, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) + { + XAAssemblyResolverNew resolver = MakeResolver (useMarshalMethods, arch, assemblies); + var tdCache = new TypeDefinitionCache (); + (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); - if (classifier.RejectedMethodCount > 0) { - Log.LogWarning ($"Number of methods in the project that will be registered dynamically: {classifier.RejectedMethodCount}"); - } + var jcwContext = new JCWGeneratorContext (arch, resolver, assemblies.Values, javaTypesForJCW, tdCache, useMarshalMethods); + var jcwGenerator = new JCWGenerator (Log, jcwContext); + bool success; - if (classifier.WrappedMethodCount > 0) { - // TODO: change to LogWarning once the generator can output code which requires no non-blittable wrappers - Log.LogDebugMessage ($"Number of methods in the project that need marshal method wrappers: {classifier.WrappedMethodCount}"); - } + if (generateJavaCode) { + success = jcwGenerator.GenerateAndClassify (AndroidSdkPlatform, outputPath: Path.Combine (OutputDirectory, "src"), ApplicationJavaClass); + } else { + success = jcwGenerator.Classify (AndroidSdkPlatform); } - void MaybeAddAbiSpecifcAssembly (ITaskItem assembly, string fileName) - { - if (abiSpecificAssembliesByPath == null) { - return; - } - - string? abi = assembly.GetMetadata ("Abi"); - if (!String.IsNullOrEmpty (abi)) { - if (!abiSpecificAssembliesByPath.TryGetValue (fileName, out List? items)) { - items = new List (); - abiSpecificAssembliesByPath.Add (fileName, items); - } - - items.Add (assembly); - } + if (!success) { + return (false, null); } - string GetAssemblyKey (ITaskItem assembly, out AndroidTargetArch arch) - { - arch = MonoAndroidHelper.GetTargetArch (assembly); - if (arch == AndroidTargetArch.None) { - throw new InvalidOperationException ($"Internal error: assembly '{assembly.ItemSpec}' doesn't specify its ABI. This is not supported."); - } - - return $"[{arch}]{assembly.ItemSpec}"; - } + return (true, new NativeCodeGenState (arch, tdCache, resolver, allJavaTypes, javaTypesForJCW, jcwGenerator.Classifier)); } - AssemblyDefinition LoadAssembly (string path, XAAssemblyResolver? resolver = null) + (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) { - string pdbPath = Path.ChangeExtension (path, ".pdb"); - var readerParameters = new ReaderParameters { - AssemblyResolver = resolver, - InMemory = false, - ReadingMode = ReadingMode.Immediate, - ReadSymbols = File.Exists (pdbPath), - ReadWrite = false, + var scanner = new XAJavaTypeScanner (Log, cache) { + ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; + List allJavaTypes = scanner.GetJavaTypes (assemblies.Values, res); + var javaTypesForJCW = new List (); - MemoryMappedViewStream? viewStream = null; - try { - // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict - using var fileStream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); - using var mappedFile = MemoryMappedFile.CreateFromFile ( - fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); - viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); - - AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, readerParameters).Assembly; - - // We transferred the ownership of the viewStream to the collection. - viewStream = null; - - return result; - } finally { - viewStream?.Dispose (); - } - } - - bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache cache, MarshalMethodsClassifier classifier, bool useMarshalMethods) - { - // if (useMarshalMethods && classifier == null) { - // throw new ArgumentNullException (nameof (classifier)); - // } - - string outputPath = Path.Combine (OutputDirectory, "src"); - string monoInit = GetMonoInitSource (AndroidSdkPlatform); - bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); - bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; - - bool ok = true; - foreach (JavaType jt in newJavaTypes) { - TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids - if (t.IsInterface) { - // Interfaces are in typemap but they shouldn't have JCW generated for them + foreach (JavaType jt in allJavaTypes) { + // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during + // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android + // build and stored in a jar file. + if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { continue; } - - using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache, classifier) { - GenerateOnCreateOverrides = generateOnCreateOverrides, - ApplicationJavaClass = ApplicationJavaClass, - MonoRuntimeInitialization = monoInit, - }; - - jti.Generate (writer); - // if (useMarshalMethods) { - // if (classifier.FoundDynamicallyRegisteredMethods (t)) { - // Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName (cache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); - // } - // } - writer.Flush (); - - var path = jti.GetDestinationPath (outputPath); - Files.CopyIfStreamChanged (writer.BaseStream, path); - if (jti.HasExport && !hasExportReference) - Diagnostic.Error (4210, Properties.Resources.XA4210); - } catch (XamarinAndroidException xae) { - ok = false; - Log.LogError ( - subcategory: "", - errorCode: "XA" + xae.Code, - helpKeyword: string.Empty, - file: xae.SourceFile, - lineNumber: xae.SourceLine, - columnNumber: 0, - endLineNumber: 0, - endColumnNumber: 0, - message: xae.MessageWithoutCode, - messageArgs: Array.Empty () - ); - } catch (DirectoryNotFoundException ex) { - ok = false; - if (OS.IsWindows) { - Diagnostic.Error (5301, Properties.Resources.XA5301, t.FullName, ex); - } else { - Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); - } - } catch (Exception ex) { - ok = false; - Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); - } - } + javaTypesForJCW.Add (jt); } - // if (useMarshalMethods) { - // var state = new MarshalMethodsState (classifier.MarshalMethods); - // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (CreateJavaSources/state)", state.MarshalMethods); - // BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), state, RegisteredTaskObjectLifetime.Build); - // var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); - // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after state registered and retrieved (CreateJavaSources/state)", state.MarshalMethods); - // } - - return ok; + return (allJavaTypes, javaTypesForJCW); } - static string GetMonoInitSource (string androidSdkPlatform) + void RewriteMarshalMethods (NativeCodeGenState state, bool brokenExceptionTransitionsEnabled) { - // Lookup the mono init section from MonoRuntimeProvider: - // Mono Runtime Initialization {{{ - // }}} - var builder = new StringBuilder (); - var runtime = "Bundled"; - var api = ""; - if (int.TryParse (androidSdkPlatform, out int apiLevel) && apiLevel < 21) { - api = ".20"; - } - var assembly = Assembly.GetExecutingAssembly (); - using (var s = assembly.GetManifestResourceStream ($"MonoRuntimeProvider.{runtime}{api}.java")) - using (var reader = new StreamReader (s)) { - bool copy = false; - string line; - while ((line = reader.ReadLine ()) != null) { - if (string.CompareOrdinal ("\t\t// Mono Runtime Initialization {{{", line) == 0) - copy = true; - if (copy) - builder.AppendLine (line); - if (string.CompareOrdinal ("\t\t// }}}", line) == 0) - break; - } + if (state.Classifier == null) { + return; } - return builder.ToString (); + + var rewriter = new MarshalMethodsAssemblyRewriter (Log, state.TargetArch, state.Classifier, state.Resolver); + rewriter.Rewrite (brokenExceptionTransitionsEnabled); } string GetResource (string resource) @@ -792,147 +427,5 @@ void WriteTypeMappings (NativeCodeGenState state) } GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); } - - void WriteTypeMappings (List types, TypeDefinitionCache cache) - { - // var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); - // if (!tmg.Generate (Debug, SkipJniAddNativeMethodRegistrationAttributeScan, types, cache, TypemapOutputDirectory, GenerateNativeAssembly, out ApplicationConfigTaskState appConfState)) { - // throw new XamarinAndroidException (4308, Properties.Resources.XA4308); - // } - // GeneratedBinaryTypeMaps = tmg.GeneratedBinaryTypeMaps.ToArray (); - // BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), appConfState, RegisteredTaskObjectLifetime.Build); - } - - /// - /// - /// Classifier will see only unique assemblies, since that's what's processed by the JI type scanner - even though some assemblies may have - /// abi-specific features (e.g. inlined `IntPtr.Size` or processor-specific intrinsics), the **types** and **methods** will all be the same and, thus, - /// there's no point in scanning all of the additional copies of the same assembly. - /// - /// - /// This, however, doesn't work for the rewriter which needs to rewrite all of the copies so that they all have the same generated wrappers. In - /// order to do that, we need to go over the list of assemblies found by the classifier, see if they are abi-specific ones and then add all the - /// marshal methods from the abi-specific assembly copies, so that the rewriter can easily rewrite them all. - /// - /// - /// This method returns a dictionary matching `AssemblyDefinition` instances to the path on disk to the assembly file they were loaded from. It is necessary - /// because uses a stream to load the data, in order to avoid later sharing violation issues when writing the assemblies. Path - /// information is required by to be available for each - /// - /// - Dictionary AddMethodsFromAbiSpecificAssemblies (MarshalMethodsClassifier classifier, XAAssemblyResolver resolver, Dictionary> abiSpecificAssemblies) - { - IDictionary> marshalMethods = classifier.MarshalMethods; - ICollection assemblies = classifier.Assemblies; - var newAssemblies = new List (); - var assemblyPaths = new Dictionary (); - - foreach (AssemblyDefinition asmdef in assemblies) { - string fileName = Path.GetFileName (asmdef.MainModule.FileName); - if (!abiSpecificAssemblies.TryGetValue (fileName, out List? abiAssemblyItems)) { - continue; - } - - List assemblyMarshalMethods = FindMarshalMethodsForAssembly (marshalMethods, asmdef);; - Log.LogDebugMessage ($"Assembly {fileName} is ABI-specific"); - foreach (ITaskItem abiAssemblyItem in abiAssemblyItems) { - if (String.Compare (abiAssemblyItem.ItemSpec, asmdef.MainModule.FileName, StringComparison.Ordinal) == 0) { - continue; - } - - Log.LogDebugMessage ($"Looking for matching mashal methods in {abiAssemblyItem.ItemSpec}"); - FindMatchingMethodsInAssembly (abiAssemblyItem, classifier, assemblyMarshalMethods, resolver, newAssemblies, assemblyPaths); - } - } - - if (newAssemblies.Count > 0) { - foreach (AssemblyDefinition asmdef in newAssemblies) { - assemblies.Add (asmdef); - } - } - - return assemblyPaths; - } - - List FindMarshalMethodsForAssembly (IDictionary> marshalMethods, AssemblyDefinition asm) - { - var seenNativeCallbacks = new HashSet (); - var assemblyMarshalMethods = new List (); - - foreach (var kvp in marshalMethods) { - foreach (MarshalMethodEntry method in kvp.Value) { - if (method.NativeCallback.Module.Assembly != asm) { - continue; - } - - // More than one overriden method can use the same native callback method, we're interested only in unique native - // callbacks, since that's what gets rewritten. - if (seenNativeCallbacks.Contains (method.NativeCallback)) { - continue; - } - - seenNativeCallbacks.Add (method.NativeCallback); - assemblyMarshalMethods.Add (method); - } - } - - return assemblyMarshalMethods; - } - - void FindMatchingMethodsInAssembly (ITaskItem assemblyItem, MarshalMethodsClassifier classifier, List assemblyMarshalMethods, XAAssemblyResolver resolver, List newAssemblies, Dictionary assemblyPaths) - { - AssemblyDefinition asm = LoadAssembly (assemblyItem.ItemSpec, resolver); - newAssemblies.Add (asm); - assemblyPaths.Add (asm, assemblyItem.ItemSpec); - - foreach (MarshalMethodEntry methodEntry in assemblyMarshalMethods) { - TypeDefinition wantedType = methodEntry.NativeCallback.DeclaringType; - TypeDefinition? type = asm.MainModule.FindType (wantedType.FullName); - if (type == null) { - throw new InvalidOperationException ($"Internal error: type '{wantedType.FullName}' not found in assembly '{assemblyItem.ItemSpec}', a linker error?"); - } - - if (type.MetadataToken != wantedType.MetadataToken) { - throw new InvalidOperationException ($"Internal error: type '{type.FullName}' in assembly '{assemblyItem.ItemSpec}' has a different token ID than the original type"); - } - - FindMatchingMethodInType (methodEntry, type, classifier); - } - } - - void FindMatchingMethodInType (MarshalMethodEntry methodEntry, TypeDefinition type, MarshalMethodsClassifier classifier) - { - string callbackName = methodEntry.NativeCallback.FullName; - - foreach (MethodDefinition typeNativeCallbackMethod in type.Methods) { - if (String.Compare (typeNativeCallbackMethod.FullName, callbackName, StringComparison.Ordinal) != 0) { - continue; - } - - if (typeNativeCallbackMethod.Parameters.Count != methodEntry.NativeCallback.Parameters.Count) { - continue; - } - - if (typeNativeCallbackMethod.MetadataToken != methodEntry.NativeCallback.MetadataToken) { - throw new InvalidOperationException ($"Internal error: tokens don't match for '{typeNativeCallbackMethod.FullName}'"); - } - - bool allMatch = true; - for (int i = 0; i < typeNativeCallbackMethod.Parameters.Count; i++) { - if (String.Compare (typeNativeCallbackMethod.Parameters[i].ParameterType.FullName, methodEntry.NativeCallback.Parameters[i].ParameterType.FullName, StringComparison.Ordinal) != 0) { - allMatch = false; - break; - } - } - - if (!allMatch) { - continue; - } - - Log.LogDebugMessage ($"Found match for '{typeNativeCallbackMethod.FullName}' in {type.Module.FileName}"); - string methodKey = methodEntry.GetStoreMethodKey (classifier.TypeDefinitionCache); - classifier.MarshalMethods[methodKey].Add (new MarshalMethodEntry (methodEntry, typeNativeCallbackMethod)); - } - } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs new file mode 100644 index 00000000000..e8293ea1839 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; + +using Java.Interop.Tools.Cecil; +using Java.Interop.Tools.TypeNameMappings; +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Utilities; +using Mono.Cecil; + +namespace Xamarin.Android.Tasks; + +class ACWMapGenerator +{ + readonly TaskLoggingHelper log; + + public ACWMapGenerator (TaskLoggingHelper log) + { + this.log = log; + } + + public bool Generate (NativeCodeGenState codeGenState, string acwMapFile) + { + List javaTypes = codeGenState.JavaTypesForJCW; + TypeDefinitionCache cache = codeGenState.TypeCache; + + // We need to save a map of .NET type -> ACW type for resource file fixups + var managed = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + var java = new Dictionary (javaTypes.Count, StringComparer.Ordinal); + + var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); + var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); + + bool success = true; + + using var acw_map = MemoryStreamPool.Shared.CreateStreamWriter (); + foreach (JavaType jt in javaTypes) { + TypeDefinition type = jt.Type; + string managedKey = type.FullName.Replace ('/', '.'); + string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); + + acw_map.Write (type.GetPartialAssemblyQualifiedName (cache)); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + TypeDefinition conflict; + bool hasConflict = false; + if (managed.TryGetValue (managedKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!managedConflicts.TryGetValue (managedKey, out var list)) + managedConflicts.Add (managedKey, list = new List { conflict.GetPartialAssemblyName (cache) }); + list.Add (type.GetPartialAssemblyName (cache)); + success = false; + } + hasConflict = true; + } + if (java.TryGetValue (javaKey, out conflict)) { + if (!conflict.Module.Name.Equals (type.Module.Name)) { + if (!javaConflicts.TryGetValue (javaKey, out var list)) + javaConflicts.Add (javaKey, list = new List { conflict.GetAssemblyQualifiedName (cache) }); + list.Add (type.GetAssemblyQualifiedName (cache)); + success = false; + } + hasConflict = true; + } + if (!hasConflict) { + managed.Add (managedKey, type); + java.Add (javaKey, type); + + acw_map.Write (managedKey); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + + acw_map.Write (JavaNativeTypeManager.ToCompatJniName (type, cache).Replace ('/', '.')); + acw_map.Write (';'); + acw_map.Write (javaKey); + acw_map.WriteLine (); + } + } + + acw_map.Flush (); + Files.CopyIfStreamChanged (acw_map.BaseStream, acwMapFile); + + foreach (var kvp in managedConflicts) { + log.LogCodedWarning ("XA4214", Properties.Resources.XA4214, kvp.Key, string.Join (", ", kvp.Value)); + log.LogCodedWarning ("XA4214", Properties.Resources.XA4214_Result, kvp.Key, kvp.Value [0]); + } + + foreach (var kvp in javaConflicts) { + log.LogCodedError ("XA4215", Properties.Resources.XA4215, kvp.Key); + foreach (var typeName in kvp.Value) { + log.LogCodedError ("XA4215", Properties.Resources.XA4215_Details, kvp.Key, typeName); + } + } + + if (javaConflicts.Count > 0) { + return false; + } + + return success; + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs index ec46b31e03e..8896ae2a4fa 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -27,16 +27,18 @@ class NativeCodeGenState /// public List AllJavaTypes { get; } + public List JavaTypesForJCW { get; } public XAAssemblyResolverNew Resolver { get; } public TypeDefinitionCache TypeCache { get; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } - public NativeCodeGenState (AndroidTargetArch arch, TypeDefinitionCache tdCache, XAAssemblyResolverNew resolver, List allJavaTypes, MarshalMethodsClassifier? classifier) + public NativeCodeGenState (AndroidTargetArch arch, TypeDefinitionCache tdCache, XAAssemblyResolverNew resolver, List allJavaTypes, List javaTypesForJCW, MarshalMethodsClassifier? classifier) { TargetArch = arch; TypeCache = tdCache; Resolver = resolver; AllJavaTypes = allJavaTypes; + JavaTypesForJCW = javaTypesForJCW; Classifier = classifier; } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 3c4687ff238..97771cc8269 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -449,6 +449,8 @@ void GenerateNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule void WriteNativeAssembly (LLVMIR.LlvmIrComposer composer, LLVMIR.LlvmIrModule typeMapModule, string outputFile) { + // TODO: each .ll file should have a comment which lists paths to all the DLLs that were used to generate + // the native code using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { try { composer.Generate (typeMapModule, state.TargetArch, sw, outputFile); From 354ade98a3bba8f5f4d2dd94b61f41624aa9272f Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 6 Dec 2023 22:04:36 +0100 Subject: [PATCH 040/143] Onto marshal method generation and GeneratePackageManagerJava --- .../Tasks/GenerateJavaStubs.cs | 23 +++--- .../Tasks/GeneratePackageManagerJava.cs | 81 +++++++++++-------- .../MarshalMethodsNativeAssemblyGenerator.cs | 16 ++-- .../Utilities/NativeCodeGenState.cs | 2 + 4 files changed, 70 insertions(+), 52 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index dc61884a20b..c5cd2c5dec2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -27,7 +27,7 @@ namespace Xamarin.Android.Tasks public class GenerateJavaStubs : AndroidTask { - public const string MarshalMethodsRegisterTaskKey = ".:!MarshalMethods!:."; + public const string NativeCodeGenStateRegisterTaskKey = ".:!MarshalMethods!:."; public override string TaskPrefix => "GJS"; @@ -196,11 +196,8 @@ void Run (bool useMarshalMethods) return; } - if (templateCodeGenState == null) { - templateCodeGenState = state; - } - if (generateJavaCode) { + templateCodeGenState = state; generateJavaCode = false; } @@ -211,6 +208,7 @@ void Run (bool useMarshalMethods) throw new InvalidOperationException ($"Internal error: no native code generator state defined"); } + NativeCodeGenState.Template = templateCodeGenState; JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, nativeCodeGenStates); if (useMarshalMethods) { @@ -257,6 +255,12 @@ void Run (bool useMarshalMethods) IList additionalProviders = MergeManifest (templateCodeGenState, userAssembliesPerArch[templateCodeGenState.TargetArch]); GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders); + + if (!useMarshalMethods) { + return; + } + + BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build); } void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList additionalProviders) @@ -299,14 +303,6 @@ void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ()) ); - - // if (useMarshalMethods) { - // var state = new MarshalMethodsState (classifier.MarshalMethods); - // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after rewriter (CreateJavaSources/state)", state.MarshalMethods); - // BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (MarshalMethodsRegisterTaskKey), state, RegisteredTaskObjectLifetime.Build); - // var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); - // MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods after state registered and retrieved (CreateJavaSources/state)", state.MarshalMethods); - // } } IList MergeManifest (NativeCodeGenState codeGenState, Dictionary userAssemblies) @@ -355,7 +351,6 @@ IList MergeManifest (NativeCodeGenState codeGenState, Dictionary allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); - var jcwContext = new JCWGeneratorContext (arch, resolver, assemblies.Values, javaTypesForJCW, tdCache, useMarshalMethods); var jcwGenerator = new JCWGenerator (Log, jcwContext); bool success; diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 95f29ee9e2f..a19bbc4af49 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -367,21 +367,14 @@ void AddEnvironment () }; LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); - var marshalMethodsState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJavaStubs.MarshalMethodsRegisterTaskKey), RegisteredTaskObjectLifetime.Build); - MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; - - if (enableMarshalMethods) { - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods in GeneratePackageManagerJava (from registered state)", marshalMethodsState.MarshalMethods!); - marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( - assemblyCount, - uniqueAssemblyNames, - marshalMethodsState?.MarshalMethods, - Log - ); - } else { - marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator (assemblyCount, uniqueAssemblyNames); + var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( + ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), + RegisteredTaskObjectLifetime.Build + ); + + if (enableMarshalMethods && nativeCodeGenStates == null) { + throw new InvalidOperationException ("Internal error: native code generation states not registered"); } - LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); foreach (string abi in SupportedAbis) { string targetAbi = abi.ToLowerInvariant (); @@ -391,29 +384,53 @@ void AddEnvironment () string marshalMethodsLlFilePath = $"{marshalMethodsBaseAsmFilePath}.ll"; AndroidTargetArch targetArch = GetAndroidTargetArchForAbi (abi); - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - appConfigAsmGen.Generate (appConfigModule, targetArch, sw, environmentLlFilePath); - } catch { - throw; - } finally { - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, environmentLlFilePath); - } + using var appConfigWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + appConfigAsmGen.Generate (appConfigModule, targetArch, appConfigWriter, environmentLlFilePath); + } catch { + throw; + } finally { + appConfigWriter.Flush (); + Files.CopyIfStreamChanged (appConfigWriter.BaseStream, environmentLlFilePath); } - using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { - try { - marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, sw, marshalMethodsLlFilePath); - } catch { - throw; - } finally { - sw.Flush (); - Files.CopyIfStreamChanged (sw.BaseStream, marshalMethodsLlFilePath); - } + MarshalMethodsNativeAssemblyGenerator marshalMethodsAsmGen; + if (enableMarshalMethods) { + marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( + Log, + assemblyCount, + uniqueAssemblyNames, + EnsureCodeGenState (targetArch) + ); + } else { + marshalMethodsAsmGen = new MarshalMethodsNativeAssemblyGenerator ( + targetArch, + assemblyCount, + uniqueAssemblyNames + ); + } + + LLVMIR.LlvmIrModule marshalMethodsModule = marshalMethodsAsmGen.Construct (); + using var marshalMethodsWriter = MemoryStreamPool.Shared.CreateStreamWriter (); + try { + marshalMethodsAsmGen.Generate (marshalMethodsModule, targetArch, marshalMethodsWriter, marshalMethodsLlFilePath); + } catch { + throw; + } finally { + marshalMethodsWriter.Flush (); + Files.CopyIfStreamChanged (marshalMethodsWriter.BaseStream, marshalMethodsLlFilePath); } } + NativeCodeGenState EnsureCodeGenState (AndroidTargetArch targetArch) + { + if (nativeCodeGenStates == null || !nativeCodeGenStates.TryGetValue (targetArch, out NativeCodeGenState? state)) { + throw new InvalidOperationException ($"Internal error: missing native code generation state for architecture '{targetArch}'"); + } + + return state; + } + void AddEnvironmentVariable (string name, string value) { if (Char.IsUpper(name [0]) || !Char.IsLetter(name [0])) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 3e3a2f1b9b0..0c75810cb62 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -8,6 +8,7 @@ using Microsoft.Build.Utilities; using Xamarin.Android.Tasks.LLVMIR; +using Xamarin.Android.Tools; using CecilMethodDefinition = global::Mono.Cecil.MethodDefinition; using CecilParameterDefinition = global::Mono.Cecil.ParameterDefinition; @@ -239,12 +240,15 @@ public MarshalMethodAssemblyIndexValuePlaceholder (MarshalMethodInfo mmi, Assemb LlvmIrCallMarker defaultCallMarker; readonly bool generateEmptyCode; + readonly AndroidTargetArch targetArch; + readonly NativeCodeGenState? codeGenState; /// /// Constructor to be used ONLY when marshal methods are DISABLED /// - public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, ICollection uniqueAssemblyNames) + public MarshalMethodsNativeAssemblyGenerator (AndroidTargetArch targetArch, int numberOfAssembliesInApk, ICollection uniqueAssemblyNames) { + this.targetArch = targetArch; this.numberOfAssembliesInApk = numberOfAssembliesInApk; this.uniqueAssemblyNames = uniqueAssemblyNames ?? throw new ArgumentNullException (nameof (uniqueAssemblyNames)); generateEmptyCode = true; @@ -254,17 +258,17 @@ public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, IColl /// /// Constructor to be used ONLY when marshal methods are ENABLED /// - public MarshalMethodsNativeAssemblyGenerator (int numberOfAssembliesInApk, ICollection uniqueAssemblyNames, IDictionary> marshalMethods, TaskLoggingHelper logger) + public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper logger, int numberOfAssembliesInApk, ICollection uniqueAssemblyNames, NativeCodeGenState codeGenState) { - MonoAndroidHelper.DumpMarshalMethodsToConsole ("Classified methods in MarshalMethodsNativeAssemblyGenerator ctor", marshalMethods); - + this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); this.numberOfAssembliesInApk = numberOfAssembliesInApk; this.uniqueAssemblyNames = uniqueAssemblyNames ?? throw new ArgumentNullException (nameof (uniqueAssemblyNames)); - this.marshalMethods = marshalMethods; - this.logger = logger ?? throw new ArgumentNullException (nameof (logger)); + this.codeGenState = codeGenState ?? throw new ArgumentNullException (nameof (codeGenState)); generateEmptyCode = false; defaultCallMarker = LlvmIrCallMarker.Tail; + + throw new NotImplementedException ("WIP for per-RID assemblies"); } void Init () diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs index 8896ae2a4fa..dd618177e25 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -11,6 +11,8 @@ namespace Xamarin.Android.Tasks; /// class NativeCodeGenState { + public static NativeCodeGenState? Template { get; set; } + /// /// Target architecture for which this instance was created. /// From fa6fe8bfd4f8be641a60b076354f93962eb438f6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 7 Dec 2023 13:18:30 +0100 Subject: [PATCH 041/143] Let's see which tests break now --- .../Tasks/GenerateJavaStubs.cs | 54 +-- .../Tasks/GeneratePackageManagerJava.cs | 21 +- .../Utilities/ACWMapGenerator.cs | 5 +- .../Utilities/ApplicationConfigTaskState.cs | 9 - .../Utilities/JCWGenerator.cs | 25 +- .../LlvmIrGenerator/LlvmIrComposer.cs | 2 - .../Utilities/ManifestDocument.cs | 15 +- .../MarshalMethodsAssemblyRewriter.cs | 169 +--------- .../MarshalMethodsNativeAssemblyGenerator.cs | 26 +- .../Utilities/NativeCodeGenState.cs | 9 +- .../Utilities/TypeMapGenerator.cs | 10 +- .../Utilities/XAAssemblyResolver.cs | 319 +----------------- .../Utilities/XAJavaTypeScanner.cs | 90 +---- 13 files changed, 97 insertions(+), 657 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index c5cd2c5dec2..630af5375f0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -2,14 +2,9 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.IO.MemoryMappedFiles; using System.Linq; -using System.Reflection; -using System.Text; using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; using Mono.Cecil; @@ -119,15 +114,15 @@ public override bool RunTask () return !Log.HasLoggedErrors; } - XAAssemblyResolverNew MakeResolver (bool useMarshalMethods, AndroidTargetArch targetArch, Dictionary assemblies) + XAAssemblyResolver MakeResolver (bool useMarshalMethods, AndroidTargetArch targetArch, Dictionary assemblies) { - var readerParams = new ReaderParameters(); + var readerParams = new ReaderParameters (); if (useMarshalMethods) { readerParams.ReadWrite = true; readerParams.InMemory = true; } - var res = new XAAssemblyResolverNew (targetArch, Log, loadDebugSymbols: true, loadReaderParameters: readerParams); + var res = new XAAssemblyResolver (targetArch, Log, loadDebugSymbols: true, loadReaderParameters: readerParams); var uniqueDirs = new HashSet (StringComparer.OrdinalIgnoreCase); Log.LogDebugMessage ($"Adding search directories to new architecture {targetArch} resolver:"); @@ -207,9 +202,10 @@ void Run (bool useMarshalMethods) if (templateCodeGenState == null) { throw new InvalidOperationException ($"Internal error: no native code generator state defined"); } + JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, nativeCodeGenStates); NativeCodeGenState.Template = templateCodeGenState; - JCWGenerator.EnsureAllArchitecturesAreIdentical (Log, nativeCodeGenStates); + BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build); if (useMarshalMethods) { // We need to parse the environment files supplied by the user to see if they want to use broken exception transitions. This information is needed @@ -255,12 +251,6 @@ void Run (bool useMarshalMethods) IList additionalProviders = MergeManifest (templateCodeGenState, userAssembliesPerArch[templateCodeGenState.TargetArch]); GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders); - - if (!useMarshalMethods) { - return; - } - - BuildEngine4.RegisterTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (NativeCodeGenStateRegisterTaskKey), nativeCodeGenStates, RegisteredTaskObjectLifetime.Build); } void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList additionalProviders) @@ -278,8 +268,7 @@ void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList MergeManifest (NativeCodeGenState codeGenState, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) { - XAAssemblyResolverNew resolver = MakeResolver (useMarshalMethods, arch, assemblies); + Console.WriteLine ("GenerateJavaSourcesAndMaybeClassifyMarshalMethods"); + PrintAssemblies ($"[{arch}] Assemblies", assemblies); + PrintAssemblies ($"[{arch}] User assemblies", userAssemblies); + + XAAssemblyResolver resolver = MakeResolver (useMarshalMethods, arch, assemblies); var tdCache = new TypeDefinitionCache (); - (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); + (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); var jcwContext = new JCWGeneratorContext (arch, resolver, assemblies.Values, javaTypesForJCW, tdCache, useMarshalMethods); var jcwGenerator = new JCWGenerator (Log, jcwContext); bool success; @@ -366,24 +359,33 @@ IList MergeManifest (NativeCodeGenState codeGenState, Dictionary dict) + { + Console.WriteLine ($" {label}"); + foreach (var kvp in dict) { + Console.WriteLine ($" [{kvp.Key}] {kvp.Value.ItemSpec}"); + } + Console.WriteLine (); + } } - (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolverNew res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) + (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) { - var scanner = new XAJavaTypeScanner (Log, cache) { + var scanner = new XAJavaTypeScanner (res.TargetArch, Log, cache) { ErrorOnCustomJavaObject = ErrorOnCustomJavaObject, }; - List allJavaTypes = scanner.GetJavaTypes (assemblies.Values, res); - var javaTypesForJCW = new List (); + List allJavaTypes = scanner.GetJavaTypes (assemblies.Values, res); + var javaTypesForJCW = new List (); - foreach (JavaType jt in allJavaTypes) { + foreach (TypeDefinition type in allJavaTypes) { // When marshal methods are in use we do not want to skip non-user assemblies (such as Mono.Android) - we need to generate JCWs for them during // application build, unlike in Debug configuration or when marshal methods are disabled, in which case we use JCWs generated during Xamarin.Android // build and stored in a jar file. - if ((!useMarshalMethods && !userAssemblies.ContainsKey (jt.Type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (jt.Type, cache)) { + if ((!useMarshalMethods && !userAssemblies.ContainsKey (type.Module.Assembly.Name.Name)) || JavaTypeScanner.ShouldSkipJavaCallableWrapperGeneration (type, cache)) { continue; } - javaTypesForJCW.Add (jt); + javaTypesForJCW.Add (type); } return (allJavaTypes, javaTypesForJCW); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index a19bbc4af49..1ea58a98a9e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -337,8 +337,16 @@ void AddEnvironment () } } + var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( + ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), + RegisteredTaskObjectLifetime.Build + ); + + if (nativeCodeGenStates == null) { + throw new InvalidOperationException ("Internal error: native code generation states not registered"); + } + bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); - var appConfState = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (ApplicationConfigTaskState.RegisterTaskObjectKey), RegisteredTaskObjectLifetime.Build); var jniRemappingNativeCodeInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (ProjectSpecificTaskObjectKey (GenerateJniRemappingNativeCode.JniRemappingNativeCodeInfoKey), RegisteredTaskObjectLifetime.Build); var appConfigAsmGen = new ApplicationConfigNativeAssemblyGenerator (environmentVariables, systemProperties, Log) { UsesMonoAOT = usesMonoAOT, @@ -351,7 +359,7 @@ void AddEnvironment () PackageNamingPolicy = pnp, BoundExceptionType = boundExceptionType, InstantRunEnabled = InstantRunEnabled, - JniAddNativeMethodRegistrationAttributePresent = appConfState != null ? appConfState.JniAddNativeMethodRegistrationAttributePresent : false, + JniAddNativeMethodRegistrationAttributePresent = NativeCodeGenState.Template != null ? NativeCodeGenState.Template.JniAddNativeMethodRegistrationAttributePresent : false, HaveRuntimeConfigBlob = haveRuntimeConfigBlob, NumberOfAssembliesInApk = assemblyCount, BundledAssemblyNameWidth = assemblyNameWidth, @@ -367,15 +375,6 @@ void AddEnvironment () }; LLVMIR.LlvmIrModule appConfigModule = appConfigAsmGen.Construct (); - var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( - ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), - RegisteredTaskObjectLifetime.Build - ); - - if (enableMarshalMethods && nativeCodeGenStates == null) { - throw new InvalidOperationException ("Internal error: native code generation states not registered"); - } - foreach (string abi in SupportedAbis) { string targetAbi = abi.ToLowerInvariant (); string environmentBaseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{targetAbi}"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs index e8293ea1839..e624c826b31 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ACWMapGenerator.cs @@ -20,7 +20,7 @@ public ACWMapGenerator (TaskLoggingHelper log) public bool Generate (NativeCodeGenState codeGenState, string acwMapFile) { - List javaTypes = codeGenState.JavaTypesForJCW; + List javaTypes = codeGenState.JavaTypesForJCW; TypeDefinitionCache cache = codeGenState.TypeCache; // We need to save a map of .NET type -> ACW type for resource file fixups @@ -33,8 +33,7 @@ public bool Generate (NativeCodeGenState codeGenState, string acwMapFile) bool success = true; using var acw_map = MemoryStreamPool.Shared.CreateStreamWriter (); - foreach (JavaType jt in javaTypes) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in javaTypes) { string managedKey = type.FullName.Replace ('/', '.'); string javaKey = JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs deleted file mode 100644 index 5cdd3a6e6a2..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigTaskState.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Xamarin.Android.Tasks -{ - class ApplicationConfigTaskState - { - public const string RegisterTaskObjectKey = "Xamarin.Android.Tasks.ApplicationConfigTaskState"; - - public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } = false; - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs index 57d280940b6..97728e5a3be 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/JCWGenerator.cs @@ -21,11 +21,11 @@ class JCWGeneratorContext public bool UseMarshalMethods { get; } public AndroidTargetArch Arch { get; } public TypeDefinitionCache TypeDefinitionCache { get; } - public XAAssemblyResolverNew Resolver { get; } - public IList JavaTypes { get; } + public XAAssemblyResolver Resolver { get; } + public IList JavaTypes { get; } public ICollection ResolvedAssemblies { get; } - public JCWGeneratorContext (AndroidTargetArch arch, XAAssemblyResolverNew res, ICollection resolvedAssemblies, List javaTypesForJCW, TypeDefinitionCache tdCache, bool useMarshalMethods) + public JCWGeneratorContext (AndroidTargetArch arch, XAAssemblyResolver res, ICollection resolvedAssemblies, List javaTypesForJCW, TypeDefinitionCache tdCache, bool useMarshalMethods) { Arch = arch; Resolver = res; @@ -98,8 +98,7 @@ bool ProcessTypes (bool generateCode, string androidSdkPlatform, MarshalMethodsC bool hasExportReference = context.ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); bool ok = true; - foreach (JavaType jt in context.JavaTypes) { - TypeDefinition type = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids + foreach (TypeDefinition type in context.JavaTypes) { if (type.IsInterface) { // Interfaces are in typemap but they shouldn't have JCW generated for them continue; @@ -238,8 +237,8 @@ static void EnsureIdenticalCollections (TaskLoggingHelper logger, NativeCodeGenS { logger.LogDebugMessage ($"Ensuring Java type collection in architecture '{state.TargetArch}' matches the one in architecture '{templateState.TargetArch}'"); - List templateTypes = templateState.AllJavaTypes; - List types = state.AllJavaTypes; + List templateTypes = templateState.AllJavaTypes; + List types = state.AllJavaTypes; if (types.Count != templateTypes.Count) { throw new InvalidOperationException ($"Internal error: architecture '{state.TargetArch}' has a different number of types ({types.Count}) than the template architecture '{templateState.TargetArch}' ({templateTypes.Count})"); @@ -248,21 +247,21 @@ static void EnsureIdenticalCollections (TaskLoggingHelper logger, NativeCodeGenS var matchedTemplateTypes = new HashSet (); var mismatchedTypes = new List (); - foreach (JavaType type in types) { + foreach (TypeDefinition type in types) { TypeDefinition? matchedType = null; - foreach (JavaType templateType in templateTypes) { - if (matchedTemplateTypes.Contains (templateType.Type) || !CheckWhetherTypesMatch (templateType.Type, type.Type)) { + foreach (TypeDefinition templateType in templateTypes) { + if (matchedTemplateTypes.Contains (templateType) || !CheckWhetherTypesMatch (templateType, type)) { continue; } - matchedTemplateTypes.Add (templateType.Type); - matchedType = templateType.Type; + matchedTemplateTypes.Add (templateType); + matchedType = templateType; break; } if (matchedType == null) { - mismatchedTypes.Add (type.Type); + mismatchedTypes.Add (type); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index ed9dacb6352..1d876774171 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -1,7 +1,5 @@ using System; using System.IO; -using System.IO.Hashing; -using System.Text; using Xamarin.Android.Tools; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs index 8a1b8996794..62e795cbac6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ManifestDocument.cs @@ -255,7 +255,7 @@ void ReorderActivityAliases (TaskLoggingHelper log, XElement app) } } - public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) + public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, List subclasses, string applicationClass, bool embed, string bundledWearApplicationName, IEnumerable mergedManifestDocuments) { var manifest = doc.Root; @@ -330,8 +330,7 @@ public IList Merge (TaskLoggingHelper log, TypeDefinitionCache cache, Li throw new InvalidOperationException (string.Format ("The targetSdkVersion ({0}) is not a valid API level", targetSdkVersion)); int targetSdkVersionValue = tryTargetSdkVersion.Value; - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { if (t.IsAbstract) continue; @@ -568,7 +567,7 @@ Func GetGenerator (T return null; } - XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) + XElement CreateApplicationElement (XElement manifest, string applicationClass, List subclasses, TypeDefinitionCache cache) { var application = manifest.Descendants ("application").FirstOrDefault (); @@ -592,8 +591,7 @@ XElement CreateApplicationElement (XElement manifest, string applicationClass, L List typeAttr = new List (); List typeUsesLibraryAttr = new List (); List typeUsesConfigurationAttr = new List (); - foreach (JavaType jt in subclasses) { - TypeDefinition t = jt.Type; + foreach (TypeDefinition t in subclasses) { ApplicationAttribute aa = ApplicationAttribute.FromCustomAttributeProvider (t); if (aa == null) continue; @@ -925,7 +923,7 @@ void AddSupportsGLTextures (XElement application, TypeDefinitionCache cache) } } - void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) + void AddInstrumentations (XElement manifest, IList subclasses, int targetSdkVersion, TypeDefinitionCache cache) { var assemblyAttrs = Assemblies.SelectMany (path => InstrumentationAttribute.FromCustomAttributeProvider (Resolver.GetAssembly (path))); @@ -938,8 +936,7 @@ void AddInstrumentations (XElement manifest, IList subclasses, int tar manifest.Add (ia.ToElement (PackageName, cache)); } - foreach (JavaType jt in subclasses) { - TypeDefinition type = jt.Type; + foreach (TypeDefinition type in subclasses) { if (type.IsSubclassOf ("Android.App.Instrumentation", cache)) { var xe = InstrumentationFromTypeDefinition (type, JavaNativeTypeManager.ToJniName (type, cache).Replace ('/', '.'), cache); if (xe != null) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 397d056da7e..5b574077ac0 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; -using Java.Interop.Tools.Cecil; using Microsoft.Android.Build.Tasks; using Microsoft.Build.Utilities; using Mono.Cecil; @@ -22,15 +21,12 @@ sealed class AssemblyImports public MethodReference WaitForBridgeProcessingMethod; } - // IDictionary> methods; - // ICollection uniqueAssemblies; - // IDictionary assemblyPaths; readonly TaskLoggingHelper log; readonly MarshalMethodsClassifier classifier; - readonly XAAssemblyResolverNew resolver; + readonly XAAssemblyResolver resolver; readonly AndroidTargetArch targetArch; - public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch targetArch, MarshalMethodsClassifier classifier, XAAssemblyResolverNew resolver) + public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch targetArch, MarshalMethodsClassifier classifier, XAAssemblyResolver resolver) { this.log = log ?? throw new ArgumentNullException (nameof (log)); this.targetArch = targetArch; @@ -38,14 +34,6 @@ public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch this.resolver = resolver ?? throw new ArgumentNullException (nameof (resolver));; } - // public MarshalMethodsAssemblyRewriter (IDictionary> methods, ICollection uniqueAssemblies, IDictionary assemblyPaths, TaskLoggingHelper log) - // { - // this.assemblyPaths = assemblyPaths; - // this.methods = methods ?? throw new ArgumentNullException (nameof (methods)); - // this.uniqueAssemblies = uniqueAssemblies ?? throw new ArgumentNullException (nameof (uniqueAssemblies)); - // this.log = log ?? throw new ArgumentNullException (nameof (log)); - // } - // TODO: do away with broken exception transitions, there's no point in supporting them public void Rewrite (bool brokenExceptionTransitions) { @@ -193,148 +181,6 @@ void RemoveFile (string? path) } } - // TODO: do away with broken exception transitions, there's no point in supporting them - // public void Rewrite (XAAssemblyResolver resolver, bool brokenExceptionTransitions) - // { - // if (resolver == null) { - // throw new ArgumentNullException (nameof (resolver)); - // } - - // AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); - // if (monoAndroidRuntime == null) { - // throw new InvalidOperationException ($"Internal error: unable to load the Mono.Android.Runtime assembly"); - // } - - // TypeDefinition runtime = FindType (monoAndroidRuntime, "Android.Runtime.AndroidRuntimeInternal", required: true)!; - // MethodDefinition waitForBridgeProcessingMethod = FindMethod (runtime, "WaitForBridgeProcessing", required: true)!; - - // TypeDefinition androidEnvironment = FindType (monoAndroidRuntime, "Android.Runtime.AndroidEnvironmentInternal", required: true)!; - // MethodDefinition unhandledExceptionMethod = FindMethod (androidEnvironment, "UnhandledException", required: true)!; - - // TypeDefinition runtimeNativeMethods = FindType (monoAndroidRuntime, "Android.Runtime.RuntimeNativeMethods", required: true); - // MethodDefinition monoUnhandledExceptionMethod = FindMethod (runtimeNativeMethods, "monodroid_debugger_unhandled_exception", required: true); - - // AssemblyDefinition corlib = resolver.Resolve ("System.Private.CoreLib"); - // TypeDefinition systemException = FindType (corlib, "System.Exception", required: true); - - // MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver); - // var assemblyImports = new Dictionary (); - // foreach (AssemblyDefinition asm in uniqueAssemblies) { - // var imports = new AssemblyImports { - // MonoUnhandledExceptionMethod = asm.MainModule.ImportReference (monoUnhandledExceptionMethod), - // SystemException = asm.MainModule.ImportReference (systemException), - // UnhandledExceptionMethod = asm.MainModule.ImportReference (unhandledExceptionMethod), - // UnmanagedCallersOnlyAttribute = CreateImportedUnmanagedCallersOnlyAttribute (asm, unmanagedCallersOnlyAttributeCtor), - // WaitForBridgeProcessingMethod = asm.MainModule.ImportReference (waitForBridgeProcessingMethod), - // }; - - // assemblyImports.Add (asm, imports); - // } - - // log.LogDebugMessage ("Rewriting assemblies for marshal methods support"); - - // var processedMethods = new Dictionary (StringComparer.Ordinal); - // foreach (IList methodList in methods.Values) { - // foreach (MarshalMethodEntry method in methodList) { - // string fullNativeCallbackName = method.NativeCallback.FullName; - // if (processedMethods.TryGetValue (fullNativeCallbackName, out MethodDefinition nativeCallbackWrapper)) { - // method.NativeCallbackWrapper = nativeCallbackWrapper; - // continue; - // } - - // method.NativeCallbackWrapper = GenerateWrapper (method, assemblyImports, brokenExceptionTransitions); - // if (method.Connector != null) { - // if (method.Connector.IsStatic && method.Connector.IsPrivate) { - // log.LogDebugMessage ($"Removing connector method {method.Connector.FullName}"); - // method.Connector.DeclaringType?.Methods?.Remove (method.Connector); - // } else { - // log.LogWarning ($"NOT removing connector method {method.Connector.FullName} because it's either not static or not private"); - // } - // } - - // if (method.CallbackField != null) { - // if (method.CallbackField.IsStatic && method.CallbackField.IsPrivate) { - // log.LogDebugMessage ($"Removing callback delegate backing field {method.CallbackField.FullName}"); - // method.CallbackField.DeclaringType?.Fields?.Remove (method.CallbackField); - // } else { - // log.LogWarning ($"NOT removing callback field {method.CallbackField.FullName} because it's either not static or not private"); - // } - // } - - // processedMethods.Add (fullNativeCallbackName, method.NativeCallback); - // } - // } - - // foreach (AssemblyDefinition asm in uniqueAssemblies) { - // string path = GetAssemblyPath (asm); - // string pathPdb = Path.ChangeExtension (path, ".pdb"); - // bool havePdb = File.Exists (pathPdb); - - // var writerParams = new WriterParameters { - // WriteSymbols = havePdb, - // }; - - // string directory = Path.Combine (Path.GetDirectoryName (path), "new"); - // Directory.CreateDirectory (directory); - // string output = Path.Combine (directory, Path.GetFileName (path)); - // log.LogDebugMessage ($"Writing new version of '{path}' assembly: {output}"); - - // // TODO: this should be used eventually, but it requires that all the types are reloaded from the assemblies before typemaps are generated - // // since Cecil doesn't update the MVID in the already loaded types - // //asm.MainModule.Mvid = Guid.NewGuid (); - // asm.Write (output, writerParams); - - // CopyFile (output, path); - // RemoveFile (output); - - // if (havePdb) { - // string outputPdb = Path.ChangeExtension (output, ".pdb"); - // if (File.Exists (outputPdb)) { - // CopyFile (outputPdb, pathPdb); - // } - // RemoveFile (outputPdb); - // } - // } - - // void CopyFile (string source, string target) - // { - // log.LogDebugMessage ($"Copying rewritten assembly: {source} -> {target}"); - - // string targetBackup = $"{target}.bak"; - // if (File.Exists (target)) { - // // Try to avoid sharing violations by first renaming the target - // File.Move (target, targetBackup); - // } - - // File.Copy (source, target, true); - - // if (File.Exists (targetBackup)) { - // try { - // File.Delete (targetBackup); - // } catch (Exception ex) { - // // On Windows the deletion may fail, depending on lock state of the original `target` file before the move. - // log.LogDebugMessage ($"While trying to delete '{targetBackup}', exception was thrown: {ex}"); - // log.LogDebugMessage ($"Failed to delete backup file '{targetBackup}', ignoring."); - // } - // } - // } - - // void RemoveFile (string? path) - // { - // if (String.IsNullOrEmpty (path) || !File.Exists (path)) { - // return; - // } - - // try { - // log.LogDebugMessage ($"Deleting: {path}"); - // File.Delete (path); - // } catch (Exception ex) { - // log.LogWarning ($"Unable to delete source file '{path}'"); - // log.LogDebugMessage (ex.ToString ()); - // } - // } - // } - MethodDefinition GenerateWrapper (MarshalMethodEntry method, Dictionary assemblyImports, bool brokenExceptionTransitions) { MethodDefinition callback = method.NativeCallback; @@ -608,17 +454,6 @@ TypeReference ReturnValid (Type typeToLookUp) } } - // string GetAssemblyPath (AssemblyDefinition asm) - // { - // string filePath = asm.MainModule.FileName; - // if (!String.IsNullOrEmpty (filePath)) { - // return filePath; - // } - - // // No checking on purpose - the assembly **must** be there if its MainModule.FileName property returns a null or empty string - // return assemblyPaths[asm]; - // } - MethodDefinition GetUnmanagedCallersOnlyAttributeConstructor (IAssemblyResolver resolver) { AssemblyDefinition asm = resolver.Resolve ("System.Runtime.InteropServices"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 0c75810cb62..14ce7306de7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -226,10 +226,9 @@ public MarshalMethodAssemblyIndexValuePlaceholder (MarshalMethodInfo mmi, Assemb { 'L', typeof(_jobjectArray) }, }; - ICollection uniqueAssemblyNames; - int numberOfAssembliesInApk; - IDictionary> marshalMethods; - TaskLoggingHelper logger; + readonly ICollection uniqueAssemblyNames; + readonly int numberOfAssembliesInApk; + readonly TaskLoggingHelper logger; StructureInfo marshalMethodsManagedClassStructureInfo; StructureInfo marshalMethodNameStructureInfo; @@ -237,8 +236,7 @@ public MarshalMethodAssemblyIndexValuePlaceholder (MarshalMethodInfo mmi, Assemb List methods; List> classes = new List> (); - LlvmIrCallMarker defaultCallMarker; - + readonly LlvmIrCallMarker defaultCallMarker; readonly bool generateEmptyCode; readonly AndroidTargetArch targetArch; readonly NativeCodeGenState? codeGenState; @@ -267,18 +265,17 @@ public MarshalMethodsNativeAssemblyGenerator (TaskLoggingHelper logger, int numb generateEmptyCode = false; defaultCallMarker = LlvmIrCallMarker.Tail; - - throw new NotImplementedException ("WIP for per-RID assemblies"); } void Init () { - if (generateEmptyCode || marshalMethods == null || marshalMethods.Count == 0) { + if (generateEmptyCode || codeGenState.Classifier == null || codeGenState.Classifier.MarshalMethods.Count == 0) { return; } var seenClasses = new Dictionary (StringComparer.Ordinal); var allMethods = new List (); + IDictionary> marshalMethods = codeGenState.Classifier.MarshalMethods; // It's possible that several otherwise different methods (from different classes, but with the same // names and similar signatures) will actually share the same **short** native symbol name. In this case we must @@ -856,12 +853,11 @@ LlvmIrFunctionAttributeSet MakeXamarinAppInitAttributeSet (LlvmIrModule module) new NosyncFunctionAttribute (), new NounwindFunctionAttribute (), new WillreturnFunctionAttribute (), - // TODO: LLVM 16+ feature, enable when we switch to this version - // new MemoryFunctionAttribute { - // Default = MemoryAttributeAccessKind.Write, - // Argmem = MemoryAttributeAccessKind.None, - // InaccessibleMem = MemoryAttributeAccessKind.None, - // }, + new MemoryFunctionAttribute { + Default = MemoryAttributeAccessKind.Write, + Argmem = MemoryAttributeAccessKind.None, + InaccessibleMem = MemoryAttributeAccessKind.None, + }, new UwtableFunctionAttribute (), new MinLegalVectorWidthFunctionAttribute (0), new NoTrappingMathFunctionAttribute (true), diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs index dd618177e25..96d95391a49 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/NativeCodeGenState.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using Java.Interop.Tools.Cecil; +using Mono.Cecil; using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks; @@ -27,14 +28,14 @@ class NativeCodeGenState /// /// All the Java types discovered in the target architecture's assemblies. /// - public List AllJavaTypes { get; } + public List AllJavaTypes { get; } - public List JavaTypesForJCW { get; } - public XAAssemblyResolverNew Resolver { get; } + public List JavaTypesForJCW { get; } + public XAAssemblyResolver Resolver { get; } public TypeDefinitionCache TypeCache { get; } public bool JniAddNativeMethodRegistrationAttributePresent { get; set; } - public NativeCodeGenState (AndroidTargetArch arch, TypeDefinitionCache tdCache, XAAssemblyResolverNew resolver, List allJavaTypes, List javaTypesForJCW, MarshalMethodsClassifier? classifier) + public NativeCodeGenState (AndroidTargetArch arch, TypeDefinitionCache tdCache, XAAssemblyResolver resolver, List allJavaTypes, List javaTypesForJCW, MarshalMethodsClassifier? classifier) { TargetArch = arch; TypeCache = tdCache; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index 97771cc8269..e86ac3ddbd9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -180,8 +180,7 @@ bool GenerateDebugFiles (bool skipJniAddNativeMethodRegistrationAttributeScan, s int maxModuleNameWidth = 0; var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in state.AllJavaTypes) { - TypeDefinition td = jt.Type; + foreach (TypeDefinition td in state.AllJavaTypes) { UpdateApplicationConfig (td); string moduleName = td.Module.Assembly.Name.Name; ModuleDebugData module; @@ -245,8 +244,7 @@ bool GenerateDebugNativeAssembly (bool skipJniAddNativeMethodRegistrationAttribu var managedToJava = new List (); var javaDuplicates = new Dictionary> (StringComparer.Ordinal); - foreach (JavaType jt in state.AllJavaTypes) { - TypeDefinition td = jt.Type; + foreach (TypeDefinition td in state.AllJavaTypes) { UpdateApplicationConfig (td); TypeMapDebugEntry entry = GetDebugEntry (td, state.TypeCache); @@ -407,8 +405,8 @@ void ProcessReleaseType (ReleaseGenerationState genState, TypeDefinition td) bool GenerateRelease (bool skipJniAddNativeMethodRegistrationAttributeScan, string outputDirectory) { var genState = new ReleaseGenerationState (); - foreach (JavaType jt in state.AllJavaTypes) { - ProcessReleaseType (genState, jt.Type); + foreach (TypeDefinition td in state.AllJavaTypes) { + ProcessReleaseType (genState, td); } ModuleReleaseData[] modules = genState.TempModules.Values.ToArray (); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs index f501bb3e41e..940b0e7e291 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAAssemblyResolver.cs @@ -9,7 +9,7 @@ namespace Xamarin.Android.Tasks; -class XAAssemblyResolverNew : IAssemblyResolver +class XAAssemblyResolver : IAssemblyResolver { readonly List viewStreams = new List (); readonly Dictionary cache; @@ -26,12 +26,12 @@ class XAAssemblyResolverNew : IAssemblyResolver public ICollection SearchDirectories { get; } = new List (); public AndroidTargetArch TargetArch => targetArch; - public XAAssemblyResolverNew (AndroidTargetArch targetArch, TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) + public XAAssemblyResolver (AndroidTargetArch targetArch, TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) { this.targetArch = targetArch; this.log = log; this.loadDebugSymbols = loadDebugSymbols; - this.readerParameters = loadReaderParameters ?? new ReaderParameters(); + this.readerParameters = loadReaderParameters ?? new ReaderParameters (); cache = new Dictionary (StringComparer.OrdinalIgnoreCase); } @@ -70,7 +70,7 @@ public XAAssemblyResolverNew (AndroidTargetArch targetArch, TaskLoggingHelper lo } if (!name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) { - name = "${name}.dll"; + name = $"{name}.dll"; } var file = Path.Combine (directory, name); @@ -192,314 +192,3 @@ protected virtual void Dispose (bool disposing) disposed = true; } } - -class XAAssemblyResolver : IAssemblyResolver -{ - sealed class CacheEntry : IDisposable - { - bool disposed; - - Dictionary assemblies; - TaskLoggingHelper log; - AndroidTargetArch defaultArch; - - /// - /// This field is to be used by the `Resolve` overloads which don't have a way of indicating the desired ABI target for the assembly, but only when the - /// `AndroidTargetArch.None` entry for the assembly in question is **absent**. The field is always set to some value: either the very first assembly added - /// or the one with the `AndroidTargetArch.None` ABI. The latter always wins. - /// - public AssemblyDefinition Default { get; private set; } - public Dictionary Assemblies => assemblies; - - public CacheEntry (TaskLoggingHelper log, string filePath, AssemblyDefinition asm, AndroidTargetArch arch) - { - if (asm == null) { - throw new ArgumentNullException (nameof (asm)); - } - - this.log = log; - Default = asm; - defaultArch = arch; - assemblies = new Dictionary { - { arch, asm }, - }; - } - - public void Add (AndroidTargetArch arch, AssemblyDefinition asm) - { - if (asm == null) { - throw new ArgumentNullException (nameof (asm)); - } - - if (assemblies.ContainsKey (arch)) { - log.LogWarning ($"Entry for assembly '{asm}', architecture '{arch}' already exists. Replacing the old entry."); - } - - assemblies[arch] = asm; - if (arch == AndroidTargetArch.None && defaultArch != AndroidTargetArch.None) { - Default = asm; - defaultArch = arch; - } - } - - void Dispose (bool disposing) - { - if (disposed || !disposing) { - return; - } - - Default = null; - foreach (var kvp in assemblies) { - kvp.Value?.Dispose (); - } - assemblies.Clear (); - disposed = true; - } - - public void Dispose () - { - Dispose (disposing: true); - GC.SuppressFinalize (this); - } - } - - /// - /// Contains a collection of directories where framework assemblies can be found. This collection **must not** - /// contain any directories which contain ABI-specific assemblies. For those, use - /// - public ICollection FrameworkSearchDirectories { get; } = new List (); - - /// - /// Contains a collection of directories where Xamarin.Android (via linker, for instance) has placed the ABI - /// specific assemblies. Each ABI has its own set of directories to search. - /// - public IDictionary> AbiSearchDirectories { get; } = new Dictionary> (); - - readonly List viewStreams = new List (); - bool disposed; - TaskLoggingHelper log; - bool loadDebugSymbols; - ReaderParameters readerParameters; - readonly Dictionary cache; - - public XAAssemblyResolver (TaskLoggingHelper log, bool loadDebugSymbols, ReaderParameters? loadReaderParameters = null) - { - this.log = log; - this.loadDebugSymbols = loadDebugSymbols; - this.readerParameters = loadReaderParameters ?? new ReaderParameters(); - - cache = new Dictionary (StringComparer.OrdinalIgnoreCase); - } - - public AssemblyDefinition? Resolve (AssemblyNameReference name) - { - return Resolve (name, null); - } - - public AssemblyDefinition? Resolve (AssemblyNameReference name, ReaderParameters? parameters) - { - return Resolve (AndroidTargetArch.None, name, parameters); - } - - public AssemblyDefinition? Resolve (AndroidTargetArch arch, AssemblyNameReference name, ReaderParameters? parameters = null) - { - string shortName = name.Name; - if (cache.TryGetValue (shortName, out CacheEntry? entry)) { - return SelectAssembly (arch, name.FullName, entry, loading: false); - } - - if (arch == AndroidTargetArch.None) { - return FindAndLoadFromDirectories (arch, FrameworkSearchDirectories, name, parameters); - } - - if (!AbiSearchDirectories.TryGetValue (arch, out ICollection? directories) || directories == null) { - throw CreateLoadException (name); - } - - return FindAndLoadFromDirectories (arch, directories, name, parameters); - } - - AssemblyDefinition? FindAndLoadFromDirectories (AndroidTargetArch arch, ICollection directories, AssemblyNameReference name, ReaderParameters? parameters) - { - string? assemblyFile; - foreach (string dir in directories) { - if ((assemblyFile = SearchDirectory (name.Name, dir)) != null) { - return Load (arch, assemblyFile, parameters); - } - } - - return null; - } - - static FileNotFoundException CreateLoadException (AssemblyNameReference name) - { - return new FileNotFoundException ($"Could not load assembly '{name}'."); - } - - static string? SearchDirectory (string name, string directory) - { - if (Path.IsPathRooted (name) && File.Exists (name)) { - return name; - } - - var file = Path.Combine (directory, $"{name}.dll"); - if (File.Exists (file)) { - return file; - } - - return null; - } - - public virtual AssemblyDefinition? Load (AndroidTargetArch arch, string filePath, ReaderParameters? readerParameters = null) - { - string name = Path.GetFileNameWithoutExtension (filePath); - AssemblyDefinition? assembly; - if (cache.TryGetValue (name, out CacheEntry? entry)) { - assembly = SelectAssembly (arch, name, entry, loading: true); - if (assembly != null) { - return assembly; - } - } - - try { - assembly = ReadAssembly (filePath, readerParameters); - } catch (Exception e) when (e is FileNotFoundException || e is DirectoryNotFoundException) { - // These are ok, we can return null - return null; - } - - if (!cache.TryGetValue (name, out entry)) { - entry = new CacheEntry (log, filePath, assembly, arch); - cache.Add (name, entry); - } else { - entry.Add (arch, assembly); - } - - return assembly; - } - - AssemblyDefinition ReadAssembly (string filePath, ReaderParameters? readerParametersOverride = null) - { - ReaderParameters templateParameters = readerParametersOverride ?? this.readerParameters; - bool haveDebugSymbols = loadDebugSymbols && File.Exists (Path.ChangeExtension (filePath, ".pdb")); - var loadReaderParams = new ReaderParameters () { - ApplyWindowsRuntimeProjections = templateParameters.ApplyWindowsRuntimeProjections, - AssemblyResolver = this, - MetadataImporterProvider = templateParameters.MetadataImporterProvider, - InMemory = templateParameters.InMemory, - MetadataResolver = templateParameters.MetadataResolver, - ReadingMode = templateParameters.ReadingMode, - ReadSymbols = haveDebugSymbols, - ReadWrite = templateParameters.ReadWrite, - ReflectionImporterProvider = templateParameters.ReflectionImporterProvider, - SymbolReaderProvider = templateParameters.SymbolReaderProvider, - SymbolStream = templateParameters.SymbolStream, - }; - try { - return LoadFromMemoryMappedFile (filePath, loadReaderParams); - } catch (Exception ex) { - log.LogWarning ($"Failed to read '{filePath}' with debugging symbols. Retrying to load it without it. Error details are logged below."); - log.LogWarning ($"{ex.ToString ()}"); - loadReaderParams.ReadSymbols = false; - return LoadFromMemoryMappedFile (filePath, loadReaderParams); - } - } - - AssemblyDefinition LoadFromMemoryMappedFile (string file, ReaderParameters options) - { - // We can't use MemoryMappedFile when ReadWrite is true - if (options.ReadWrite) { - return AssemblyDefinition.ReadAssembly (file, options); - } - - bool origReadSymbols = options.ReadSymbols; - MemoryMappedViewStream? viewStream = null; - try { - // We must disable reading of symbols, even if they were present, because Cecil is unable to find the symbols file when - // assembly file name is unknown, and this is precisely the case when reading module from a stream. - // Until this issue is resolved, skipping symbol read saves time because reading exception isn't thrown and we don't - // retry the load. - options.ReadSymbols = false; - - // Create stream because CreateFromFile(string, ...) uses FileShare.None which is too strict - using var fileStream = new FileStream (file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, false); - using var mappedFile = MemoryMappedFile.CreateFromFile (fileStream, null, fileStream.Length, MemoryMappedFileAccess.Read, HandleInheritability.None, true); - viewStream = mappedFile.CreateViewStream (0, 0, MemoryMappedFileAccess.Read); - - AssemblyDefinition result = ModuleDefinition.ReadModule (viewStream, options).Assembly; - viewStreams.Add (viewStream); - - // We transferred the ownership of the viewStream to the collection. - viewStream = null; - - return result; - } finally { - options.ReadSymbols = origReadSymbols; - viewStream?.Dispose (); - } - } - - AssemblyDefinition? SelectAssembly (AndroidTargetArch arch, string assemblyName, CacheEntry? entry, bool loading) - { - if (entry == null) { - // Should "never" happen... - throw new ArgumentNullException (nameof (entry)); - } - - if (arch == AndroidTargetArch.None) { - // Disabled for now, generates too much noise. - // if (entry.Assemblies.Count > 1) { - // log.LogWarning ($"Architecture-agnostic entry requested for architecture-specific assembly '{assemblyName}'"); - // } - return entry.Default; - } - - if (!entry.Assemblies.TryGetValue (arch, out AssemblyDefinition? asm)) { - if (loading) { - return null; - } - - if (!entry.Assemblies.TryGetValue (AndroidTargetArch.None, out asm)) { - throw new InvalidOperationException ($"Internal error: assembly '{assemblyName}' for architecture '{arch}' not found in cache entry and architecture-agnostic entry is missing as well"); - } - - if (asm == null) { - throw new InvalidOperationException ($"Internal error: architecture-agnostic cache entry for assembly '{assemblyName}' is null"); - } - - log.LogWarning ($"Returning architecture-agnostic cache entry for assembly '{assemblyName}'. Requested architecture was: {arch}"); - return asm; - } - - if (asm == null) { - throw new InvalidOperationException ($"Internal error: null reference for assembly '{assemblyName}' in assembly cache entry"); - } - - return asm; - } - - public void Dispose () - { - Dispose (disposing: true); - GC.SuppressFinalize (this); - } - - protected virtual void Dispose (bool disposing) - { - if (disposed || !disposing) { - return; - } - - foreach (var kvp in cache) { - kvp.Value?.Dispose (); - } - cache.Clear (); - - foreach (MemoryMappedViewStream viewStream in viewStreams) { - viewStream.Dispose (); - } - viewStreams.Clear (); - - disposed = true; - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index 8c6677fb17f..a937a3daa24 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; using Java.Interop.Tools.Cecil; using Microsoft.Build.Framework; @@ -10,71 +9,31 @@ namespace Xamarin.Android.Tasks; -class JavaType -{ - public readonly TypeDefinition Type; - public readonly IDictionary? PerAbiTypes; - - public JavaType (TypeDefinition type, IDictionary perAbiTypes) - { - Type = type; - PerAbiTypes = new ReadOnlyDictionary (perAbiTypes); - } -} - class XAJavaTypeScanner { - sealed class TypeData - { - public readonly TypeDefinition FirstType; - public readonly Dictionary PerAbi; - - public TypeData (TypeDefinition firstType) - { - FirstType = firstType; - PerAbi = new Dictionary (); - } - } - public bool ErrorOnCustomJavaObject { get; set; } - TaskLoggingHelper log; - TypeDefinitionCache cache; + readonly TaskLoggingHelper log; + readonly TypeDefinitionCache cache; + readonly AndroidTargetArch targetArch; - public XAJavaTypeScanner (TaskLoggingHelper log, TypeDefinitionCache cache) + public XAJavaTypeScanner (AndroidTargetArch targetArch, TaskLoggingHelper log, TypeDefinitionCache cache) { + this.targetArch = targetArch; this.log = log; this.cache = cache; } - public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolverNew resolver) + public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) { - var types = new Dictionary (StringComparer.Ordinal); + var types = new List (); foreach (ITaskItem asmItem in inputAssemblies) { AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); - AssemblyDefinition asmdef = resolver.Load (asmItem.ItemSpec); - - foreach (ModuleDefinition md in asmdef.Modules) { - foreach (TypeDefinition td in md.Types) { - AddJavaType (td, types, arch); - } + if (arch != targetArch) { + throw new InvalidOperationException ($"Internal error: assembly '{asmItem.ItemSpec}' should be in the '{targetArch}' architecture, but is in '{arch}' instead."); } - } - var ret = new List (); - foreach (var kvp in types) { - ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.PerAbi)); - } - - return ret; - } - - public List GetJavaTypes (ICollection inputAssemblies, XAAssemblyResolver resolver) - { - var types = new Dictionary (StringComparer.Ordinal); - foreach (ITaskItem asmItem in inputAssemblies) { - AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); - AssemblyDefinition asmdef = resolver.Load (arch, asmItem.ItemSpec); + AssemblyDefinition asmdef = resolver.Load (asmItem.ItemSpec); foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { @@ -83,37 +42,14 @@ public List GetJavaTypes (ICollection inputAssemblies, XAAs } } - var ret = new List (); - foreach (var kvp in types) { - ret.Add (new JavaType (kvp.Value.FirstType, kvp.Value.PerAbi)); - } - - return ret; + return types; } - void AddJavaType (TypeDefinition type, Dictionary types, AndroidTargetArch arch) + void AddJavaType (TypeDefinition type, List types, AndroidTargetArch arch) { if (type.IsSubclassOf ("Java.Lang.Object", cache) || type.IsSubclassOf ("Java.Lang.Throwable", cache) || (type.IsInterface && type.ImplementsInterface ("Java.Interop.IJavaPeerable", cache))) { // For subclasses of e.g. Android.App.Activity. - string typeName = type.GetPartialAssemblyQualifiedName (cache); - if (!types.TryGetValue (typeName, out TypeData typeData)) { - typeData = new TypeData (type); - types.Add (typeName, typeData); - } - - if (typeData.PerAbi.ContainsKey (AndroidTargetArch.None)) { - if (arch == AndroidTargetArch.None) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}"); - } - - throw new InvalidOperationException ($"Previously added type '{type.FullName}' was in ABI-agnostic assembly, new one comes from ABI {arch} assembly"); - } - - if (typeData.PerAbi.ContainsKey (arch)) { - throw new InvalidOperationException ($"Duplicate type '{type.FullName}' in assembly {type.Module.FileName}, for ABI {arch}"); - } - - typeData.PerAbi.Add (arch, type); + types.Add (type); } else if (type.IsClass && !type.IsSubclassOf ("System.Exception", cache) && type.ImplementsInterface ("Android.Runtime.IJavaObject", cache)) { string message = $"XA4212: Type `{type.FullName}` implements `Android.Runtime.IJavaObject` but does not inherit `Java.Lang.Object` or `Java.Lang.Throwable`. This is not supported."; From 7f932f6e382a2cb2dc953f5855828e0883e23698 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 7 Dec 2023 16:06:04 +0100 Subject: [PATCH 042/143] Scan less assemblies + some debug spam cleanup --- .../Tasks/GenerateJavaStubs.cs | 13 ------ .../MarshalMethodsAssemblyRewriter.cs | 4 -- .../Utilities/XAJavaTypeScanner.cs | 42 +++++++++++++++++-- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 630af5375f0..0527fc2e6f7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -337,10 +337,6 @@ IList MergeManifest (NativeCodeGenState codeGenState, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods, bool generateJavaCode) { - Console.WriteLine ("GenerateJavaSourcesAndMaybeClassifyMarshalMethods"); - PrintAssemblies ($"[{arch}] Assemblies", assemblies); - PrintAssemblies ($"[{arch}] User assemblies", userAssemblies); - XAAssemblyResolver resolver = MakeResolver (useMarshalMethods, arch, assemblies); var tdCache = new TypeDefinitionCache (); (List allJavaTypes, List javaTypesForJCW) = ScanForJavaTypes (resolver, tdCache, assemblies, userAssemblies, useMarshalMethods); @@ -359,15 +355,6 @@ IList MergeManifest (NativeCodeGenState codeGenState, Dictionary dict) - { - Console.WriteLine ($" {label}"); - foreach (var kvp in dict) { - Console.WriteLine ($" [{kvp.Key}] {kvp.Value.ItemSpec}"); - } - Console.WriteLine (); - } } (List allJavaTypes, List javaTypesForJCW) ScanForJavaTypes (XAAssemblyResolver res, TypeDefinitionCache cache, Dictionary assemblies, Dictionary userAssemblies, bool useMarshalMethods) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs index 5b574077ac0..9f1034bd6b8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsAssemblyRewriter.cs @@ -37,8 +37,6 @@ public MarshalMethodsAssemblyRewriter (TaskLoggingHelper log, AndroidTargetArch // TODO: do away with broken exception transitions, there's no point in supporting them public void Rewrite (bool brokenExceptionTransitions) { - Console.WriteLine ("[new] Rewriting assemblies"); - AssemblyDefinition? monoAndroidRuntime = resolver.Resolve ("Mono.Android.Runtime"); if (monoAndroidRuntime == null) { throw new InvalidOperationException ($"[{targetArch}] Internal error: unable to load the Mono.Android.Runtime assembly"); @@ -59,9 +57,7 @@ public void Rewrite (bool brokenExceptionTransitions) MethodDefinition unmanagedCallersOnlyAttributeCtor = GetUnmanagedCallersOnlyAttributeConstructor (resolver); var assemblyImports = new Dictionary (); - Console.WriteLine (" populating assembly imports dict"); foreach (AssemblyDefinition asm in classifier.Assemblies) { - Console.WriteLine ($" assembly path: {asm.MainModule.FileName}"); var imports = new AssemblyImports { MonoUnhandledExceptionMethod = asm.MainModule.ImportReference (monoUnhandledExceptionMethod), SystemException = asm.MainModule.ImportReference (systemException), diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs index a937a3daa24..59933cbc360 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XAJavaTypeScanner.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.IO; using Java.Interop.Tools.Cecil; +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Mono.Cecil; @@ -11,6 +13,14 @@ namespace Xamarin.Android.Tasks; class XAJavaTypeScanner { + // Names of assemblies which don't have Mono.Android.dll references, or are framework assemblies, but which must + // be scanned for Java types. + static readonly HashSet SpecialAssemblies = new HashSet (StringComparer.OrdinalIgnoreCase) { + "Java.Interop.dll", + "Mono.Android.dll", + "Mono.Android.Runtime.dll", + }; + public bool ErrorOnCustomJavaObject { get; set; } readonly TaskLoggingHelper log; @@ -28,6 +38,12 @@ public List GetJavaTypes (ICollection inputAssemblies { var types = new List (); foreach (ITaskItem asmItem in inputAssemblies) { + if (!ShouldScan (asmItem)) { + log.LogDebugMessage ($"[{targetArch}] Skipping Java type scanning in assembly '{asmItem.ItemSpec}'"); + continue; + } + log.LogDebugMessage ($"[{targetArch}] Scanning assembly '{asmItem.ItemSpec}' for Java types"); + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (asmItem); if (arch != targetArch) { throw new InvalidOperationException ($"Internal error: assembly '{asmItem.ItemSpec}' should be in the '{targetArch}' architecture, but is in '{arch}' instead."); @@ -37,7 +53,7 @@ public List GetJavaTypes (ICollection inputAssemblies foreach (ModuleDefinition md in asmdef.Modules) { foreach (TypeDefinition td in md.Types) { - AddJavaType (td, types, arch); + AddJavaType (td, types); } } } @@ -45,7 +61,27 @@ public List GetJavaTypes (ICollection inputAssemblies return types; } - void AddJavaType (TypeDefinition type, List types, AndroidTargetArch arch) + bool ShouldScan (ITaskItem assembly) + { + string name = Path.GetFileName (assembly.ItemSpec); + if (SpecialAssemblies.Contains (name)) { + return true; + } + + string? hasMonoAndroidReferenceMetadata = assembly.GetMetadata ("HasMonoAndroidReference"); + if (String.IsNullOrEmpty (hasMonoAndroidReferenceMetadata)) { + return true; // Just in case - the metadata missing might be a false negative + } + + if (Boolean.TryParse (hasMonoAndroidReferenceMetadata, out bool hasMonoAndroidReference)) { + return hasMonoAndroidReference; + } + + // A catch-all, it's better to do more work than to miss something important. + return true; + } + + void AddJavaType (TypeDefinition type, List types) { if (type.IsSubclassOf ("Java.Lang.Object", cache) || type.IsSubclassOf ("Java.Lang.Throwable", cache) || (type.IsInterface && type.ImplementsInterface ("Java.Interop.IJavaPeerable", cache))) { // For subclasses of e.g. Android.App.Activity. @@ -66,7 +102,7 @@ void AddJavaType (TypeDefinition type, List types, AndroidTarget } foreach (TypeDefinition nested in type.NestedTypes) { - AddJavaType (nested, types, arch); + AddJavaType (nested, types); } } } From 0233691696a5e00d4e71b8ff71d39f5e38c47fd3 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 8 Dec 2023 16:20:16 +0100 Subject: [PATCH 043/143] Move individual assemblies to lib/{ARCH}/ and fix a couple of tests --- .../Tasks/BuildApk.cs | 18 +++--- .../MarshalMethodsNativeAssemblyGenerator.cs | 19 ++++-- src/monodroid/jni/embedded-assemblies-zip.cc | 12 +--- src/monodroid/jni/embedded-assemblies.hh | 4 +- src/monodroid/jni/mono-image-loader.hh | 4 +- src/monodroid/jni/shared-constants.hh | 4 +- src/monodroid/jni/xamarin-app.hh | 2 +- .../Tests/BundleToolTests.cs | 64 ++++++++++--------- 8 files changed, 63 insertions(+), 64 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 1dcfcd342ad..b2342237cc1 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -23,7 +23,7 @@ namespace Xamarin.Android.Tasks { public class BuildApk : AndroidTask { - const string ArchiveAssembliesPath = "assemblies"; + const string ArchiveAssembliesPath = "lib"; const string ArchiveLibPath = "lib"; public override string TaskPrefix => "BLD"; @@ -465,7 +465,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) if (UseAssemblyStore) { storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly); } else { - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec), compressionMethod: UncompressedMethod); + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec) + ".so", compressionMethod: UncompressedMethod); } // Try to add config if exists @@ -475,7 +475,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) storeAssemblyInfo.ConfigFile = new FileInfo (config); } } else { - AddAssemblyConfigEntry (apk, assemblyPath, config); + AddAssemblyConfigEntry (apk, assemblyPath, config + ".so"); } // Try to add symbols if Debug @@ -491,7 +491,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) if (UseAssemblyStore) { storeAssemblyInfo.SymbolsFile = new FileInfo (symbolsPath); } else { - AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols), compressionMethod: UncompressedMethod); + AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols) + ".so", compressionMethod: UncompressedMethod); } } } @@ -604,11 +604,11 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) { var parts = new List (); - bool haveRootPath = !String.IsNullOrEmpty (RootPath); + // bool haveRootPath = !String.IsNullOrEmpty (RootPath); - if (haveRootPath) { - parts.Add (ArchiveAssembliesPath); - } + // if (haveRootPath) { + // parts.Add (ArchiveAssembliesPath); + // } // There's no need anymore to treat satellite assemblies specially here. The PrepareSatelliteAssemblies task takes care of // properly setting `DestinationSubDirectory`, so we can just use it here. @@ -618,7 +618,7 @@ string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) } parts.Add (subDirectory.Replace ('\\', '/')); - return MonoAndroidHelper.MakeZipArchivePath (haveRootPath ? RootPath : ArchiveAssembliesPath, parts) + "/"; + return MonoAndroidHelper.MakeZipArchivePath (/*haveRootPath ? RootPath : */ArchiveAssembliesPath, parts) + "/"; } sealed class LibInfo diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 14ce7306de7..0a3dc3827f6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -109,7 +109,7 @@ public override string GetComment (object data, string fieldName) var klass = EnsureType (data); if (String.Compare ("token", fieldName, StringComparison.Ordinal) == 0) { - return $" token 0x{klass.token:x}; class name: {klass.ClassName}"; + return $" class name: {klass.ClassName}"; } return String.Empty; @@ -119,7 +119,7 @@ public override string GetComment (object data, string fieldName) [NativeAssemblerStructContextDataProvider (typeof(MarshalMethodsManagedClassDataProvider))] sealed class MarshalMethodsManagedClass { - [NativeAssembler (UsesDataProvider = true)] + [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint token; [NativePointer (IsNull = true)] @@ -136,7 +136,7 @@ public override string GetComment (object data, string fieldName) var methodName = EnsureType (data); if (String.Compare ("id", fieldName, StringComparison.Ordinal) == 0) { - return $" id 0x{methodName.id:x}; name: {methodName.name}"; + return $" name: {methodName.name}"; } return String.Empty; @@ -152,7 +152,7 @@ sealed class MarshalMethodName [NativeAssembler (Ignore = true)] public ulong Id64; - [NativeAssembler (UsesDataProvider = true)] + [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public ulong id; public string name; } @@ -985,10 +985,14 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) foreach (string name in uniqueAssemblyNames) { // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here - string clippedName = Path.Combine (Path.GetDirectoryName (name) ?? String.Empty, Path.GetFileNameWithoutExtension (name)); + string dirName = Path.GetDirectoryName (name) ?? String.Empty; + string clippedName = Path.Combine (dirName, Path.GetFileNameWithoutExtension (name)); + string soName = Path.Combine (dirName, $"{Path.GetFileName (name)}.so"); ulong hashFull32 = MonoAndroidHelper.GetXxHash (name, is64Bit: false); + ulong hashSo32 = MonoAndroidHelper.GetXxHash (soName, is64Bit: false); ulong hashClipped32 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: false); ulong hashFull64 = MonoAndroidHelper.GetXxHash (name, is64Bit: true); + ulong hashSo64 = MonoAndroidHelper.GetXxHash (soName, is64Bit: true); ulong hashClipped64 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: true); // @@ -996,8 +1000,10 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) // `number_of_assembly_name_forms_in_image_cache` constant to the number of forms. // acs.Hashes32.Add ((uint)Convert.ChangeType (hashFull32, typeof(uint)), (name, index)); + acs.Hashes32.Add ((uint)Convert.ChangeType (hashSo32, typeof(uint)), (soName, index)); acs.Hashes32.Add ((uint)Convert.ChangeType (hashClipped32, typeof(uint)), (clippedName, index)); acs.Hashes64.Add (hashFull64, (name, index)); + acs.Hashes64.Add (hashSo64, (soName, index)); acs.Hashes64.Add (hashClipped64, (clippedName, index)); index++; @@ -1025,6 +1031,7 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) BeforeWriteCallbackCallerState = acs, GetArrayItemCommentCallback = GetAssemblyImageCacheItemComment, GetArrayItemCommentCallbackCallerState = acs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (assembly_image_cache_hashes); @@ -1070,7 +1077,7 @@ void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget i = acs.Hashes32[v32].index; } - return $" {index}: {name} => 0x{value:x} => {i}"; + return $" {index}: {name} => {i}"; } void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index f5f8411c92f..fd9d73f68fe 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -26,16 +26,6 @@ using read_count_type = unsigned int; using read_count_type = size_t; #endif -force_inline bool -EmbeddedAssemblies::is_debug_file (dynamic_local_string const& name) noexcept -{ - return utils.ends_with (name, ".pdb") -#if !defined (NET) - || utils.ends_with (name, ".mdb") -#endif - ; -} - force_inline bool EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& buf, dynamic_local_string &entry_name, ZipEntryLoadState &state) noexcept { @@ -120,7 +110,7 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c constexpr bool entry_is_overridden = false; #endif - if (register_debug_symbols && !entry_is_overridden && is_debug_file (entry_name)) { + if (register_debug_symbols && !entry_is_overridden && utils.ends_with (entry_name, SharedConstants::PDB_EXTENSION)) { if (bundled_debug_data == nullptr) { bundled_debug_data = new std::vector (); bundled_debug_data->reserve (application_config.number_of_assemblies_in_apk); diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index dea1847025f..77d7f403060 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -96,7 +96,7 @@ namespace xamarin::android::internal { static constexpr off_t ZIP_CENTRAL_LEN = 46; static constexpr off_t ZIP_LOCAL_LEN = 30; static constexpr char zip_path_separator[] = "/"; - static constexpr auto assemblies_prefix = concat_const ("assemblies", zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); + static constexpr auto assemblies_prefix = concat_const ("lib", zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); // We have two records for each assembly, for names with and without the extension static constexpr uint32_t assembly_store_index_entries_per_assembly = 2; @@ -298,8 +298,6 @@ namespace xamarin::android::internal { return c_unique_ptr (mono_string_to_utf8 (const_cast(s))); } - bool is_debug_file (dynamic_local_string const& name) noexcept; - template static const Entry* binary_search (const Key *key, const Entry *base, size_t nmemb, size_t extra_size = 0) noexcept; diff --git a/src/monodroid/jni/mono-image-loader.hh b/src/monodroid/jni/mono-image-loader.hh index 66a73a1afed..a21c780885a 100644 --- a/src/monodroid/jni/mono-image-loader.hh +++ b/src/monodroid/jni/mono-image-loader.hh @@ -120,8 +120,8 @@ namespace xamarin::android::internal { #if defined (USE_CACHE) ssize_t index = find_index (hash); if (index < 0) { - log_warn (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); - return image; + log_fatal (LOG_ASSEMBLY, "Failed to look up image index for hash 0x%zx", hash); + Helpers::abort_application (); } // We don't need to worry about locking here. Even if we're overwriting an entry just set by another diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index b0b2d2856c8..3ebef32a3b4 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -29,8 +29,8 @@ namespace xamarin::android::internal static constexpr char ANDROID_ENVIRONMENT_CLASS_NAME[] = "AndroidEnvironment"; static constexpr char ANDROID_RUNTIME_INTERNAL_CLASS_NAME[] = "AndroidRuntimeInternal"; - static constexpr char DLL_EXTENSION[] = ".dll"; - + static constexpr char DLL_EXTENSION[] = ".dll.so"; + static constexpr char PDB_EXTENSION[] = ".pdb.so"; #if defined (NET) static constexpr char RUNTIME_CONFIG_BLOB_NAME[] = "arc.bin.so"; #endif // def NET diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 42eeee0ce15..39384cbeea0 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -333,7 +333,7 @@ struct MarshalMethodsManagedClass // Number of assembly name forms for which we generate hashes (essentially file name mutations. For instance // `HelloWorld.dll`, `HelloWorld`, `en-US/HelloWorld` etc). This is multiplied by the number of assemblies in the apk to // obtain number of entries in the `assembly_image_cache_hashes` and `assembly_image_cache_indices` entries -constexpr uint32_t number_of_assembly_name_forms_in_image_cache = 2; +constexpr uint32_t number_of_assembly_name_forms_in_image_cache = 3; // These 3 arrays constitute the cache used to store pointers to loaded managed assemblies. // Three arrays are used so that we can have multiple hashes pointing to the same MonoImage*. diff --git a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs index cc83d72b5d3..0b8799f04cf 100644 --- a/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/BundleToolTests.cs @@ -153,19 +153,6 @@ public void BaseZip () }; string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; - if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); - expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); - expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); - } else { - expectedFiles.Add ("root/assemblies/Java.Interop.dll"); - expectedFiles.Add ("root/assemblies/Mono.Android.dll"); - expectedFiles.Add ("root/assemblies/Localization.dll"); - expectedFiles.Add ("root/assemblies/es/Localization.resources.dll"); - expectedFiles.Add ("root/assemblies/UnnamedProject.dll"); - } //These are random files from Google Play Services .aar files expectedFiles.Add ("root/play-services-base.properties"); @@ -174,13 +161,28 @@ public void BaseZip () expectedFiles.Add ("root/play-services-tasks.properties"); foreach (var abi in Abis) { + // All assemblies are in per-abi directories now + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}{abi}/Java.Interop.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/Mono.Android.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/Localization.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/es/Localization.resources.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/UnnamedProject.dll"); + } else { + expectedFiles.Add ($"lib/{abi}/Java.Interop.dll.so"); + expectedFiles.Add ($"lib/{abi}/Mono.Android.dll.so"); + expectedFiles.Add ($"lib/{abi}/Localization.dll.so"); + expectedFiles.Add ($"lib/{abi}/es/Localization.resources.dll.so"); + expectedFiles.Add ($"lib/{abi}/UnnamedProject.dll.so"); + } + expectedFiles.Add ($"lib/{abi}/libmonodroid.so"); expectedFiles.Add ($"lib/{abi}/libmonosgen-2.0.so"); expectedFiles.Add ($"lib/{abi}/libxamarin-app.so"); if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}System.Private.CoreLib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/System.Private.CoreLib.dll"); } else { - expectedFiles.Add ($"root/assemblies/{abi}/System.Private.CoreLib.dll"); + expectedFiles.Add ($"lib/{abi}/System.Private.CoreLib.dll.so"); } expectedFiles.Add ($"lib/{abi}/libSystem.IO.Compression.Native.so"); expectedFiles.Add ($"lib/{abi}/libSystem.Native.so"); @@ -211,19 +213,6 @@ public void AppBundle () }; string blobEntryPrefix = ArchiveAssemblyHelper.DefaultAssemblyStoreEntryPrefix; - if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}Java.Interop.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Mono.Android.dll"); - expectedFiles.Add ($"{blobEntryPrefix}Localization.dll"); - expectedFiles.Add ($"{blobEntryPrefix}es/Localization.resources.dll"); - expectedFiles.Add ($"{blobEntryPrefix}UnnamedProject.dll"); - } else { - expectedFiles.Add ("base/root/assemblies/Java.Interop.dll"); - expectedFiles.Add ("base/root/assemblies/Mono.Android.dll"); - expectedFiles.Add ("base/root/assemblies/Localization.dll"); - expectedFiles.Add ("base/root/assemblies/es/Localization.resources.dll"); - expectedFiles.Add ("base/root/assemblies/UnnamedProject.dll"); - } //These are random files from Google Play Services .aar files expectedFiles.Add ("base/root/play-services-base.properties"); @@ -232,13 +221,28 @@ public void AppBundle () expectedFiles.Add ("base/root/play-services-tasks.properties"); foreach (var abi in Abis) { + // All assemblies are in per-abi directories now + if (usesAssemblyBlobs) { + expectedFiles.Add ($"{blobEntryPrefix}{abi}/Java.Interop.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/Mono.Android.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/Localization.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/es/Localization.resources.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/UnnamedProject.dll"); + } else { + expectedFiles.Add ($"base/lib/{abi}/Java.Interop.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/Mono.Android.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/Localization.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/es/Localization.resources.dll.so"); + expectedFiles.Add ($"base/lib/{abi}/UnnamedProject.dll.so"); + } + expectedFiles.Add ($"base/lib/{abi}/libmonodroid.so"); expectedFiles.Add ($"base/lib/{abi}/libmonosgen-2.0.so"); expectedFiles.Add ($"base/lib/{abi}/libxamarin-app.so"); if (usesAssemblyBlobs) { - expectedFiles.Add ($"{blobEntryPrefix}System.Private.CoreLib.dll"); + expectedFiles.Add ($"{blobEntryPrefix}{abi}/System.Private.CoreLib.dll"); } else { - expectedFiles.Add ($"base/root/assemblies/{abi}/System.Private.CoreLib.dll"); + expectedFiles.Add ($"base/lib/{abi}/System.Private.CoreLib.dll.so"); } expectedFiles.Add ($"base/lib/{abi}/libSystem.IO.Compression.Native.so"); expectedFiles.Add ($"base/lib/{abi}/libSystem.Native.so"); From 4630e3c3735dbe26bec71034134ee2214690baeb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 8 Dec 2023 16:48:02 +0100 Subject: [PATCH 044/143] Fix handling of satellite assemblies (can lead to build failures) --- .../Tests/Xamarin.Android.Build.Tests/BuildTest2.cs | 12 ++++++++++-- .../Utilities/MonoAndroidHelper.cs | 8 +++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index 0e4ed3dcd45..452fd5a4539 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -42,11 +42,15 @@ public partial class BuildTest2 : BaseTest public void MarshalMethodsDefaultEnabledStatus (bool isRelease, bool marshalMethodsEnabled) { var abis = new [] { "armeabi-v7a", "x86" }; + AndroidTargetArch[] supportedArches = new [] { + AndroidTargetArch.Arm, + AndroidTargetArch.X86, + }; var proj = new XamarinAndroidApplicationProject { IsRelease = isRelease }; proj.SetProperty (KnownProperties.AndroidEnableMarshalMethods, marshalMethodsEnabled.ToString ()); - proj.SetAndroidSupportedAbis (abis); + proj.SetRuntimeIdentifiers (abis); bool shouldMarshalMethodsBeEnabled = isRelease && marshalMethodsEnabled; using (var b = CreateApkBuilder ()) { @@ -57,7 +61,11 @@ public void MarshalMethodsDefaultEnabledStatus (bool isRelease, bool marshalMeth ); string objPath = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath); - List envFiles = EnvironmentHelper.GatherEnvironmentFiles (objPath, String.Join (";", abis), true); + List envFiles = EnvironmentHelper.GatherEnvironmentFiles ( + objPath, + String.Join (";", supportedArches.Select (arch => MonoAndroidHelper.ArchToAbi (arch))), + true + ); EnvironmentHelper.ApplicationConfig app_config = EnvironmentHelper.ReadApplicationConfig (envFiles); Assert.That (app_config, Is.Not.Null, "application_config must be present in the environment files"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 86c1f90cabe..cfaed2bbfb6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -594,7 +594,13 @@ public static Dictionary> GetPe assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); assembliesPerArch.Add (arch, assemblies); } - assemblies.Add (Path.GetFileName (assembly.ItemSpec), assembly); + + string name = Path.GetFileName (assembly.ItemSpec); + string? culture = assembly.GetMetadata ("Culture"); + if (!String.IsNullOrEmpty (culture)) { + name = $"{culture}/{name}"; + } + assemblies.Add (name, assembly); } if (!validate) { From ef715a4aab5144482779ac34f2452af22685d110 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 8 Dec 2023 16:51:24 +0100 Subject: [PATCH 045/143] Update apkdesc files --- .../BuildReleaseArm64SimpleDotNet.apkdesc | 66 ++-- .../BuildReleaseArm64XFormsDotNet.apkdesc | 292 +++++++++--------- 2 files changed, 179 insertions(+), 179 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 353fa1774d5..f1c724bea93 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -4,71 +4,71 @@ "AndroidManifest.xml": { "Size": 3036 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 1028 + "assemblies/arm64-v8a/_Microsoft.Android.Resource.Designer.dll": { + "Size": 1027 }, - "assemblies/Java.Interop.dll": { - "Size": 61348 + "assemblies/arm64-v8a/Java.Interop.dll": { + "Size": 61441 }, - "assemblies/Mono.Android.dll": { - "Size": 90896 + "assemblies/arm64-v8a/Mono.Android.dll": { + "Size": 90528 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5577 + "assemblies/arm64-v8a/Mono.Android.Runtime.dll": { + "Size": 5520 }, - "assemblies/rc.bin": { - "Size": 1512 - }, - "assemblies/System.Console.dll": { - "Size": 6548 + "assemblies/arm64-v8a/System.Console.dll": { + "Size": 6540 }, - "assemblies/System.Linq.dll": { - "Size": 7171 + "assemblies/arm64-v8a/System.Linq.dll": { + "Size": 7160 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 553674 + "assemblies/arm64-v8a/System.Private.CoreLib.dll": { + "Size": 552250 }, - "assemblies/System.Runtime.dll": { - "Size": 2630 + "assemblies/arm64-v8a/System.Runtime.dll": { + "Size": 2618 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 4033 + "assemblies/arm64-v8a/System.Runtime.InteropServices.dll": { + "Size": 4019 }, - "assemblies/UnnamedProject.dll": { - "Size": 2935 + "assemblies/arm64-v8a/UnnamedProject.dll": { + "Size": 2933 }, "classes.dex": { - "Size": 377956 + "Size": 377312 + }, + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 86944 + "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 334496 + "Size": 340088 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3182072 + "Size": 3183424 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 }, "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94208 + "Size": 94504 }, "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 11680 + "Size": 11888 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1213 + "Size": 1223 }, "META-INF/BNDLTOOL.SF": { - "Size": 3037 + "Size": 3144 }, "META-INF/MANIFEST.MF": { - "Size": 2910 + "Size": 3017 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2783562 + "PackageSize": 2779573 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index e92a694b67e..bd4dba3fd19 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -4,230 +4,230 @@ "AndroidManifest.xml": { "Size": 3572 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 2105 + "assemblies/arm64-v8a/_Microsoft.Android.Resource.Designer.dll": { + "Size": 2104 }, - "assemblies/FormsViewGroup.dll": { - "Size": 7113 + "assemblies/arm64-v8a/FormsViewGroup.dll": { + "Size": 7106 }, - "assemblies/Java.Interop.dll": { - "Size": 69478 + "assemblies/arm64-v8a/Java.Interop.dll": { + "Size": 69522 }, - "assemblies/Mono.Android.dll": { - "Size": 447443 + "assemblies/arm64-v8a/Mono.Android.dll": { + "Size": 445422 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5577 + "assemblies/arm64-v8a/Mono.Android.Runtime.dll": { + "Size": 5520 }, - "assemblies/mscorlib.dll": { - "Size": 3863 + "assemblies/arm64-v8a/mscorlib.dll": { + "Size": 3850 }, - "assemblies/netstandard.dll": { - "Size": 5581 + "assemblies/arm64-v8a/netstandard.dll": { + "Size": 5561 }, - "assemblies/rc.bin": { - "Size": 1512 - }, - "assemblies/System.Collections.Concurrent.dll": { - "Size": 11516 + "assemblies/arm64-v8a/System.Collections.Concurrent.dll": { + "Size": 11498 }, - "assemblies/System.Collections.dll": { - "Size": 15409 + "assemblies/arm64-v8a/System.Collections.dll": { + "Size": 15387 }, - "assemblies/System.Collections.NonGeneric.dll": { - "Size": 7454 + "assemblies/arm64-v8a/System.Collections.NonGeneric.dll": { + "Size": 7440 }, - "assemblies/System.ComponentModel.dll": { - "Size": 1939 + "assemblies/arm64-v8a/System.ComponentModel.dll": { + "Size": 1933 }, - "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2553 + "assemblies/arm64-v8a/System.ComponentModel.Primitives.dll": { + "Size": 2544 }, - "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6037 + "assemblies/arm64-v8a/System.ComponentModel.TypeConverter.dll": { + "Size": 6026 }, - "assemblies/System.Console.dll": { - "Size": 6579 + "assemblies/arm64-v8a/System.Console.dll": { + "Size": 6572 }, - "assemblies/System.Core.dll": { - "Size": 1993 + "assemblies/arm64-v8a/System.Core.dll": { + "Size": 1982 }, - "assemblies/System.Diagnostics.DiagnosticSource.dll": { - "Size": 9071 + "assemblies/arm64-v8a/System.Diagnostics.DiagnosticSource.dll": { + "Size": 9058 }, - "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6555 + "assemblies/arm64-v8a/System.Diagnostics.TraceSource.dll": { + "Size": 6544 }, - "assemblies/System.dll": { - "Size": 2345 + "assemblies/arm64-v8a/System.dll": { + "Size": 2336 }, - "assemblies/System.Drawing.dll": { - "Size": 1940 + "assemblies/arm64-v8a/System.Drawing.dll": { + "Size": 1929 }, - "assemblies/System.Drawing.Primitives.dll": { - "Size": 11975 + "assemblies/arm64-v8a/System.Drawing.Primitives.dll": { + "Size": 11958 }, - "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11194 + "assemblies/arm64-v8a/System.IO.Compression.Brotli.dll": { + "Size": 11185 }, - "assemblies/System.IO.Compression.dll": { - "Size": 15863 + "assemblies/arm64-v8a/System.IO.Compression.dll": { + "Size": 15862 }, - "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9875 + "assemblies/arm64-v8a/System.IO.IsolatedStorage.dll": { + "Size": 9864 }, - "assemblies/System.Linq.dll": { - "Size": 19604 + "assemblies/arm64-v8a/System.Linq.dll": { + "Size": 19571 }, - "assemblies/System.Linq.Expressions.dll": { - "Size": 165118 + "assemblies/arm64-v8a/System.Linq.Expressions.dll": { + "Size": 164630 }, - "assemblies/System.Net.Http.dll": { - "Size": 67636 + "assemblies/arm64-v8a/System.Net.Http.dll": { + "Size": 67509 }, - "assemblies/System.Net.Primitives.dll": { - "Size": 22437 + "assemblies/arm64-v8a/System.Net.Primitives.dll": { + "Size": 22418 }, - "assemblies/System.Net.Requests.dll": { - "Size": 3604 + "assemblies/arm64-v8a/System.Net.Requests.dll": { + "Size": 3586 }, - "assemblies/System.ObjectModel.dll": { - "Size": 8119 + "assemblies/arm64-v8a/System.ObjectModel.dll": { + "Size": 8107 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 848850 + "assemblies/arm64-v8a/System.Private.CoreLib.dll": { + "Size": 846536 }, - "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 193994 + "assemblies/arm64-v8a/System.Private.DataContractSerialization.dll": { + "Size": 193444 }, - "assemblies/System.Private.Uri.dll": { - "Size": 42863 + "assemblies/arm64-v8a/System.Private.Uri.dll": { + "Size": 42798 }, - "assemblies/System.Private.Xml.dll": { - "Size": 216250 + "assemblies/arm64-v8a/System.Private.Xml.dll": { + "Size": 215580 }, - "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16646 + "assemblies/arm64-v8a/System.Private.Xml.Linq.dll": { + "Size": 16622 }, - "assemblies/System.Runtime.dll": { - "Size": 2755 + "assemblies/arm64-v8a/System.Runtime.dll": { + "Size": 2746 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 4033 + "assemblies/arm64-v8a/System.Runtime.InteropServices.dll": { + "Size": 4019 }, - "assemblies/System.Runtime.Serialization.dll": { - "Size": 1866 + "assemblies/arm64-v8a/System.Runtime.Serialization.dll": { + "Size": 1855 }, - "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2485 + "assemblies/arm64-v8a/System.Runtime.Serialization.Formatters.dll": { + "Size": 2477 }, - "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3760 + "assemblies/arm64-v8a/System.Runtime.Serialization.Primitives.dll": { + "Size": 3747 }, - "assemblies/System.Security.Cryptography.dll": { - "Size": 8109 + "assemblies/arm64-v8a/System.Security.Cryptography.dll": { + "Size": 8098 }, - "assemblies/System.Text.RegularExpressions.dll": { - "Size": 159114 + "assemblies/arm64-v8a/System.Text.RegularExpressions.dll": { + "Size": 158792 }, - "assemblies/System.Xml.dll": { - "Size": 1757 + "assemblies/arm64-v8a/System.Xml.dll": { + "Size": 1750 }, - "assemblies/System.Xml.Linq.dll": { - "Size": 1777 + "assemblies/arm64-v8a/System.Xml.Linq.dll": { + "Size": 1765 }, - "assemblies/UnnamedProject.dll": { - "Size": 4990 + "assemblies/arm64-v8a/UnnamedProject.dll": { + "Size": 4983 }, - "assemblies/Xamarin.AndroidX.Activity.dll": { - "Size": 5944 + "assemblies/arm64-v8a/Xamarin.AndroidX.Activity.dll": { + "Size": 5936 }, - "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6036 + "assemblies/arm64-v8a/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { + "Size": 6026 }, - "assemblies/Xamarin.AndroidX.AppCompat.dll": { - "Size": 119847 + "assemblies/arm64-v8a/Xamarin.AndroidX.AppCompat.dll": { + "Size": 119193 }, - "assemblies/Xamarin.AndroidX.CardView.dll": { - "Size": 6802 + "assemblies/arm64-v8a/Xamarin.AndroidX.CardView.dll": { + "Size": 6785 }, - "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { - "Size": 17260 + "assemblies/arm64-v8a/Xamarin.AndroidX.CoordinatorLayout.dll": { + "Size": 17222 }, - "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 100665 + "assemblies/arm64-v8a/Xamarin.AndroidX.Core.dll": { + "Size": 100216 }, - "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 14631 + "assemblies/arm64-v8a/Xamarin.AndroidX.DrawerLayout.dll": { + "Size": 14602 }, - "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 41736 + "assemblies/arm64-v8a/Xamarin.AndroidX.Fragment.dll": { + "Size": 41548 }, - "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { - "Size": 6083 + "assemblies/arm64-v8a/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { + "Size": 6075 }, - "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { - "Size": 6471 + "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.Common.dll": { + "Size": 6464 }, - "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { - "Size": 6618 + "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { + "Size": 6607 }, - "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { - "Size": 3071 + "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { + "Size": 3065 }, - "assemblies/Xamarin.AndroidX.Loader.dll": { - "Size": 12926 + "assemblies/arm64-v8a/Xamarin.AndroidX.Loader.dll": { + "Size": 12907 }, - "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 89997 + "assemblies/arm64-v8a/Xamarin.AndroidX.RecyclerView.dll": { + "Size": 89550 }, - "assemblies/Xamarin.AndroidX.SavedState.dll": { - "Size": 4909 + "assemblies/arm64-v8a/Xamarin.AndroidX.SavedState.dll": { + "Size": 4905 }, - "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 10575 + "assemblies/arm64-v8a/Xamarin.AndroidX.SwipeRefreshLayout.dll": { + "Size": 10558 }, - "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 18594 + "assemblies/arm64-v8a/Xamarin.AndroidX.ViewPager.dll": { + "Size": 18553 }, - "assemblies/Xamarin.Forms.Core.dll": { - "Size": 528450 + "assemblies/arm64-v8a/Xamarin.Forms.Core.dll": { + "Size": 526687 }, - "assemblies/Xamarin.Forms.Platform.Android.dll": { - "Size": 337925 + "assemblies/arm64-v8a/Xamarin.Forms.Platform.Android.dll": { + "Size": 336895 }, - "assemblies/Xamarin.Forms.Platform.dll": { - "Size": 11083 + "assemblies/arm64-v8a/Xamarin.Forms.Platform.dll": { + "Size": 11076 }, - "assemblies/Xamarin.Forms.Xaml.dll": { - "Size": 60774 + "assemblies/arm64-v8a/Xamarin.Forms.Xaml.dll": { + "Size": 60656 }, - "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 42285 + "assemblies/arm64-v8a/Xamarin.Google.Android.Material.dll": { + "Size": 42137 }, "classes.dex": { - "Size": 3514468 + "Size": 3291740 + }, + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 86944 + "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 334496 + "Size": 340088 }, "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3182072 + "Size": 3183424 }, "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { "Size": 723560 }, "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94208 + "Size": 94504 }, "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 102920 + "Size": 103128 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -338,16 +338,16 @@ "Size": 6 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1213 + "Size": 1223 }, "META-INF/BNDLTOOL.SF": { - "Size": 77024 + "Size": 77700 }, "META-INF/com.google.android.material_material.version": { "Size": 10 }, "META-INF/MANIFEST.MF": { - "Size": 76897 + "Size": 77573 }, "META-INF/proguard/androidx-annotations.pro": { "Size": 339 @@ -1886,7 +1886,7 @@ "Size": 440 }, "res/layout/rootlayout.xml": { - "Size": 1352 + "Size": 1368 }, "res/layout/select_dialog_item_material.xml": { "Size": 640 @@ -1916,5 +1916,5 @@ "Size": 325240 } }, - "PackageSize": 7949326 + "PackageSize": 7888553 } \ No newline at end of file From b7ce0e68728bf7921f6e14c00d450e1980fcdcbc Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 8 Dec 2023 17:54:19 +0100 Subject: [PATCH 046/143] Fix JCW generation --- src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index cfaed2bbfb6..d177688555d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -595,7 +595,7 @@ public static Dictionary> GetPe assembliesPerArch.Add (arch, assemblies); } - string name = Path.GetFileName (assembly.ItemSpec); + string name = Path.GetFileNameWithoutExtension (assembly.ItemSpec); string? culture = assembly.GetMetadata ("Culture"); if (!String.IsNullOrEmpty (culture)) { name = $"{culture}/{name}"; From 99f86d9a549ddf9284195b6c3b8b97b9af47a900 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 11 Dec 2023 22:36:37 +0100 Subject: [PATCH 047/143] Some test fixes Satellite assemblies are a bummer... When discrete assembly files are placed in the apk, they end up in `lib/{ABI}`, but that directory mustn't have any subdirectories, so `lib/{ABI}/{CULTURE}/assembly.dll.so` is out of question. To work around that, we need to mangle the satellite assembly name somehow and that breaks image loader. Will figure out how to fix this tomorrow. --- .../Tasks/BuildApk.cs | 39 +++- .../Xamarin.Android.Build.Tests/AotTests.cs | 4 +- .../PackagingTest.cs | 10 +- .../Tasks/LinkerTests.cs | 47 ++-- .../Utilities/ArchiveAssemblyHelper.cs | 95 +++++---- .../BuildReleaseArm64SimpleDotNet.apkdesc | 72 +++---- .../BuildReleaseArm64XFormsDotNet.apkdesc | 200 +++++++++--------- 7 files changed, 259 insertions(+), 208 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index b2342237cc1..385faf4a43e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -604,21 +604,40 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) { var parts = new List (); - // bool haveRootPath = !String.IsNullOrEmpty (RootPath); - // if (haveRootPath) { - // parts.Add (ArchiveAssembliesPath); - // } - - // There's no need anymore to treat satellite assemblies specially here. The PrepareSatelliteAssemblies task takes care of - // properly setting `DestinationSubDirectory`, so we can just use it here. - string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory"); + // The PrepareSatelliteAssemblies task takes care of properly setting `DestinationSubDirectory`, so we can just use it here. + string? subDirectory = assembly.GetMetadata ("DestinationSubDirectory")?.Replace ('\\', '/'); if (string.IsNullOrEmpty (subDirectory)) { throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); } - parts.Add (subDirectory.Replace ('\\', '/')); - return MonoAndroidHelper.MakeZipArchivePath (/*haveRootPath ? RootPath : */ArchiveAssembliesPath, parts) + "/"; + bool needTrailingSlash = true; + if (UseAssemblyStore) { + parts.Add (subDirectory); + } else { + // For discrete assembly entries we need to treat satellite assemblies specially. + // If we encounter one of them, we will return the culture as part of the path transformed + // so that it forms a `_culture_` assembly file name prefix, not a `culture/` subdirectory. + // This is necessary because Android doesn't allow subdirectories in `lib/{ABI}/` + + string[] subdirParts = subDirectory.TrimEnd ('/').Split ('/'); + if (subdirParts.Length == 1) { + // Not a satellite assembly + parts.Add (subDirectory); + } else if (subdirParts.Length == 2) { + parts.Add ($"{subdirParts[0]}/_{subdirParts[1]}_"); + needTrailingSlash = false; + } else { + throw new InvalidOperationException ($"Internal error: '{assembly}' `DestinationSubDirectory` metadata has too many components ({parts.Count} instead of 1 or 2)"); + } + } + + string ret = MonoAndroidHelper.MakeZipArchivePath (ArchiveAssembliesPath, parts); + if (needTrailingSlash) { + return ret + "/"; + } + + return ret; } sealed class LibInfo diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index 0b89a082870..1dd5a0bd167 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -205,7 +205,7 @@ public void BuildAotApplicationWithNdkAndBundleAndÜmläüts (string supportedAb proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ("assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), @@ -250,7 +250,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/UnnamedProject.dll"), $"UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ("assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index 54c0ee636f5..b85f6147908 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -484,7 +484,10 @@ public void MissingSatelliteAssemblyInLibrary () var helper = new ArchiveAssemblyHelper (apk); foreach (string lang in languages) { - Assert.IsTrue (helper.Exists ($"assemblies/{lang}/{lib.ProjectName}.resources.dll"), $"Apk should contain satellite assembly for language '{lang}'!"); + foreach (string rid in appBuilder.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/{lang}/{lib.ProjectName}.resources.dll"), $"Apk should contain satellite assembly for language '{lang}'!"); + } } } } @@ -513,7 +516,10 @@ public void MissingSatelliteAssemblyInApp () var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk); - Assert.IsTrue (helper.Exists ($"assemblies/es/{proj.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/es/{proj.ProjectName}.resources.dll"), "Apk should contain satellite assemblies!"); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 80b92522828..6e6695ae92a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -9,6 +9,7 @@ using Mono.Tuner; using MonoDroid.Tuner; using NUnit.Framework; +using Xamarin.Android.Tasks; using Xamarin.ProjectTools; using SR = System.Reflection; @@ -228,26 +229,29 @@ public void RemoveDesigner ([Values (true, false)] bool useAssemblyStore) proj.SetProperty ("AndroidLinkResources", "True"); proj.SetProperty ("AndroidUseAssemblyStore", useAssemblyStore.ToString ()); string assemblyName = proj.ProjectName; - using (var b = CreateApkBuilder ()) { - Assert.IsTrue (b.Build (proj), "build should have succeeded."); - var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); - FileAssert.Exists (apk); - var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); - Assert.IsTrue (helper.Exists ($"assemblies/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); - using (var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll")) { - stream.Position = 0; - using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { - var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); - var intType = typeof(int); - foreach (var nestedType in type.NestedTypes) { - int count = 0; - foreach (var field in nestedType.Fields) { - if (field.FieldType.FullName == intType.FullName) - count++; - } - Assert.AreEqual (0, count, "All Nested Resource Type int fields should be removed."); - } + + using var b = CreateApkBuilder (); + Assert.IsTrue (b.Build (proj), "build should have succeeded."); + var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); + FileAssert.Exists (apk); + var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/{assemblyName}.dll"), $"{assemblyName}.dll should exist in apk!"); + + using var stream = helper.ReadEntry ($"assemblies/{assemblyName}.dll"); + stream.Position = 0; + + using var assembly = AssemblyDefinition.ReadAssembly (stream); + var type = assembly.MainModule.GetType ($"{assemblyName}.Resource"); + var intType = typeof(int); + foreach (var nestedType in type.NestedTypes) { + int count = 0; + foreach (var field in nestedType.Fields) { + if (field.FieldType.FullName == intType.FullName) + count++; } + Assert.AreEqual (0, count, "All Nested Resource Type int fields should be removed."); } } } @@ -288,7 +292,10 @@ public void LinkDescription ([Values (true, false)] bool useAssemblyStore) var apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); FileAssert.Exists (apk); var helper = new ArchiveAssemblyHelper (apk, useAssemblyStore); - Assert.IsTrue (helper.Exists ($"assemblies/{assembly_name}.dll"), $"{assembly_name}.dll should exist in apk!"); + foreach (string rid in b.GetBuildRuntimeIdentifiers ()) { + string abi = MonoAndroidHelper.RidToAbi (rid); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/{assembly_name}.dll"), $"{assembly_name}.dll should exist in apk!"); + } using (var stream = helper.ReadEntry ($"assemblies/{assembly_name}.dll")) { stream.Position = 0; using (var assembly = AssemblyDefinition.ReadAssembly (stream)) { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 90f9e32136c..438659617ca 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -45,11 +45,11 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, string extension = Path.GetExtension (archivePath) ?? String.Empty; if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "base/root/assemblies"; + assembliesRootDir = "base/lib/assemblies"; } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "assemblies/"; + assembliesRootDir = "lib/"; } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "root/assemblies/"; + assembliesRootDir = "lib/"; } else { assembliesRootDir = String.Empty; } @@ -253,20 +253,17 @@ public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool force /// /// Takes "old style" `assemblies/assembly.dll` path and returns (if possible) a set of paths that reflect the new - /// location of `assemblies/{ARCH}/assembly.dll`. A list is returned because, if `arch` is `None`, we'll return all + /// location of `lib/{ARCH}/assembly.dll.so`. A list is returned because, if `arch` is `None`, we'll return all /// the possible architectural paths. /// An exception is thrown if we cannot transform the path for some reason. It should **not** be handled. /// static List? TransformArchiveAssemblyPath (string path, AndroidTargetArch arch) { - const string AssembliesPath = "assemblies"; - const string AssembliesPathTerminated = AssembliesPath + "/"; - if (String.IsNullOrEmpty (path)) { throw new ArgumentException (nameof (path), "must not be null or empty"); } - if (!path.StartsWith (AssembliesPathTerminated, StringComparison.Ordinal)) { + if (!path.StartsWith ("assemblies/", StringComparison.Ordinal)) { return new List { path }; } @@ -278,52 +275,70 @@ public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool force // We accept: // assemblies/assembly.dll // assemblies/{CULTURE}/assembly.dll - // assemblies/{ARCH}/assembly.dll - // assemblies/{ARCH}/{CULTURE}/assembly.dll + // assemblies/{ABI}/assembly.dll + // assemblies/{ABI}/{CULTURE}/assembly.dll if (parts.Length > 4) { throw new InvalidOperationException ($"Path '{path}' must not consist of more than 4 segments separated by `/`"); } - var ret = new List (); - if (parts.Length == 4) { - // It's a full satellite assembly path that includes the ABI, no need to change anything - ret.Add (path); - return ret; + string? fileName = null; + string? culture = null; + string? abi = null; + + switch (parts.Length) { + // Full satellite assembly path, with abi + case 4: + abi = parts[1]; + culture = parts[2]; + fileName = parts[3]; + break; + + // Assembly path with abi or culture + case 3: + // If the middle part isn't a valid abi, we treat it as a culture name + if (MonoAndroidHelper.IsValidAbi (parts[1])) { + abi = parts[1]; + } else { + culture = parts[1]; + } + fileName = parts[2]; + break; + + // Assembly path without abi or culture + case 2: + fileName = parts[1]; + break; } - if (parts.Length == 3) { - // We need to check whether the middle part is a culture or an ABI - if (MonoAndroidHelper.IsValidAbi (parts[1])) { - // Nothing more to do - ret.Add (path); - return ret; + var abis = new List (); + if (!String.IsNullOrEmpty (abi)) { + abis.Add (abi); + } else if (arch == AndroidTargetArch.None) { + foreach (AndroidTargetArch targetArch in MonoAndroidHelper.SupportedTargetArchitectures) { + abis.Add (MonoAndroidHelper.ArchToAbi (targetArch)); } + } else { + abis.Add (MonoAndroidHelper.ArchToAbi (arch)); } - // We need to add the ABI(s) + if (!String.IsNullOrEmpty (culture)) { + // Android doesn't allow us to put satellite assemblies in lib/{CULTURE}/assembly.dll.so, we must instead + // mangle the name. + fileName = $"_{culture}_{fileName}"; + } + + var ret = new List (); var newParts = new List { String.Empty, // ABI placeholder + $"{fileName}.so", }; - for (int i = 1; i < parts.Length; i++) { - newParts.Add (parts[i]); - } - - if (arch != AndroidTargetArch.None) { - ret.Add (MakeAbiArchivePath (arch)); - } else { - foreach (AndroidTargetArch targetArch in MonoAndroidHelper.SupportedTargetArchitectures) { - ret.Add (MakeAbiArchivePath (targetArch)); - } + foreach (string a in abis) { + newParts[0] = a; + ret.Add (MonoAndroidHelper.MakeZipArchivePath ("lib", newParts)); } return ret; - - string MakeAbiArchivePath (AndroidTargetArch targetArch) - { - newParts[0] = MonoAndroidHelper.ArchToAbi (targetArch); - return MonoAndroidHelper.MakeZipArchivePath (AssembliesPath, newParts); - } } static bool ArchiveContains (List archiveContents, string entryPath, AndroidTargetArch arch) @@ -348,6 +363,10 @@ static bool ArchiveContains (List archiveContents, string entryPath, And return false; } + /// + /// Checks whether exists in the archive or assembly store. The path should use the + /// "old style" `assemblies/{ABI}/assembly.dll` format. + /// public bool Exists (string entryPath, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) { return ArchiveContains (ListArchiveContents (assembliesRootDir, forceRefresh), entryPath, arch); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index f1c724bea93..7d277007391 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -4,47 +4,23 @@ "AndroidManifest.xml": { "Size": 3036 }, - "assemblies/arm64-v8a/_Microsoft.Android.Resource.Designer.dll": { - "Size": 1027 - }, - "assemblies/arm64-v8a/Java.Interop.dll": { - "Size": 61441 - }, - "assemblies/arm64-v8a/Mono.Android.dll": { - "Size": 90528 - }, - "assemblies/arm64-v8a/Mono.Android.Runtime.dll": { - "Size": 5520 - }, - "assemblies/arm64-v8a/System.Console.dll": { - "Size": 6540 - }, - "assemblies/arm64-v8a/System.Linq.dll": { - "Size": 7160 - }, - "assemblies/arm64-v8a/System.Private.CoreLib.dll": { - "Size": 552250 - }, - "assemblies/arm64-v8a/System.Runtime.dll": { - "Size": 2618 - }, - "assemblies/arm64-v8a/System.Runtime.InteropServices.dll": { - "Size": 4019 - }, - "assemblies/arm64-v8a/UnnamedProject.dll": { - "Size": 2933 - }, "classes.dex": { - "Size": 377312 + "Size": 377904 + }, + "lib/arm64-v8a/_Microsoft.Android.Resource.Designer.dll.so": { + "Size": 1027 }, "lib/arm64-v8a/arc.bin.so": { "Size": 1512 }, + "lib/arm64-v8a/Java.Interop.dll.so": { + "Size": 61441 + }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 340088 + "Size": 340136 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3183424 @@ -59,16 +35,40 @@ "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 11888 + "Size": 12016 + }, + "lib/arm64-v8a/Mono.Android.dll.so": { + "Size": 90523 + }, + "lib/arm64-v8a/Mono.Android.Runtime.dll.so": { + "Size": 5514 + }, + "lib/arm64-v8a/System.Console.dll.so": { + "Size": 6540 + }, + "lib/arm64-v8a/System.Linq.dll.so": { + "Size": 7160 + }, + "lib/arm64-v8a/System.Private.CoreLib.dll.so": { + "Size": 552250 + }, + "lib/arm64-v8a/System.Runtime.dll.so": { + "Size": 2618 + }, + "lib/arm64-v8a/System.Runtime.InteropServices.dll.so": { + "Size": 4019 + }, + "lib/arm64-v8a/UnnamedProject.dll.so": { + "Size": 2931 }, "META-INF/BNDLTOOL.RSA": { "Size": 1223 }, "META-INF/BNDLTOOL.SF": { - "Size": 3144 + "Size": 3104 }, "META-INF/MANIFEST.MF": { - "Size": 3017 + "Size": 2977 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2779573 + "PackageSize": 2656653 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index bd4dba3fd19..2cfb8eba153 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -4,231 +4,231 @@ "AndroidManifest.xml": { "Size": 3572 }, - "assemblies/arm64-v8a/_Microsoft.Android.Resource.Designer.dll": { + "classes.dex": { + "Size": 3514416 + }, + "lib/arm64-v8a/_Microsoft.Android.Resource.Designer.dll.so": { "Size": 2104 }, - "assemblies/arm64-v8a/FormsViewGroup.dll": { + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 + }, + "lib/arm64-v8a/FormsViewGroup.dll.so": { "Size": 7106 }, - "assemblies/arm64-v8a/Java.Interop.dll": { + "lib/arm64-v8a/Java.Interop.dll.so": { "Size": 69522 }, - "assemblies/arm64-v8a/Mono.Android.dll": { - "Size": 445422 + "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { + "Size": 87080 }, - "assemblies/arm64-v8a/Mono.Android.Runtime.dll": { - "Size": 5520 + "lib/arm64-v8a/libmonodroid.so": { + "Size": 340136 + }, + "lib/arm64-v8a/libmonosgen-2.0.so": { + "Size": 3183424 }, - "assemblies/arm64-v8a/mscorlib.dll": { + "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { + "Size": 723560 + }, + "lib/arm64-v8a/libSystem.Native.so": { + "Size": 94504 + }, + "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { + "Size": 155568 + }, + "lib/arm64-v8a/libxamarin-app.so": { + "Size": 103912 + }, + "lib/arm64-v8a/Mono.Android.dll.so": { + "Size": 445420 + }, + "lib/arm64-v8a/Mono.Android.Runtime.dll.so": { + "Size": 5514 + }, + "lib/arm64-v8a/mscorlib.dll.so": { "Size": 3850 }, - "assemblies/arm64-v8a/netstandard.dll": { + "lib/arm64-v8a/netstandard.dll.so": { "Size": 5561 }, - "assemblies/arm64-v8a/System.Collections.Concurrent.dll": { + "lib/arm64-v8a/System.Collections.Concurrent.dll.so": { "Size": 11498 }, - "assemblies/arm64-v8a/System.Collections.dll": { + "lib/arm64-v8a/System.Collections.dll.so": { "Size": 15387 }, - "assemblies/arm64-v8a/System.Collections.NonGeneric.dll": { + "lib/arm64-v8a/System.Collections.NonGeneric.dll.so": { "Size": 7440 }, - "assemblies/arm64-v8a/System.ComponentModel.dll": { + "lib/arm64-v8a/System.ComponentModel.dll.so": { "Size": 1933 }, - "assemblies/arm64-v8a/System.ComponentModel.Primitives.dll": { + "lib/arm64-v8a/System.ComponentModel.Primitives.dll.so": { "Size": 2544 }, - "assemblies/arm64-v8a/System.ComponentModel.TypeConverter.dll": { + "lib/arm64-v8a/System.ComponentModel.TypeConverter.dll.so": { "Size": 6026 }, - "assemblies/arm64-v8a/System.Console.dll": { + "lib/arm64-v8a/System.Console.dll.so": { "Size": 6572 }, - "assemblies/arm64-v8a/System.Core.dll": { + "lib/arm64-v8a/System.Core.dll.so": { "Size": 1982 }, - "assemblies/arm64-v8a/System.Diagnostics.DiagnosticSource.dll": { + "lib/arm64-v8a/System.Diagnostics.DiagnosticSource.dll.so": { "Size": 9058 }, - "assemblies/arm64-v8a/System.Diagnostics.TraceSource.dll": { + "lib/arm64-v8a/System.Diagnostics.TraceSource.dll.so": { "Size": 6544 }, - "assemblies/arm64-v8a/System.dll": { + "lib/arm64-v8a/System.dll.so": { "Size": 2336 }, - "assemblies/arm64-v8a/System.Drawing.dll": { + "lib/arm64-v8a/System.Drawing.dll.so": { "Size": 1929 }, - "assemblies/arm64-v8a/System.Drawing.Primitives.dll": { + "lib/arm64-v8a/System.Drawing.Primitives.dll.so": { "Size": 11958 }, - "assemblies/arm64-v8a/System.IO.Compression.Brotli.dll": { + "lib/arm64-v8a/System.IO.Compression.Brotli.dll.so": { "Size": 11185 }, - "assemblies/arm64-v8a/System.IO.Compression.dll": { + "lib/arm64-v8a/System.IO.Compression.dll.so": { "Size": 15862 }, - "assemblies/arm64-v8a/System.IO.IsolatedStorage.dll": { + "lib/arm64-v8a/System.IO.IsolatedStorage.dll.so": { "Size": 9864 }, - "assemblies/arm64-v8a/System.Linq.dll": { + "lib/arm64-v8a/System.Linq.dll.so": { "Size": 19571 }, - "assemblies/arm64-v8a/System.Linq.Expressions.dll": { + "lib/arm64-v8a/System.Linq.Expressions.dll.so": { "Size": 164630 }, - "assemblies/arm64-v8a/System.Net.Http.dll": { + "lib/arm64-v8a/System.Net.Http.dll.so": { "Size": 67509 }, - "assemblies/arm64-v8a/System.Net.Primitives.dll": { + "lib/arm64-v8a/System.Net.Primitives.dll.so": { "Size": 22418 }, - "assemblies/arm64-v8a/System.Net.Requests.dll": { + "lib/arm64-v8a/System.Net.Requests.dll.so": { "Size": 3586 }, - "assemblies/arm64-v8a/System.ObjectModel.dll": { + "lib/arm64-v8a/System.ObjectModel.dll.so": { "Size": 8107 }, - "assemblies/arm64-v8a/System.Private.CoreLib.dll": { + "lib/arm64-v8a/System.Private.CoreLib.dll.so": { "Size": 846536 }, - "assemblies/arm64-v8a/System.Private.DataContractSerialization.dll": { + "lib/arm64-v8a/System.Private.DataContractSerialization.dll.so": { "Size": 193444 }, - "assemblies/arm64-v8a/System.Private.Uri.dll": { + "lib/arm64-v8a/System.Private.Uri.dll.so": { "Size": 42798 }, - "assemblies/arm64-v8a/System.Private.Xml.dll": { + "lib/arm64-v8a/System.Private.Xml.dll.so": { "Size": 215580 }, - "assemblies/arm64-v8a/System.Private.Xml.Linq.dll": { + "lib/arm64-v8a/System.Private.Xml.Linq.dll.so": { "Size": 16622 }, - "assemblies/arm64-v8a/System.Runtime.dll": { + "lib/arm64-v8a/System.Runtime.dll.so": { "Size": 2746 }, - "assemblies/arm64-v8a/System.Runtime.InteropServices.dll": { + "lib/arm64-v8a/System.Runtime.InteropServices.dll.so": { "Size": 4019 }, - "assemblies/arm64-v8a/System.Runtime.Serialization.dll": { + "lib/arm64-v8a/System.Runtime.Serialization.dll.so": { "Size": 1855 }, - "assemblies/arm64-v8a/System.Runtime.Serialization.Formatters.dll": { + "lib/arm64-v8a/System.Runtime.Serialization.Formatters.dll.so": { "Size": 2477 }, - "assemblies/arm64-v8a/System.Runtime.Serialization.Primitives.dll": { + "lib/arm64-v8a/System.Runtime.Serialization.Primitives.dll.so": { "Size": 3747 }, - "assemblies/arm64-v8a/System.Security.Cryptography.dll": { + "lib/arm64-v8a/System.Security.Cryptography.dll.so": { "Size": 8098 }, - "assemblies/arm64-v8a/System.Text.RegularExpressions.dll": { + "lib/arm64-v8a/System.Text.RegularExpressions.dll.so": { "Size": 158792 }, - "assemblies/arm64-v8a/System.Xml.dll": { + "lib/arm64-v8a/System.Xml.dll.so": { "Size": 1750 }, - "assemblies/arm64-v8a/System.Xml.Linq.dll": { + "lib/arm64-v8a/System.Xml.Linq.dll.so": { "Size": 1765 }, - "assemblies/arm64-v8a/UnnamedProject.dll": { + "lib/arm64-v8a/UnnamedProject.dll.so": { "Size": 4983 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Activity.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Activity.dll.so": { "Size": 5936 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.AppCompat.AppCompatResources.dll.so": { "Size": 6026 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.AppCompat.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.AppCompat.dll.so": { "Size": 119193 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.CardView.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.CardView.dll.so": { "Size": 6785 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.CoordinatorLayout.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.CoordinatorLayout.dll.so": { "Size": 17222 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Core.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Core.dll.so": { "Size": 100216 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.DrawerLayout.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.DrawerLayout.dll.so": { "Size": 14602 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Fragment.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Fragment.dll.so": { "Size": 41548 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Legacy.Support.Core.UI.dll.so": { "Size": 6075 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.Common.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Lifecycle.Common.dll.so": { "Size": 6464 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll.so": { "Size": 6607 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Lifecycle.ViewModel.dll.so": { "Size": 3065 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.Loader.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.Loader.dll.so": { "Size": 12907 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.RecyclerView.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.RecyclerView.dll.so": { "Size": 89550 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.SavedState.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.SavedState.dll.so": { "Size": 4905 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.SwipeRefreshLayout.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.SwipeRefreshLayout.dll.so": { "Size": 10558 }, - "assemblies/arm64-v8a/Xamarin.AndroidX.ViewPager.dll": { + "lib/arm64-v8a/Xamarin.AndroidX.ViewPager.dll.so": { "Size": 18553 }, - "assemblies/arm64-v8a/Xamarin.Forms.Core.dll": { + "lib/arm64-v8a/Xamarin.Forms.Core.dll.so": { "Size": 526687 }, - "assemblies/arm64-v8a/Xamarin.Forms.Platform.Android.dll": { + "lib/arm64-v8a/Xamarin.Forms.Platform.Android.dll.so": { "Size": 336895 }, - "assemblies/arm64-v8a/Xamarin.Forms.Platform.dll": { + "lib/arm64-v8a/Xamarin.Forms.Platform.dll.so": { "Size": 11076 }, - "assemblies/arm64-v8a/Xamarin.Forms.Xaml.dll": { + "lib/arm64-v8a/Xamarin.Forms.Xaml.dll.so": { "Size": 60656 }, - "assemblies/arm64-v8a/Xamarin.Google.Android.Material.dll": { + "lib/arm64-v8a/Xamarin.Google.Android.Material.dll.so": { "Size": 42137 }, - "classes.dex": { - "Size": 3291740 - }, - "lib/arm64-v8a/arc.bin.so": { - "Size": 1512 - }, - "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { - "Size": 87080 - }, - "lib/arm64-v8a/libmonodroid.so": { - "Size": 340088 - }, - "lib/arm64-v8a/libmonosgen-2.0.so": { - "Size": 3183424 - }, - "lib/arm64-v8a/libSystem.IO.Compression.Native.so": { - "Size": 723560 - }, - "lib/arm64-v8a/libSystem.Native.so": { - "Size": 94504 - }, - "lib/arm64-v8a/libSystem.Security.Cryptography.Native.Android.so": { - "Size": 155568 - }, - "lib/arm64-v8a/libxamarin-app.so": { - "Size": 103128 - }, "META-INF/android.support.design_material.version": { "Size": 12 }, @@ -338,16 +338,16 @@ "Size": 6 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1223 + "Size": 1221 }, "META-INF/BNDLTOOL.SF": { - "Size": 77700 + "Size": 77430 }, "META-INF/com.google.android.material_material.version": { "Size": 10 }, "META-INF/MANIFEST.MF": { - "Size": 77573 + "Size": 77303 }, "META-INF/proguard/androidx-annotations.pro": { "Size": 339 @@ -1886,7 +1886,7 @@ "Size": 440 }, "res/layout/rootlayout.xml": { - "Size": 1368 + "Size": 1352 }, "res/layout/select_dialog_item_material.xml": { "Size": 640 @@ -1916,5 +1916,5 @@ "Size": 325240 } }, - "PackageSize": 7888553 + "PackageSize": 7277985 } \ No newline at end of file From 776ab5ffa05bf8a44a3ab602bdce395e7e034ca4 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 12 Dec 2023 18:33:25 +0100 Subject: [PATCH 048/143] Mangle assembly entry names in `lib/{ABI}/` Assemblies are mangled by prepending a "marker" character, `#` for regular assemblies and `%` for the satellite ones. The latter is followed by `{CULTURE}%` so that the final satellite assembly name looks similar to `%es%Mono.Android.dll.so`. This is done because Android doesn't support subdirectories under `lib/{ABI}` and we must "encode" the culture name somehow. The "markers" for regular and satellite assemblies will be important for improving performance at application startup, since they will allow us to skip `lib/{ABI}/` entries that are not assemblies (or their debug symbols and config files), additionally allowing us to quickly morph satellite assembly name to its canonical `culture/Assembly.dll` form. --- .../Tasks/BuildApk.cs | 49 ++++++++++++------- .../Utilities/ArchiveAssemblyHelper.cs | 33 ++++++++++--- .../Utilities/AssemblyStoreAssemblyInfo.cs | 31 +++++++----- .../Utilities/AssemblyStoreGenerator.cs | 6 +-- .../Utilities/MonoAndroidHelper.Basic.cs | 16 ++++++ src/monodroid/jni/xamarin-app.hh | 2 +- 6 files changed, 93 insertions(+), 44 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 385faf4a43e..17f5748b99d 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -461,11 +461,11 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) sourcePath = CompressAssembly (assembly); // Add assembly - var assemblyPath = GetInArchiveAssemblyPath (assembly, frameworkAssembly: false); + (string assemblyPath, string assemblyDirectory) = GetInArchiveAssemblyPath (assembly); if (UseAssemblyStore) { - storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assemblyPath, assembly); + storeAssemblyInfo = new AssemblyStoreAssemblyInfo (sourcePath, assembly); } else { - AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath + Path.GetFileName (assembly.ItemSpec) + ".so", compressionMethod: UncompressedMethod); + AddFileToArchiveIfNewer (apk, sourcePath, assemblyPath, compressionMethod: UncompressedMethod); } // Try to add config if exists @@ -475,7 +475,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) storeAssemblyInfo.ConfigFile = new FileInfo (config); } } else { - AddAssemblyConfigEntry (apk, assemblyPath, config + ".so"); + AddAssemblyConfigEntry (apk, assemblyDirectory, config); } // Try to add symbols if Debug @@ -491,7 +491,12 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) if (UseAssemblyStore) { storeAssemblyInfo.SymbolsFile = new FileInfo (symbolsPath); } else { - AddFileToArchiveIfNewer (apk, symbolsPath, assemblyPath + Path.GetFileName (symbols) + ".so", compressionMethod: UncompressedMethod); + AddFileToArchiveIfNewer ( + apk, + symbolsPath, + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyDirectory + Path.GetFileName (symbols)), + compressionMethod: UncompressedMethod + ); } } } @@ -576,11 +581,12 @@ bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePat void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string configFile) { - string inArchivePath = assemblyPath + Path.GetFileName (configFile); + string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile)); existingEntries.Remove (inArchivePath); - if (!File.Exists (configFile)) + if (!File.Exists (configFile)) { return; + } CompressionMethod compressionMethod = UncompressedMethod; if (apk.SkipExistingFile (configFile, inArchivePath, compressionMethod)) { @@ -601,7 +607,7 @@ void AddAssemblyConfigEntry (ZipArchiveEx apk, string assemblyPath, string confi /// /// Returns the in-archive path for an assembly /// - string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) + (string assemblyFilePath, string assemblyDirectoryPath) GetInArchiveAssemblyPath (ITaskItem assembly) { var parts = new List (); @@ -611,33 +617,38 @@ string GetInArchiveAssemblyPath (ITaskItem assembly, bool frameworkAssembly) throw new InvalidOperationException ($"Internal error: assembly '{assembly}' lacks the required `DestinationSubDirectory` metadata"); } - bool needTrailingSlash = true; + string assemblyName = Path.GetFileName (assembly.ItemSpec); if (UseAssemblyStore) { parts.Add (subDirectory); + parts.Add (assemblyName); } else { - // For discrete assembly entries we need to treat satellite assemblies specially. + // For discrete assembly entries we need to treat assemblies specially. + // All of the assemblies have their names mangled so that the possibility to clash with "real" shared + // library names is minimized. All of the assembly entries will start with a special character: + // + // # - for regular assemblies (e.g. `#Mono.Android.dll.so`) + // % - for satellite assemblies (e.g. `%es%Mono.Android.dll.so`) + // + // Second of all, we need to treat satellite assemblies with even more care. // If we encounter one of them, we will return the culture as part of the path transformed - // so that it forms a `_culture_` assembly file name prefix, not a `culture/` subdirectory. + // so that it forms a `%culture%` assembly file name prefix, not a `culture/` subdirectory. // This is necessary because Android doesn't allow subdirectories in `lib/{ABI}/` string[] subdirParts = subDirectory.TrimEnd ('/').Split ('/'); if (subdirParts.Length == 1) { // Not a satellite assembly parts.Add (subDirectory); + parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName)); } else if (subdirParts.Length == 2) { - parts.Add ($"{subdirParts[0]}/_{subdirParts[1]}_"); - needTrailingSlash = false; + parts.Add (subdirParts[0]); + parts.Add (MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyName, subdirParts[1])); } else { throw new InvalidOperationException ($"Internal error: '{assembly}' `DestinationSubDirectory` metadata has too many components ({parts.Count} instead of 1 or 2)"); } } - string ret = MonoAndroidHelper.MakeZipArchivePath (ArchiveAssembliesPath, parts); - if (needTrailingSlash) { - return ret + "/"; - } - - return ret; + string assemblyFilePath = MonoAndroidHelper.MakeZipArchivePath (ArchiveAssembliesPath, parts); + return (assemblyFilePath, Path.GetDirectoryName (assemblyFilePath) + "/"); } sealed class LibInfo diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 438659617ca..1c5a683cebd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -189,13 +189,27 @@ void SynthetizeAssemblies (AssemblyStoreExplorer? explorer) string abi = MonoAndroidHelper.ArchToAbi (asm.TargetArch); prefix = $"{prefix}{abi}/"; - entries.Add ($"{prefix}{asm.Name}"); + int cultureIndex = asm.Name.IndexOf ('/'); + string? culture = null; + string name; + + if (cultureIndex > 0) { + culture = asm.Name.Substring (0, cultureIndex); + name = asm.Name.Substring (cultureIndex + 1); + } else { + name = asm.Name; + } + + // Mangle name in in the same fashion the discrete assembly entries are named, makes other + // code in this class simpler. + string mangledName = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (name, culture); + entries.Add ($"{prefix}{mangledName}"); if (asm.DebugOffset > 0) { - entries.Add ($"{prefix}{Path.GetFileNameWithoutExtension (asm.Name)}.pdb"); + entries.Add ($"{prefix}{Path.ChangeExtension (mangledName, "pdb")}"); } if (asm.ConfigOffset > 0) { - entries.Add ($"{prefix}{asm.Name}.config"); + entries.Add ($"{prefix}{Path.ChangeExtension (mangledName, "config")}"); } } } @@ -233,7 +247,7 @@ void SynthetizeAssemblies (AssemblyStoreExplorer? explorer) public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) { List contents = ListArchiveContents (assembliesRootDir, forceRefresh); - var dlls = contents.Where (x => x.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)); + var dlls = contents.Where (x => x.EndsWith (".dll.so", StringComparison.OrdinalIgnoreCase)); if (!countAbiAssembliesOnce) { return dlls.Count (); @@ -310,6 +324,7 @@ public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool force break; } + char fileTypeMarker = '#'; var abis = new List (); if (!String.IsNullOrEmpty (abi)) { abis.Add (abi); @@ -324,13 +339,14 @@ public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool force if (!String.IsNullOrEmpty (culture)) { // Android doesn't allow us to put satellite assemblies in lib/{CULTURE}/assembly.dll.so, we must instead // mangle the name. - fileName = $"_{culture}_{fileName}"; + fileName = $"{culture}%{fileName}"; + fileTypeMarker = '%'; } var ret = new List (); var newParts = new List { String.Empty, // ABI placeholder - $"{fileName}.so", + $"{fileTypeMarker}{fileName}.so", }; foreach (string a in abis) { @@ -352,8 +368,9 @@ static bool ArchiveContains (List archiveContents, string entryPath, And return false; } - foreach (string existingEntry in archiveContents) { - foreach (string wantedEntry in potentialEntries) { + foreach (string wantedEntry in potentialEntries) { + Console.WriteLine ($"Wanted entry: {wantedEntry}"); + foreach (string existingEntry in archiveContents) { if (String.Compare (existingEntry, wantedEntry, StringComparison.Ordinal) == 0) { return true; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs index 002a84d9bf3..ad4a04cf95a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreAssemblyInfo.cs @@ -8,15 +8,16 @@ namespace Xamarin.Android.Tasks; class AssemblyStoreAssemblyInfo { - public AndroidTargetArch Arch { get; } - public string InArchivePath { get; } - public FileInfo SourceFile { get; } - public string AssemblyName { get; } - public byte[] AssemblyNameBytes { get; } - public FileInfo? SymbolsFile { get; set; } - public FileInfo? ConfigFile { get; set; } - - public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssemblyPath, ITaskItem assembly) + public AndroidTargetArch Arch { get; } + public FileInfo SourceFile { get; } + public string AssemblyName { get; } + public byte[] AssemblyNameBytes { get; } + public string AssemblyNameNoExt { get; } + public byte[] AssemblyNameNoExtBytes { get; } + public FileInfo? SymbolsFile { get; set; } + public FileInfo? ConfigFile { get; set; } + + public AssemblyStoreAssemblyInfo (string sourceFilePath, ITaskItem assembly) { Arch = MonoAndroidHelper.GetTargetArch (assembly); if (Arch == AndroidTargetArch.None) { @@ -24,7 +25,6 @@ public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssembl } SourceFile = new FileInfo (sourceFilePath); - InArchivePath = inArchiveAssemblyPath; string? name = Path.GetFileName (SourceFile.Name); if (name == null) { @@ -35,12 +35,19 @@ public AssemblyStoreAssemblyInfo (string sourceFilePath, string inArchiveAssembl name = Path.GetFileNameWithoutExtension (name); } + string nameNoExt = Path.GetFileNameWithoutExtension (name); string? culture = assembly.GetMetadata ("Culture"); if (!String.IsNullOrEmpty (culture)) { name = $"{culture}/{name}"; + nameNoExt = $"{culture}/{nameNoExt}"; } - AssemblyName = name; - AssemblyNameBytes = MonoAndroidHelper.Utf8StringToBytes (name); + (AssemblyName, AssemblyNameBytes) = SetName (name); + (AssemblyNameNoExt, AssemblyNameNoExtBytes) = SetName (nameNoExt); + + (string name, byte[] bytes) SetName (string assemblyName) + { + return (assemblyName, MonoAndroidHelper.Utf8StringToBytes (assemblyName)); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs index fe36b5ac775..010b607f0c7 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyStoreGenerator.cs @@ -128,11 +128,9 @@ string Generate (string baseOutputDirectory, AndroidTargetArch arch, List? path return String.Join ("/", parts); } + /// + /// Mangles APK/AAB entry name for assembly and their associated pdb and config entries in the + /// way expected by our native runtime. Must **NOT** be used to mangle names when assembly stores + /// are used. Must **NOT** be used for entries other than assemblies and their associated files. + /// + public static string MakeDiscreteAssembliesEntryName (string name, string? culture = null) + { + const string ext = ".so"; + + if (!String.IsNullOrEmpty (culture)) { + return $"%{culture}%{name}{ext}"; + } + + return $"#{name}{ext}"; + } + public static byte[] Utf8StringToBytes (string str) => Encoding.UTF8.GetBytes (str); public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (Utf8StringToBytes (str), is64Bit); diff --git a/src/monodroid/jni/xamarin-app.hh b/src/monodroid/jni/xamarin-app.hh index 39384cbeea0..ae4571d4437 100644 --- a/src/monodroid/jni/xamarin-app.hh +++ b/src/monodroid/jni/xamarin-app.hh @@ -121,7 +121,7 @@ struct CompressedAssemblies CompressedAssemblyDescriptor *descriptors; }; -struct XamarinAndroidBundledAssembly final +struct XamarinAndroidBundledAssembly { int32_t apk_fd; uint32_t data_offset; From 2577394939493db4d5701045a9bc81697c520394 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 12 Dec 2023 21:44:12 +0100 Subject: [PATCH 049/143] Make mangled assembly names work at run time All the string handling will go away once I implement hashing of the names instead. --- .../Tasks/GeneratePackageManagerJava.cs | 13 +++-- .../MarshalMethodsNativeAssemblyGenerator.cs | 25 +++++++--- src/monodroid/jni/embedded-assemblies-zip.cc | 50 +++++++++++-------- src/monodroid/jni/embedded-assemblies.hh | 3 +- src/monodroid/jni/shared-constants.hh | 4 +- 5 files changed, 62 insertions(+), 33 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 1ea58a98a9e..767a3f3c7db 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -244,9 +244,16 @@ void AddEnvironment () HashSet archAssemblyNames = null; HashSet uniqueAssemblyNames = new HashSet (StringComparer.OrdinalIgnoreCase); Action updateAssemblyCount = (ITaskItem assembly) => { - // We need to use the 'RelativePath' metadata, if found, because it will give us the correct path for satellite assemblies - with the culture in the path. - string? relativePath = assembly.GetMetadata ("RelativePath"); - string assemblyName = String.IsNullOrEmpty (relativePath) ? Path.GetFileName (assembly.ItemSpec) : relativePath; + string? culture = assembly.GetMetadata ("Culture"); + string fileName = Path.GetFileName (assembly.ItemSpec); + string assemblyName; + + if (String.IsNullOrEmpty (culture)) { + assemblyName = fileName; + } else { + assemblyName = $"{culture}/{fileName}"; + } + if (!uniqueAssemblyNames.Contains (assemblyName)) { uniqueAssemblyNames.Add (assemblyName); } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index 0a3dc3827f6..e9263f4bec6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -985,14 +985,24 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) foreach (string name in uniqueAssemblyNames) { // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here - string dirName = Path.GetDirectoryName (name) ?? String.Empty; - string clippedName = Path.Combine (dirName, Path.GetFileNameWithoutExtension (name)); - string soName = Path.Combine (dirName, $"{Path.GetFileName (name)}.so"); + string cultureName = Path.GetDirectoryName (name) ?? String.Empty; + string clippedName = Path.Combine (cultureName, Path.GetFileNameWithoutExtension (name)); + string inArchiveName; + + if (cultureName.Length == 0) { + // Regular assemblies get the '#' prefix + inArchiveName = $"#{name}.so"; + } else { + // Satellite assemblies get the '%{CULTURE}%' prefix + inArchiveName = $"%{cultureName}%{Path.GetFileName (name)}.so"; + } + ulong hashFull32 = MonoAndroidHelper.GetXxHash (name, is64Bit: false); - ulong hashSo32 = MonoAndroidHelper.GetXxHash (soName, is64Bit: false); + ulong hashInArchive32 = MonoAndroidHelper.GetXxHash (inArchiveName, is64Bit: false); ulong hashClipped32 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: false); + ulong hashFull64 = MonoAndroidHelper.GetXxHash (name, is64Bit: true); - ulong hashSo64 = MonoAndroidHelper.GetXxHash (soName, is64Bit: true); + ulong hashInArchive64 = MonoAndroidHelper.GetXxHash (inArchiveName, is64Bit: true); ulong hashClipped64 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: true); // @@ -1000,10 +1010,11 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) // `number_of_assembly_name_forms_in_image_cache` constant to the number of forms. // acs.Hashes32.Add ((uint)Convert.ChangeType (hashFull32, typeof(uint)), (name, index)); - acs.Hashes32.Add ((uint)Convert.ChangeType (hashSo32, typeof(uint)), (soName, index)); + acs.Hashes32.Add ((uint)Convert.ChangeType (hashInArchive32, typeof(uint)), (inArchiveName, index)); acs.Hashes32.Add ((uint)Convert.ChangeType (hashClipped32, typeof(uint)), (clippedName, index)); + acs.Hashes64.Add (hashFull64, (name, index)); - acs.Hashes64.Add (hashSo64, (soName, index)); + acs.Hashes64.Add (hashInArchive64, (inArchiveName, index)); acs.Hashes64.Add (hashClipped64, (clippedName, index)); index++; diff --git a/src/monodroid/jni/embedded-assemblies-zip.cc b/src/monodroid/jni/embedded-assemblies-zip.cc index fd9d73f68fe..4e29dbe1970 100644 --- a/src/monodroid/jni/embedded-assemblies-zip.cc +++ b/src/monodroid/jni/embedded-assemblies-zip.cc @@ -33,9 +33,7 @@ EmbeddedAssemblies::zip_load_entry_common (size_t entry_index, std::vector const& buf, uint32_t num_entries, [[maybe_unused]] monodroid_should_register should_register, ZipEntryLoadState &state) noexcept { + // TODO: do away with all the string manipulation here. Replace it with generating xxhash for the entry name dynamic_local_string entry_name; bool bundled_assemblies_slow_path = bundled_assembly_index >= application_config.number_of_assemblies_in_apk; uint32_t max_assembly_name_size = application_config.bundled_assembly_name_width - 1; + auto unmangle_name = [] (dynamic_local_string &name, size_t start_idx) { + size_t new_size = name.length () - 4; // Includes the first ("marker") character and the .so extension + memmove (name.get () + start_idx, name.get() + start_idx + 1, new_size); + name.set_length (new_size); + log_debug (LOG_ASSEMBLY, "Unmangled name to '%s'", name.get ()); + }; + // clang-tidy claims we have a leak in the loop: // // Potential leak of memory pointed to by 'assembly_name' @@ -103,6 +108,22 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c continue; } + if (entry_name[state.prefix_len] == '#') { + unmangle_name (entry_name, state.prefix_len); + } else if (entry_name[state.prefix_len] == '%') { + // Make sure assembly name is {CULTURE}/assembly.dll + unmangle_name (entry_name, state.prefix_len); + for (size_t idx = state.prefix_len; idx < entry_name.length (); idx++) { + if (entry_name[idx] == '%') { + entry_name[idx] = '/'; + break; + } + } + } else { + continue; // Can't be an assembly, the name's not mangled + } + log_debug (LOG_ASSEMBLY, " interesting entry. Name modified to '%s'", entry_name.get ()); + #if defined (DEBUG) const char *last_slash = utils.find_last (entry_name, '/'); bool entry_is_overridden = last_slash == nullptr ? false : !should_register (last_slash + 1); @@ -121,25 +142,14 @@ EmbeddedAssemblies::zip_load_individual_assembly_entries (std::vector c continue; } -#if !defined(NET) - if (utils.ends_with (entry_name, ".config")) { - char *assembly_name = strdup (basename (entry_name.get ())); - // Remove '.config' suffix - *strrchr (assembly_name, '.') = '\0'; - - md_mmap_info map_info = md_mmap_apk_file (state.apk_fd, state.data_offset, state.file_size, entry_name.get ()); - mono_register_config_for_assembly (assembly_name, (const char*)map_info.area); - + if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) { continue; } -#endif // ndef NET - - if (!utils.ends_with (entry_name, SharedConstants::DLL_EXTENSION)) - continue; #if defined (DEBUG) - if (entry_is_overridden) + if (entry_is_overridden) { continue; + } #endif if (XA_UNLIKELY (bundled_assembly_index >= application_config.number_of_assemblies_in_apk || bundled_assemblies_slow_path)) { diff --git a/src/monodroid/jni/embedded-assemblies.hh b/src/monodroid/jni/embedded-assemblies.hh index 77d7f403060..194172ff0d2 100644 --- a/src/monodroid/jni/embedded-assemblies.hh +++ b/src/monodroid/jni/embedded-assemblies.hh @@ -103,10 +103,11 @@ namespace xamarin::android::internal { static constexpr uint32_t number_of_assembly_store_files = 1; static constexpr char dso_suffix[] = ".so"; static constexpr char apk_lib_dir_name[] = "lib"; + static constexpr char regular_assembly_marker = '#'; + static constexpr char satellite_assembly_marker = '%'; static constexpr auto apk_lib_prefix = concat_const (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator); static constexpr auto assembly_store_file_name = concat_const (apk_lib_dir_name, zip_path_separator, SharedConstants::android_lib_abi, zip_path_separator, "assemblies.", SharedConstants::android_lib_abi, ".blob.so"); - #if defined (DEBUG) || !defined (ANDROID) static constexpr char override_typemap_entry_name[] = ".__override__"; #endif diff --git a/src/monodroid/jni/shared-constants.hh b/src/monodroid/jni/shared-constants.hh index 3ebef32a3b4..dea2d55c95c 100644 --- a/src/monodroid/jni/shared-constants.hh +++ b/src/monodroid/jni/shared-constants.hh @@ -29,8 +29,8 @@ namespace xamarin::android::internal static constexpr char ANDROID_ENVIRONMENT_CLASS_NAME[] = "AndroidEnvironment"; static constexpr char ANDROID_RUNTIME_INTERNAL_CLASS_NAME[] = "AndroidRuntimeInternal"; - static constexpr char DLL_EXTENSION[] = ".dll.so"; - static constexpr char PDB_EXTENSION[] = ".pdb.so"; + static constexpr char DLL_EXTENSION[] = ".dll"; + static constexpr char PDB_EXTENSION[] = ".pdb"; #if defined (NET) static constexpr char RUNTIME_CONFIG_BLOB_NAME[] = "arc.bin.so"; #endif // def NET From 5ceece9d66c478488f4d77601da99646c993245c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 9 Jan 2024 11:25:42 +0100 Subject: [PATCH 050/143] Fix the `BuildTest.CheckAssemblyCounts` test --- .../Xamarin.Android.Build.Tests/BuildTest.cs | 13 +++++++--- .../Utilities/ArchiveAssemblyHelper.cs | 25 +++++++------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index f0819191393..6b734ca1447 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -253,7 +253,7 @@ public void CheckAssemblyCounts (bool isRelease, bool aot) proj.PackageReferences.Add (KnownPackages.XamarinGoogleAndroidMaterial); var abis = new [] { "armeabi-v7a", "x86" }; - proj.SetAndroidSupportedAbis (abis); + proj.SetRuntimeIdentifiers (abis); proj.SetProperty (proj.ActiveConfigurationProperties, "AndroidUseAssemblyStore", "True"); using (var b = CreateApkBuilder ()) { @@ -273,7 +273,14 @@ public void CheckAssemblyCounts (bool isRelease, bool aot) string apk = Path.Combine (Root, b.ProjectDirectory, proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, useAssemblyStores: true); - Assert.IsTrue (app_config.number_of_assemblies_in_apk == (uint)helper.GetNumberOfAssemblies (), "Assembly count must be equal between ApplicationConfig and the archive contents"); + foreach (string abi in abis) { + AndroidTargetArch arch = MonoAndroidHelper.AbiToTargetArch (abi); + Assert.AreEqual ( + app_config.number_of_assemblies_in_apk, + helper.GetNumberOfAssemblies (arch: arch), + $"Assembly count must be equal between ApplicationConfig and the archive contents for architecture {arch} (ABI: {abi})" + ); + } } } @@ -800,7 +807,7 @@ public void IfAndroidJarDoesNotExistThrowXA5207 ([Values(true, false)] bool buil Assert.IsTrue (builder.LastBuildOutput.ContainsText ($"Could not find android.jar for API level {proj.TargetSdkVersion}"), "XA5207 should have had a good error message."); if (buildingInsideVisualStudio) Assert.IsTrue (builder.LastBuildOutput.ContainsText ($"Either install it in the Android SDK Manager"), "XA5207 should have an error message for Visual Studio."); - else + else Assert.IsTrue (builder.LastBuildOutput.ContainsText ($"You can install the missing API level by running"), "XA5207 should have an error message for the command line."); } Directory.Delete (AndroidSdkDirectory, recursive: true); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index 1c5a683cebd..a6c3a3b7c84 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -244,25 +244,18 @@ void SynthetizeAssemblies (AssemblyStoreExplorer? explorer) return null; } - public int GetNumberOfAssemblies (bool countAbiAssembliesOnce = true, bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) + public int GetNumberOfAssemblies (bool forceRefresh = false, AndroidTargetArch arch = AndroidTargetArch.None) { - List contents = ListArchiveContents (assembliesRootDir, forceRefresh); - var dlls = contents.Where (x => x.EndsWith (".dll.so", StringComparison.OrdinalIgnoreCase)); + List contents = ListArchiveContents (assembliesRootDir, forceRefresh, arch); - if (!countAbiAssembliesOnce) { - return dlls.Count (); - } - - var cache = new HashSet (StringComparer.OrdinalIgnoreCase); - return dlls.Where (x => { - string name = Path.GetFileName (x); - if (cache.Contains (name)) { - return false; - } + // We must count only .dll.so entries starting with the '#' character, as they are the actual managed assemblies. + // Other entries in `lib/{arch}` might be AOT shared libraries, which will also have the .dll.so extension. + var dlls = contents.Where (x => { + string fileName = Path.GetFileName (x); + return fileName[0] == '#' && fileName.EndsWith (".dll.so", StringComparison.OrdinalIgnoreCase); + }); - cache.Add (name); - return true; - }).Count (); + return dlls.Count (); } /// From 52bfb9628ea37e778285f85a90da872d5890b515 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 9 Jan 2024 11:51:20 +0100 Subject: [PATCH 051/143] =?UTF-8?q?Fix=20the=20`AotTests.BuildAotApplicati?= =?UTF-8?q?onWithNdkAndBundleAnd=C3=9Cml=C3=A4=C3=BCts`=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Tests/Xamarin.Android.Build.Tests/AotTests.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index 1dd5a0bd167..c79e06a4e9a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -181,7 +181,7 @@ public void BuildAotApplicationWithNdkAndBundleAndÜmläüts (string supportedAb }; proj.SetProperty ("AndroidNdkDirectory", AndroidNdkPath); - proj.SetAndroidSupportedAbis (supportedAbis); + proj.SetRuntimeIdentifiers (supportedAbis.Split (';')); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); bool checkMinLlvmPath = enableLLVM && (supportedAbis == "armeabi-v7a" || supportedAbis == "x86"); @@ -205,7 +205,7 @@ public void BuildAotApplicationWithNdkAndBundleAndÜmläüts (string supportedAb proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), @@ -236,7 +236,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL AotAssemblies = true, PackageName = "com.xamarin.buildaotappandbundlewithspecialchars", }; - proj.SetAndroidSupportedAbis (supportedAbis); + proj.SetRuntimeIdentifiers (supportedAbis.Split (';')); proj.SetProperty ("EnableLLVM", enableLLVM.ToString ()); proj.SetProperty ("AndroidUseAssemblyStore", usesAssemblyBlobs.ToString ()); using (var b = CreateApkBuilder (path)) { @@ -250,7 +250,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in the {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ("assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), From 2aab14b406edb9632cecd3af960980a7fd111755 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 9 Jan 2024 15:27:25 +0100 Subject: [PATCH 052/143] Fix name mangling of PDB files This fixes at least the `BuildHasNoWarnings` test --- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 17f5748b99d..2206abaaf13 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -494,7 +494,7 @@ void AddAssembliesFromCollection (ITaskItem[] assemblies) AddFileToArchiveIfNewer ( apk, symbolsPath, - MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyDirectory + Path.GetFileName (symbols)), + assemblyDirectory + MonoAndroidHelper.MakeDiscreteAssembliesEntryName (Path.GetFileName (symbols)), compressionMethod: UncompressedMethod ); } From 13dd0cc3a3e3dcc9aa0587e2acbaf7ab50dc8767 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Tue, 9 Jan 2024 16:49:47 +0100 Subject: [PATCH 053/143] Fix the `PackagingTest.CheckIncludedAssemblies` test... ...and possibly others that use the same helper method --- .../Utilities/ArchiveAssemblyHelper.cs | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs index a6c3a3b7c84..41d847da8b8 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/ArchiveAssemblyHelper.cs @@ -45,7 +45,7 @@ public ArchiveAssemblyHelper (string archivePath, bool useAssemblyStores = true, string extension = Path.GetExtension (archivePath) ?? String.Empty; if (String.Compare (".aab", extension, StringComparison.OrdinalIgnoreCase) == 0) { - assembliesRootDir = "base/lib/assemblies"; + assembliesRootDir = "base/lib/"; } else if (String.Compare (".apk", extension, StringComparison.OrdinalIgnoreCase) == 0) { assembliesRootDir = "lib/"; } else if (String.Compare (".zip", extension, StringComparison.OrdinalIgnoreCase) == 0) { @@ -456,7 +456,20 @@ void ArchiveContains (ICollection fileNames, out List existingFi IEnumerable GetMissingFilesForAbi (string abi) { (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (abi); - return fileNames.Where (x => !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixAssemblies, x)) && !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixLib, x))); + return fileNames.Where (x => { + string? culture = null; + string fileName = x; + int slashIndex = x.IndexOf ('/'); + if (slashIndex > 0) { + culture = x.Substring (0, slashIndex); + fileName = x.Substring (slashIndex + 1); + } + + return !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixAssemblies, x)) && + !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixLib, x)) && + !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixAssemblies, MonoAndroidHelper.MakeDiscreteAssembliesEntryName (fileName, culture))) && + !zip.ContainsEntry (MonoAndroidHelper.MakeZipArchivePath (prefixLib, MonoAndroidHelper.MakeDiscreteAssembliesEntryName (fileName, culture))); + }); } IEnumerable GetAdditionalFilesForAbi (string abi, List existingFiles) @@ -464,6 +477,23 @@ IEnumerable GetAdditionalFilesForAbi (string abi, List existingF (string prefixAssemblies, string prefixLib) = GetArchivePrefixes (abi); return existingFiles.Where (x => !fileNames.Contains (x.Replace (prefixAssemblies, string.Empty)) && !fileNames.Contains (x.Replace (prefixLib, String.Empty))); } + + string GetUnmangledFileName (string fullName) + { + string fileName = Path.GetFileName (fullName); + switch(fileName[0]) { + case '#': + // Drop the '#' prefix and the .so extension + return Path.GetFileNameWithoutExtension (fileName.Substring (1)); + + case '%': + // Drop the '%' prefix, replace the following '%s' with `/` to get a culture/Assembly file name and drop the .so extension + return Path.GetFileNameWithoutExtension (fileName.Substring (1).Replace ('%', '/')); + + default: + return fileName; + } + } } void StoreContains (ICollection fileNames, out List existingFiles, out List missingFiles, out List additionalFiles, IEnumerable? targetArches = null) From d2ca304b54f9b527ffc14e6e0ae2715f87d4e3de Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 10 Jan 2024 11:14:57 +0100 Subject: [PATCH 054/143] =?UTF-8?q?Fix=20the=20`AotTests.BuildAotApplicati?= =?UTF-8?q?onAnd=C3=9Cml=C3=A4=C3=BCts`=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Tests/Xamarin.Android.Build.Tests/AotTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index c79e06a4e9a..20e569c618c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -250,7 +250,7 @@ public void BuildAotApplicationAndÜmläüts (string supportedAbis, bool enableL proj.OutputPath, $"{proj.PackageName}-Signed.apk"); var helper = new ArchiveAssemblyHelper (apk, usesAssemblyBlobs); - Assert.IsTrue (helper.Exists ("assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in {proj.PackageName}-Signed.apk"); + Assert.IsTrue (helper.Exists ($"assemblies/{abi}/UnnamedProject.dll"), $"{abi}/UnnamedProject.dll should be in {proj.PackageName}-Signed.apk"); using (var zipFile = ZipHelper.OpenZip (apk)) { Assert.IsNotNull (ZipHelper.ReadFileFromZip (zipFile, string.Format ("lib/{0}/libaot-UnnamedProject.dll.so", abi)), From 9b8181deaf4642b7d8e3b47be9e16150ec2d84f3 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 10 Jan 2024 11:18:39 +0100 Subject: [PATCH 055/143] Update apkdesc files --- .../BuildReleaseArm64SimpleDotNet.apkdesc | 58 ++-- .../BuildReleaseArm64XFormsDotNet.apkdesc | 284 +++++++++--------- 2 files changed, 171 insertions(+), 171 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index 3e1ed09b09d..f5aa405c5b7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -4,47 +4,47 @@ "AndroidManifest.xml": { "Size": 3036 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 1028 + "classes.dex": { + "Size": 377856 }, - "assemblies/Java.Interop.dll": { - "Size": 61588 + "lib/arm64-v8a/#_Microsoft.Android.Resource.Designer.dll.so": { + "Size": 1027 }, - "assemblies/Mono.Android.dll": { - "Size": 90724 + "lib/arm64-v8a/#Java.Interop.dll.so": { + "Size": 61443 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5147 + "lib/arm64-v8a/#Mono.Android.dll.so": { + "Size": 90523 }, - "assemblies/rc.bin": { - "Size": 1512 + "lib/arm64-v8a/#Mono.Android.Runtime.dll.so": { + "Size": 5513 }, - "assemblies/System.Console.dll": { - "Size": 6547 + "lib/arm64-v8a/#System.Console.dll.so": { + "Size": 6543 }, - "assemblies/System.Linq.dll": { - "Size": 7163 + "lib/arm64-v8a/#System.Linq.dll.so": { + "Size": 7158 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 554659 + "lib/arm64-v8a/#System.Private.CoreLib.dll.so": { + "Size": 553334 }, - "assemblies/System.Runtime.dll": { - "Size": 2551 + "lib/arm64-v8a/#System.Runtime.dll.so": { + "Size": 2623 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 4028 + "lib/arm64-v8a/#System.Runtime.InteropServices.dll.so": { + "Size": 4022 }, - "assemblies/UnnamedProject.dll": { - "Size": 2937 + "lib/arm64-v8a/#UnnamedProject.dll.so": { + "Size": 2932 }, - "classes.dex": { - "Size": 377856 + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 339864 + "Size": 340776 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3184512 @@ -59,16 +59,16 @@ "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 11648 + "Size": 12016 }, "META-INF/BNDLTOOL.RSA": { "Size": 1223 }, "META-INF/BNDLTOOL.SF": { - "Size": 3037 + "Size": 3114 }, "META-INF/MANIFEST.MF": { - "Size": 2910 + "Size": 2987 }, "res/drawable-hdpi-v4/icon.png": { "Size": 2178 @@ -95,5 +95,5 @@ "Size": 1904 } }, - "PackageSize": 2783562 + "PackageSize": 2660759 } \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 702febdc65e..2a600ef7470 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -4,215 +4,215 @@ "AndroidManifest.xml": { "Size": 3572 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 2104 + "classes.dex": { + "Size": 3502580 }, - "assemblies/FormsViewGroup.dll": { - "Size": 7113 + "lib/arm64-v8a/#_Microsoft.Android.Resource.Designer.dll.so": { + "Size": 2104 }, - "assemblies/Java.Interop.dll": { - "Size": 69705 + "lib/arm64-v8a/#FormsViewGroup.dll.so": { + "Size": 7106 }, - "assemblies/Mono.Android.dll": { - "Size": 447230 + "lib/arm64-v8a/#Java.Interop.dll.so": { + "Size": 69523 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5147 + "lib/arm64-v8a/#Mono.Android.dll.so": { + "Size": 445419 }, - "assemblies/mscorlib.dll": { - "Size": 3862 + "lib/arm64-v8a/#Mono.Android.Runtime.dll.so": { + "Size": 5513 }, - "assemblies/netstandard.dll": { - "Size": 5578 + "lib/arm64-v8a/#mscorlib.dll.so": { + "Size": 3855 }, - "assemblies/rc.bin": { - "Size": 1512 + "lib/arm64-v8a/#netstandard.dll.so": { + "Size": 5566 }, - "assemblies/System.Collections.Concurrent.dll": { - "Size": 11514 + "lib/arm64-v8a/#System.Collections.Concurrent.dll.so": { + "Size": 11499 }, - "assemblies/System.Collections.dll": { - "Size": 15408 + "lib/arm64-v8a/#System.Collections.dll.so": { + "Size": 15388 }, - "assemblies/System.Collections.NonGeneric.dll": { - "Size": 7452 + "lib/arm64-v8a/#System.Collections.NonGeneric.dll.so": { + "Size": 7443 }, - "assemblies/System.ComponentModel.dll": { - "Size": 1939 + "lib/arm64-v8a/#System.ComponentModel.dll.so": { + "Size": 1935 }, - "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2549 + "lib/arm64-v8a/#System.ComponentModel.Primitives.dll.so": { + "Size": 2547 }, - "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6033 + "lib/arm64-v8a/#System.ComponentModel.TypeConverter.dll.so": { + "Size": 6028 }, - "assemblies/System.Console.dll": { - "Size": 6578 + "lib/arm64-v8a/#System.Console.dll.so": { + "Size": 6575 }, - "assemblies/System.Core.dll": { - "Size": 1991 + "lib/arm64-v8a/#System.Core.dll.so": { + "Size": 1987 }, - "assemblies/System.Diagnostics.DiagnosticSource.dll": { - "Size": 9068 + "lib/arm64-v8a/#System.Diagnostics.DiagnosticSource.dll.so": { + "Size": 9060 }, - "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6552 + "lib/arm64-v8a/#System.Diagnostics.TraceSource.dll.so": { + "Size": 6545 }, - "assemblies/System.dll": { - "Size": 2345 + "lib/arm64-v8a/#System.dll.so": { + "Size": 2341 }, - "assemblies/System.Drawing.dll": { - "Size": 1939 + "lib/arm64-v8a/#System.Drawing.dll.so": { + "Size": 1934 }, - "assemblies/System.Drawing.Primitives.dll": { - "Size": 11975 + "lib/arm64-v8a/#System.Drawing.Primitives.dll.so": { + "Size": 11961 }, - "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11196 + "lib/arm64-v8a/#System.IO.Compression.Brotli.dll.so": { + "Size": 11185 }, - "assemblies/System.IO.Compression.dll": { - "Size": 15880 + "lib/arm64-v8a/#System.IO.Compression.dll.so": { + "Size": 15862 }, - "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9875 + "lib/arm64-v8a/#System.IO.IsolatedStorage.dll.so": { + "Size": 9866 }, - "assemblies/System.Linq.dll": { - "Size": 19599 + "lib/arm64-v8a/#System.Linq.dll.so": { + "Size": 19573 }, - "assemblies/System.Linq.Expressions.dll": { - "Size": 165117 + "lib/arm64-v8a/#System.Linq.Expressions.dll.so": { + "Size": 164633 }, - "assemblies/System.Net.Http.dll": { - "Size": 67631 + "lib/arm64-v8a/#System.Net.Http.dll.so": { + "Size": 67512 }, - "assemblies/System.Net.Primitives.dll": { - "Size": 22432 + "lib/arm64-v8a/#System.Net.Primitives.dll.so": { + "Size": 22414 }, - "assemblies/System.Net.Requests.dll": { - "Size": 3602 + "lib/arm64-v8a/#System.Net.Requests.dll.so": { + "Size": 3591 }, - "assemblies/System.ObjectModel.dll": { - "Size": 8116 + "lib/arm64-v8a/#System.ObjectModel.dll.so": { + "Size": 8108 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 849646 + "lib/arm64-v8a/#System.Private.CoreLib.dll.so": { + "Size": 847197 }, - "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 193991 + "lib/arm64-v8a/#System.Private.DataContractSerialization.dll.so": { + "Size": 193441 }, - "assemblies/System.Private.Uri.dll": { - "Size": 42860 + "lib/arm64-v8a/#System.Private.Uri.dll.so": { + "Size": 42797 }, - "assemblies/System.Private.Xml.dll": { - "Size": 216226 + "lib/arm64-v8a/#System.Private.Xml.dll.so": { + "Size": 215575 }, - "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16639 + "lib/arm64-v8a/#System.Private.Xml.Linq.dll.so": { + "Size": 16618 }, - "assemblies/System.Runtime.dll": { - "Size": 2708 + "lib/arm64-v8a/#System.Runtime.dll.so": { + "Size": 2750 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 4028 + "lib/arm64-v8a/#System.Runtime.InteropServices.dll.so": { + "Size": 4022 }, - "assemblies/System.Runtime.Serialization.dll": { - "Size": 1865 + "lib/arm64-v8a/#System.Runtime.Serialization.dll.so": { + "Size": 1860 }, - "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2484 + "lib/arm64-v8a/#System.Runtime.Serialization.Formatters.dll.so": { + "Size": 2482 }, - "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3758 + "lib/arm64-v8a/#System.Runtime.Serialization.Primitives.dll.so": { + "Size": 3750 }, - "assemblies/System.Security.Cryptography.dll": { - "Size": 8111 + "lib/arm64-v8a/#System.Security.Cryptography.dll.so": { + "Size": 8102 }, - "assemblies/System.Text.RegularExpressions.dll": { - "Size": 159112 + "lib/arm64-v8a/#System.Text.RegularExpressions.dll.so": { + "Size": 158797 }, - "assemblies/System.Xml.dll": { - "Size": 1758 + "lib/arm64-v8a/#System.Xml.dll.so": { + "Size": 1755 }, - "assemblies/System.Xml.Linq.dll": { - "Size": 1774 + "lib/arm64-v8a/#System.Xml.Linq.dll.so": { + "Size": 1770 }, - "assemblies/UnnamedProject.dll": { - "Size": 4990 + "lib/arm64-v8a/#UnnamedProject.dll.so": { + "Size": 4981 }, - "assemblies/Xamarin.AndroidX.Activity.dll": { - "Size": 5944 + "lib/arm64-v8a/#Xamarin.AndroidX.Activity.dll.so": { + "Size": 5936 }, - "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6036 + "lib/arm64-v8a/#Xamarin.AndroidX.AppCompat.AppCompatResources.dll.so": { + "Size": 6026 }, - "assemblies/Xamarin.AndroidX.AppCompat.dll": { - "Size": 119847 + "lib/arm64-v8a/#Xamarin.AndroidX.AppCompat.dll.so": { + "Size": 119193 }, - "assemblies/Xamarin.AndroidX.CardView.dll": { - "Size": 6802 + "lib/arm64-v8a/#Xamarin.AndroidX.CardView.dll.so": { + "Size": 6785 }, - "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { - "Size": 17260 + "lib/arm64-v8a/#Xamarin.AndroidX.CoordinatorLayout.dll.so": { + "Size": 17222 }, - "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 100665 + "lib/arm64-v8a/#Xamarin.AndroidX.Core.dll.so": { + "Size": 100216 }, - "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 14631 + "lib/arm64-v8a/#Xamarin.AndroidX.DrawerLayout.dll.so": { + "Size": 14602 }, - "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 41736 + "lib/arm64-v8a/#Xamarin.AndroidX.Fragment.dll.so": { + "Size": 41548 }, - "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { - "Size": 6083 + "lib/arm64-v8a/#Xamarin.AndroidX.Legacy.Support.Core.UI.dll.so": { + "Size": 6075 }, - "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { - "Size": 6471 + "lib/arm64-v8a/#Xamarin.AndroidX.Lifecycle.Common.dll.so": { + "Size": 6464 }, - "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { - "Size": 6618 + "lib/arm64-v8a/#Xamarin.AndroidX.Lifecycle.LiveData.Core.dll.so": { + "Size": 6607 }, - "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { - "Size": 3071 + "lib/arm64-v8a/#Xamarin.AndroidX.Lifecycle.ViewModel.dll.so": { + "Size": 3065 }, - "assemblies/Xamarin.AndroidX.Loader.dll": { - "Size": 12926 + "lib/arm64-v8a/#Xamarin.AndroidX.Loader.dll.so": { + "Size": 12907 }, - "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 89997 + "lib/arm64-v8a/#Xamarin.AndroidX.RecyclerView.dll.so": { + "Size": 89550 }, - "assemblies/Xamarin.AndroidX.SavedState.dll": { - "Size": 4909 + "lib/arm64-v8a/#Xamarin.AndroidX.SavedState.dll.so": { + "Size": 4905 }, - "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 10575 + "lib/arm64-v8a/#Xamarin.AndroidX.SwipeRefreshLayout.dll.so": { + "Size": 10558 }, - "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 18594 + "lib/arm64-v8a/#Xamarin.AndroidX.ViewPager.dll.so": { + "Size": 18553 }, - "assemblies/Xamarin.Forms.Core.dll": { - "Size": 528450 + "lib/arm64-v8a/#Xamarin.Forms.Core.dll.so": { + "Size": 526687 }, - "assemblies/Xamarin.Forms.Platform.Android.dll": { - "Size": 337925 + "lib/arm64-v8a/#Xamarin.Forms.Platform.Android.dll.so": { + "Size": 336895 }, - "assemblies/Xamarin.Forms.Platform.dll": { - "Size": 11083 + "lib/arm64-v8a/#Xamarin.Forms.Platform.dll.so": { + "Size": 11076 }, - "assemblies/Xamarin.Forms.Xaml.dll": { - "Size": 60774 + "lib/arm64-v8a/#Xamarin.Forms.Xaml.dll.so": { + "Size": 60656 }, - "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 42285 + "lib/arm64-v8a/#Xamarin.Google.Android.Material.dll.so": { + "Size": 42137 }, - "classes.dex": { - "Size": 3502580 + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 339864 + "Size": 340776 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3184512 @@ -227,7 +227,7 @@ "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 102888 + "Size": 103912 }, "META-INF/android.support.design_material.version": { "Size": 12 @@ -338,16 +338,16 @@ "Size": 6 }, "META-INF/BNDLTOOL.RSA": { - "Size": 1221 + "Size": 1223 }, "META-INF/BNDLTOOL.SF": { - "Size": 77024 + "Size": 77496 }, "META-INF/com.google.android.material_material.version": { "Size": 10 }, "META-INF/MANIFEST.MF": { - "Size": 76897 + "Size": 77369 }, "META-INF/proguard/androidx-annotations.pro": { "Size": 339 @@ -1916,5 +1916,5 @@ "Size": 325240 } }, - "PackageSize": 7941134 + "PackageSize": 7265763 } \ No newline at end of file From 82976133eb04b03c4900e46fb5988f13bfc2f697 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 10 Jan 2024 11:52:27 +0100 Subject: [PATCH 056/143] Fix the `BuildTest.MicrosoftExtensionsHttp` test --- .../Tasks/GenerateJavaStubs.cs | 13 +++++++++++-- .../Utilities/MonoAndroidHelper.cs | 5 ++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 0527fc2e6f7..3dfe0092d60 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -185,7 +185,7 @@ void Run (bool useMarshalMethods) foreach (var kvp in allAssembliesPerArch) { AndroidTargetArch arch = kvp.Key; Dictionary archAssemblies = kvp.Value; - (bool success, NativeCodeGenState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, userAssembliesPerArch[arch], useMarshalMethods, generateJavaCode); + (bool success, NativeCodeGenState? state) = GenerateJavaSourcesAndMaybeClassifyMarshalMethods (arch, archAssemblies, MaybeGetArchAssemblies (userAssembliesPerArch, arch), useMarshalMethods, generateJavaCode); if (!success) { return; @@ -249,8 +249,17 @@ void Run (bool useMarshalMethods) return; } - IList additionalProviders = MergeManifest (templateCodeGenState, userAssembliesPerArch[templateCodeGenState.TargetArch]); + IList additionalProviders = MergeManifest (templateCodeGenState, MaybeGetArchAssemblies (userAssembliesPerArch, templateCodeGenState.TargetArch)); GenerateAdditionalProviderSources (templateCodeGenState, additionalProviders); + + Dictionary MaybeGetArchAssemblies (Dictionary> dict, AndroidTargetArch arch) + { + if (!dict.TryGetValue (arch, out Dictionary archDict)) { + return new Dictionary (StringComparer.OrdinalIgnoreCase); + } + + return archDict; + } } void GenerateAdditionalProviderSources (NativeCodeGenState codeGenState, IList additionalProviders) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index d177688555d..b84391c9a2c 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -603,7 +603,10 @@ public static Dictionary> GetPe assemblies.Add (name, assembly); } - if (!validate) { + // It's possible some assembly collections will be empty (e.g. `ResolvedUserAssemblies` as passed to the `GenerateJavaStubs` task), which + // isn't a problem and such empty collections should not be validated, as it will end in the "should never happen" exception below being + // thrown as a false negative. + if (assembliesPerArch.Count == 0 || !validate) { return assembliesPerArch; } From 0afc041d7991c2c1243bc20b4f83426212a90f9d Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 10 Jan 2024 15:51:33 +0100 Subject: [PATCH 057/143] Fix the `BuildAMassiveApp` test and hopefully don't break others... --- .../Tasks/BuildApk.cs | 25 +++++++-- ...teCompressedAssembliesNativeSourceFiles.cs | 55 +++++++++++-------- .../Tasks/GenerateJavaStubs.cs | 10 +++- .../Utilities/MonoAndroidHelper.cs | 34 +++++++++++- 4 files changed, 93 insertions(+), 31 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 2206abaaf13..65b9441857f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -448,12 +448,29 @@ void AddAssemblies (ZipArchiveEx apk, bool debug, bool compress, IDictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies ( + assemblies, + SupportedAbis, + validate: true, + shouldSkip: (ITaskItem asm) => { + if (bool.TryParse (asm.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { + Log.LogDebugMessage ($"Skipping {asm.ItemSpec} due to 'AndroidSkipAddToPackage' == 'true' "); + return true; + } + + return false; } + ); + foreach (var kvp in perArchAssemblies) { + Log.LogDebugMessage ($"Adding assemblies for architecture '{kvp.Key}'"); + DoAddAssembliesFromArchCollection (kvp.Value); + } + } + + void DoAddAssembliesFromArchCollection (Dictionary assemblies) + { + foreach (ITaskItem assembly in assemblies.Values) { if (MonoAndroidHelper.IsReferenceAssembly (assembly.ItemSpec)) { Log.LogCodedWarning ("XA0107", assembly.ItemSpec, 0, Properties.Resources.XA0107, assembly.ItemSpec); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs index 014cde1697b..70f766de77a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateCompressedAssembliesNativeSourceFiles.cs @@ -43,37 +43,46 @@ void GenerateCompressedAssemblySources () return; } + Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies ( + ResolvedAssemblies, + SupportedAbis, + validate: true, + shouldSkip: (ITaskItem asm) => bool.TryParse (asm.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value + ); var archAssemblies = new Dictionary> (); var counters = new Dictionary (); - foreach (ITaskItem assembly in ResolvedAssemblies) { - if (bool.TryParse (assembly.GetMetadata ("AndroidSkipAddToPackage"), out bool value) && value) { - continue; - } - AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); - if (!archAssemblies.TryGetValue (arch, out Dictionary assemblies)) { - assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); - archAssemblies.Add (arch, assemblies); - } + foreach (var kvpPerArch in perArchAssemblies) { + AndroidTargetArch arch = kvpPerArch.Key; + Dictionary resolvedArchAssemblies = kvpPerArch.Value; - var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); - if (assemblies.ContainsKey (assemblyKey)) { - Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetAssemblyAbi(assembly)})"); - continue; - } + foreach (var kvp in resolvedArchAssemblies) { + ITaskItem assembly = kvp.Value; - var fi = new FileInfo (assembly.ItemSpec); - if (!fi.Exists) { - Log.LogError ($"Assembly {assembly.ItemSpec} does not exist"); - continue; - } + if (!archAssemblies.TryGetValue (arch, out Dictionary assemblies)) { + assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); + archAssemblies.Add (arch, assemblies); + } + var assemblyKey = CompressedAssemblyInfo.GetDictionaryKey (assembly); + if (assemblies.ContainsKey (assemblyKey)) { + Log.LogDebugMessage ($"Skipping duplicate assembly: {assembly.ItemSpec} (arch {MonoAndroidHelper.GetAssemblyAbi(assembly)})"); + continue; + } + + var fi = new FileInfo (assembly.ItemSpec); + if (!fi.Exists) { + Log.LogError ($"Assembly {assembly.ItemSpec} does not exist"); + continue; + } - if (!counters.TryGetValue (arch, out uint counter)) { - counter = 0; + + if (!counters.TryGetValue (arch, out uint counter)) { + counter = 0; + } + assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length), counter++, arch, Path.GetFileNameWithoutExtension (assembly.ItemSpec))); + counters[arch] = counter; } - assemblies.Add (assemblyKey, new CompressedAssemblyInfo (checked((uint)fi.Length), counter++, arch, Path.GetFileNameWithoutExtension (assembly.ItemSpec))); - counters[arch] = counter; } string key = CompressedAssemblyInfo.GetKey (ProjectFullPath); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 3dfe0092d60..cce58c94a64 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -148,11 +148,15 @@ void Run (bool useMarshalMethods) // We will process each architecture completely separately as both type maps and marshal methods are strictly per-architecture and // the assemblies should be processed strictly per architecture. Generation of JCWs, and the manifest are ABI-agnostic. // We will generate them only for the first architecture, whichever it is. - Dictionary> allAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, validate: true); + Dictionary> allAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, SupportedAbis, validate: true); // Should "never" happen... if (allAssembliesPerArch.Count != SupportedAbis.Length) { - throw new InvalidOperationException ($"Internal error: number of architectures ({allAssembliesPerArch.Count}) must equal the number of target ABIs ({SupportedAbis.Length})"); + // ...but it happens at least in our `BuildAMassiveApp` test, where `SupportedAbis` mentions only the `x86` and `armeabi-v7a` ABIs, but `ResolvedAssemblies` contains + // entries for all the ABIs we support, so let's be flexible and ignore the extra architectures but still error out if there are less architectures than supported ABIs. + if (allAssembliesPerArch.Count < SupportedAbis.Length) { + throw new InvalidOperationException ($"Internal error: number of architectures ({allAssembliesPerArch.Count}) must equal the number of target ABIs ({SupportedAbis.Length})"); + } } // ...or this... @@ -164,7 +168,7 @@ void Run (bool useMarshalMethods) } // ...as well as this - Dictionary> userAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedUserAssemblies, validate: true); + Dictionary> userAssembliesPerArch = MonoAndroidHelper.GetPerArchAssemblies (ResolvedUserAssemblies, SupportedAbis, validate: true); foreach (var kvp in userAssembliesPerArch) { if (!allAssembliesPerArch.TryGetValue (kvp.Key, out Dictionary allAssemblies)) { throw new InvalidOperationException ($"Internal error: found user assemblies for architecture '{kvp.Key}' which isn't found in ResolvedAssemblies"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index b84391c9a2c..b84204ec8c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -585,11 +585,43 @@ public static string GetNativeLibsRootDirectoryPath (string androidBinUtilsDirec return Path.GetFullPath (Path.Combine (androidBinUtilsDirectory, relPath, "lib")); } - public static Dictionary> GetPerArchAssemblies (IEnumerable input, bool validate) + /// + /// Process a collection of assembly `ITaskItem` objects, splitting it on the assembly architecture () while, at the same time, ignoring + /// all assemblies which are **not** in the collection. If necessary, the selection can be further controlled by passing a qualifier + /// function in which returns `true` if the assembly passed to it should be **skipped**. + /// + /// This method is necessary because sometimes our tasks will be given assemblies for more architectures than indicated as supported in their `SupportedAbis` properties. + /// One such example is the `AotTests.BuildAMassiveApp` test, which passes around a set of assemblies for all the supported architectures, but it supports only two ABIs + /// via the `SupportedAbis` property. + /// + public static Dictionary> GetPerArchAssemblies (IEnumerable input, ICollection supportedAbis, bool validate, Func? shouldSkip = null) + { + var supportedTargetArches = new HashSet (); + foreach (string abi in supportedAbis) { + supportedTargetArches.Add (AbiToTargetArch (abi)); + } + + return GetPerArchAssemblies ( + input, + supportedTargetArches, + validate, + shouldSkip + ); + } + + static Dictionary> GetPerArchAssemblies (IEnumerable input, HashSet supportedTargetArches, bool validate, Func? shouldSkip = null) { var assembliesPerArch = new Dictionary> (); foreach (ITaskItem assembly in input) { + if (shouldSkip != null && shouldSkip (assembly)) { + continue; + } + AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); + if (!supportedTargetArches.Contains (arch)) { + continue; + } + if (!assembliesPerArch.TryGetValue (arch, out Dictionary assemblies)) { assemblies = new Dictionary (StringComparer.OrdinalIgnoreCase); assembliesPerArch.Add (arch, assemblies); From be460e0dd9fb766df6d29468d73b04249c1923e0 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 10 Jan 2024 16:05:29 +0100 Subject: [PATCH 058/143] Fix the `TransitiveDependencyProduceReferenceAssembly` test --- .../Xamarin.Android.Build.Tests/IncrementalBuildTest.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index 05ce2386d18..051d154d339 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -646,8 +646,11 @@ public void TransitiveDependencyProduceReferenceAssembly () Assert.IsTrue (appBuilder.Build (app, doNotCleanupOnUpdate: true, saveProject: false), "app SignAndroidPackage build should have succeeded."); var lib2Output = Path.Combine (path, lib2.ProjectName, "bin", "Debug", "netstandard2.0", $"{lib2.ProjectName}.dll"); - var lib2InAppOutput = Path.Combine (path, app.ProjectName, app.IntermediateOutputPath, "android", "assets", $"{lib2.ProjectName}.dll"); - FileAssert.AreEqual (lib2Output, lib2InAppOutput, "new Library2 should have been copied to app output directory"); + + foreach (string abi in appBuilder.GetBuildAbis ()) { + var lib2InAppOutput = Path.Combine (path, app.ProjectName, app.IntermediateOutputPath, "android", "assets", abi, $"{lib2.ProjectName}.dll"); + FileAssert.AreEqual (lib2Output, lib2InAppOutput, $"new Library2 should have been copied to app output directory for abi '{abi}'"); + } } } From 6e1dd73935ce499ae31f01ec63dfe39ba34098ef Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 10 Jan 2024 16:26:23 +0100 Subject: [PATCH 059/143] Partial fix for the `AndroidAddKeepAlives` test --- src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs index 8a6c162fe8e..397e3d3bca7 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs @@ -106,6 +106,7 @@ public override bool RunTask () save |= addKeepAliveStep.AddKeepAlives (assemblyDefinition); if (save) { Log.LogDebugMessage ($"Saving modified assembly: {destination.ItemSpec}"); + Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec)); writerParameters.WriteSymbols = assemblyDefinition.MainModule.HasSymbols; assemblyDefinition.Write (destination.ItemSpec, writerParameters); continue; From c0f5b3b65e95c2607de27051fafc03ee8042a276 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 11 Jan 2024 17:46:47 +0100 Subject: [PATCH 060/143] Fix the `AndroidAddKeepAlives` test --- .../Tasks/LinkAssembliesNoShrink.cs | 146 ++++++++++++------ .../Tasks/LinkerTests.cs | 20 ++- .../Utilities/MonoAndroidHelper.cs | 3 +- 3 files changed, 110 insertions(+), 59 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs b/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs index 397e3d3bca7..9f86c110c35 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/LinkAssembliesNoShrink.cs @@ -1,4 +1,6 @@ #nullable enable +using System.Collections.Generic; + using Java.Interop.Tools.Cecil; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -6,6 +8,7 @@ using System; using System.IO; using Microsoft.Android.Build.Tasks; +using Xamarin.Android.Tools; namespace Xamarin.Android.Tasks { @@ -14,6 +17,15 @@ namespace Xamarin.Android.Tasks /// public class LinkAssembliesNoShrink : AndroidTask { + sealed class RunState + { + public DirectoryAssemblyResolver? resolver = null; + public TypeDefinitionCache? cache = null; + public FixAbstractMethodsStep? fixAbstractMethodsStep = null; + public AddKeepAlivesStep? addKeepAliveStep = null; + public FixLegacyResourceDesignerStep? fixLegacyResourceDesignerStep = null; + } + public override string TaskPrefix => "LNS"; /// @@ -59,65 +71,99 @@ public override bool RunTask () DeterministicMvid = Deterministic, }; - using (var resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: ReadSymbols, loadReaderParameters: readerParameters)) { - // Add SearchDirectories with ResolvedAssemblies - foreach (var assembly in ResolvedAssemblies) { - var path = Path.GetFullPath (Path.GetDirectoryName (assembly.ItemSpec)); - if (!resolver.SearchDirectories.Contains (path)) - resolver.SearchDirectories.Add (path); - } + Dictionary> perArchAssemblies = MonoAndroidHelper.GetPerArchAssemblies (ResolvedAssemblies, Array.Empty (), validate: false); + var runState = new RunState (); + AndroidTargetArch currentArch = AndroidTargetArch.None; - // Set up the FixAbstractMethodsStep and AddKeepAlivesStep - var cache = new TypeDefinitionCache (); - var fixAbstractMethodsStep = new FixAbstractMethodsStep (resolver, cache, Log); - var addKeepAliveStep = new AddKeepAlivesStep (resolver, cache, Log, UsingAndroidNETSdk); - var fixLegacyResourceDesignerStep = new FixLegacyResourceDesignerStep (resolver, Log); - for (int i = 0; i < SourceFiles.Length; i++) { - var source = SourceFiles [i]; - var destination = DestinationFiles [i]; - var assemblyName = Path.GetFileNameWithoutExtension (source.ItemSpec); - - // In .NET 6+, we can skip the main assembly - if (UsingAndroidNETSdk && !AddKeepAlives && assemblyName == TargetName) { - CopyIfChanged (source, destination); - continue; - } - if (fixAbstractMethodsStep.IsProductOrSdkAssembly (assemblyName)) { - CopyIfChanged (source, destination); - continue; - } + for (int i = 0; i < SourceFiles.Length; i++) { + ITaskItem source = SourceFiles [i]; + AndroidTargetArch sourceArch = GetValidArchitecture (source); + ITaskItem destination = DestinationFiles [i]; + AndroidTargetArch destinationArch = GetValidArchitecture (destination); - // Check AppDomain usage on any non-Product or Sdk assembly - AssemblyDefinition? assemblyDefinition = null; - if (!UsingAndroidNETSdk) { - assemblyDefinition = resolver.GetAssembly (source.ItemSpec); - fixAbstractMethodsStep.CheckAppDomainUsage (assemblyDefinition, (string msg) => Log.LogCodedWarning ("XA2000", msg)); - } + if (sourceArch != destinationArch) { + throw new InvalidOperationException ($"Internal error: assembly '{sourceArch}' targets architecture '{sourceArch}', while destination assembly '{destination}' targets '{destinationArch}' instead"); + } - // Only run the step on "MonoAndroid" assemblies - if (MonoAndroidHelper.IsMonoAndroidAssembly (source) && !MonoAndroidHelper.IsSharedRuntimeAssembly (source.ItemSpec)) { - if (assemblyDefinition == null) - assemblyDefinition = resolver.GetAssembly (source.ItemSpec); - - bool save = fixAbstractMethodsStep.FixAbstractMethods (assemblyDefinition); - if (UseDesignerAssembly) - save |= fixLegacyResourceDesignerStep.ProcessAssemblyDesigner (assemblyDefinition); - if (AddKeepAlives) - save |= addKeepAliveStep.AddKeepAlives (assemblyDefinition); - if (save) { - Log.LogDebugMessage ($"Saving modified assembly: {destination.ItemSpec}"); - Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec)); - writerParameters.WriteSymbols = assemblyDefinition.MainModule.HasSymbols; - assemblyDefinition.Write (destination.ItemSpec, writerParameters); - continue; + // Each architecture must have a different set of context classes, or otherwise only the first instance of the assembly may be rewritten. + if (currentArch != sourceArch) { + currentArch = sourceArch; + runState.resolver?.Dispose (); + runState.resolver = new DirectoryAssemblyResolver (this.CreateTaskLogger (), loadDebugSymbols: ReadSymbols, loadReaderParameters: readerParameters); + + // Add SearchDirectories for the current architecture's ResolvedAssemblies + foreach (var kvp in perArchAssemblies[sourceArch]) { + ITaskItem assembly = kvp.Value; + var path = Path.GetFullPath (Path.GetDirectoryName (assembly.ItemSpec)); + if (!runState.resolver.SearchDirectories.Contains (path)) { + runState.resolver.SearchDirectories.Add (path); } } - CopyIfChanged (source, destination); + // Set up the FixAbstractMethodsStep and AddKeepAlivesStep + runState.cache = new TypeDefinitionCache (); + runState.fixAbstractMethodsStep = new FixAbstractMethodsStep (runState.resolver, runState.cache, Log); + runState.addKeepAliveStep = new AddKeepAlivesStep (runState.resolver, runState.cache, Log, UsingAndroidNETSdk); + runState.fixLegacyResourceDesignerStep = new FixLegacyResourceDesignerStep (runState.resolver, Log); } - } + DoRunTask (source, destination, runState, writerParameters); + } + runState.resolver?.Dispose (); return !Log.HasLoggedErrors; + + AndroidTargetArch GetValidArchitecture (ITaskItem item) + { + AndroidTargetArch ret = MonoAndroidHelper.GetTargetArch (item); + if (ret == AndroidTargetArch.None) { + throw new InvalidOperationException ($"Internal error: assembly '{item}' doesn't target any architecture."); + } + + return ret; + } + } + + void DoRunTask (ITaskItem source, ITaskItem destination, RunState runState, WriterParameters writerParameters) + { + var assemblyName = Path.GetFileNameWithoutExtension (source.ItemSpec); + + // In .NET 6+, we can skip the main assembly + if (UsingAndroidNETSdk && !AddKeepAlives && assemblyName == TargetName) { + CopyIfChanged (source, destination); + return; + } + if (runState.fixAbstractMethodsStep!.IsProductOrSdkAssembly (assemblyName)) { + CopyIfChanged (source, destination); + return; + } + + // Check AppDomain usage on any non-Product or Sdk assembly + AssemblyDefinition? assemblyDefinition = null; + if (!UsingAndroidNETSdk) { + assemblyDefinition = runState.resolver!.GetAssembly (source.ItemSpec); + runState.fixAbstractMethodsStep.CheckAppDomainUsage (assemblyDefinition, (string msg) => Log.LogCodedWarning ("XA2000", msg)); + } + + // Only run the step on "MonoAndroid" assemblies + if (MonoAndroidHelper.IsMonoAndroidAssembly (source) && !MonoAndroidHelper.IsSharedRuntimeAssembly (source.ItemSpec)) { + if (assemblyDefinition == null) + assemblyDefinition = runState.resolver!.GetAssembly (source.ItemSpec); + + bool save = runState.fixAbstractMethodsStep.FixAbstractMethods (assemblyDefinition); + if (UseDesignerAssembly) + save |= runState.fixLegacyResourceDesignerStep!.ProcessAssemblyDesigner (assemblyDefinition); + if (AddKeepAlives) + save |= runState.addKeepAliveStep!.AddKeepAlives (assemblyDefinition); + if (save) { + Log.LogDebugMessage ($"Saving modified assembly: {destination.ItemSpec}"); + Directory.CreateDirectory (Path.GetDirectoryName (destination.ItemSpec)); + writerParameters.WriteSymbols = assemblyDefinition.MainModule.HasSymbols; + assemblyDefinition.Write (destination.ItemSpec, writerParameters); + return; + } + } + + CopyIfChanged (source, destination); } void CopyIfChanged (ITaskItem source, ITaskItem destination) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs index 5589bd37747..86f2a319cc0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Tasks/LinkerTests.cs @@ -429,29 +429,32 @@ public unsafe bool MyMethod (Android.OS.IBinder windowToken, [global::Android.Ru using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "Building a project should have succeded."); + string projectDir = Path.Combine (proj.Root, b.ProjectDirectory); var assemblyFile = "UnnamedProject.dll"; if (!isRelease || setLinkModeNone) { foreach (string abi in b.GetBuildAbis ()) { - CheckAssembly (b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, assemblyFile))); + CheckAssembly (b.Output.GetIntermediaryPath (Path.Combine ("android", "assets", abi, assemblyFile)), projectDir); } } else { - CheckAssembly (BuildTest.GetLinkedPath (b, true, assemblyFile)); + CheckAssembly (BuildTest.GetLinkedPath (b, true, assemblyFile), projectDir); } } - void CheckAssembly (string assemblyPath) + void CheckAssembly (string assemblyPath, string projectDir) { + string shortAssemblyPath = Path.GetRelativePath (projectDir, assemblyPath); + Console.WriteLine ($"CheckAssembly for '{shortAssemblyPath}'"); using var assembly = AssemblyDefinition.ReadAssembly (assemblyPath); - Assert.IsTrue (assembly != null); + Assert.IsTrue (assembly != null, $"Assembly '${shortAssemblyPath}' should have been loaded"); var td = assembly.MainModule.GetType ("UnnamedProject.MyClass"); - Assert.IsTrue (td != null); + Assert.IsTrue (td != null, $"`UnnamedProject.MyClass` type definition should have been found in assembly '{shortAssemblyPath}'"); var mr = td.GetMethods ().Where (m => m.Name == "MyMethod").FirstOrDefault (); - Assert.IsTrue (mr != null); + Assert.IsTrue (mr != null, $"`MyMethod` method reference should have been found (assembly '{shortAssemblyPath}')"); var md = mr.Resolve (); - Assert.IsTrue (md != null); + Assert.IsTrue (md != null, $"`MyMethod` method reference should have been resolved (assembly '{shortAssemblyPath}')"); bool hasKeepAliveCall = false; foreach (var i in md.Body.Instructions) { @@ -465,7 +468,8 @@ void CheckAssembly (string assemblyPath) break; } - Assert.IsTrue (hasKeepAliveCall == shouldAddKeepAlives); + string not = shouldAddKeepAlives ? String.Empty : " not"; + Assert.IsTrue (hasKeepAliveCall == shouldAddKeepAlives, $"KeepAlive call should{not} have been found (assembly '{shortAssemblyPath}')"); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index b84204ec8c5..fb598eadf2a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -611,6 +611,7 @@ public static Dictionary> GetPe static Dictionary> GetPerArchAssemblies (IEnumerable input, HashSet supportedTargetArches, bool validate, Func? shouldSkip = null) { + bool filterByTargetArches = supportedTargetArches.Count > 0; var assembliesPerArch = new Dictionary> (); foreach (ITaskItem assembly in input) { if (shouldSkip != null && shouldSkip (assembly)) { @@ -618,7 +619,7 @@ static Dictionary> GetPerArchAs } AndroidTargetArch arch = MonoAndroidHelper.GetTargetArch (assembly); - if (!supportedTargetArches.Contains (arch)) { + if (filterByTargetArches && !supportedTargetArches.Contains (arch)) { continue; } From 81a2162a3fa42ee5dd83e811c2985dab6e25f042 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 11 Jan 2024 17:47:14 +0100 Subject: [PATCH 061/143] Moo? --- .../.template.config/localize/templatestrings.cs.json | 3 ++- .../.template.config/localize/templatestrings.de.json | 3 ++- .../.template.config/localize/templatestrings.en.json | 11 ++++++----- .../.template.config/localize/templatestrings.es.json | 3 ++- .../.template.config/localize/templatestrings.fr.json | 3 ++- .../.template.config/localize/templatestrings.it.json | 3 ++- .../.template.config/localize/templatestrings.ja.json | 3 ++- .../.template.config/localize/templatestrings.ko.json | 3 ++- .../.template.config/localize/templatestrings.pl.json | 3 ++- .../localize/templatestrings.pt-BR.json | 3 ++- .../.template.config/localize/templatestrings.ru.json | 3 ++- .../.template.config/localize/templatestrings.tr.json | 3 ++- .../localize/templatestrings.zh-Hans.json | 3 ++- .../localize/templatestrings.zh-Hant.json | 3 ++- .../.template.config/localize/templatestrings.cs.json | 3 ++- .../.template.config/localize/templatestrings.de.json | 3 ++- .../.template.config/localize/templatestrings.en.json | 9 +++++---- .../.template.config/localize/templatestrings.es.json | 3 ++- .../.template.config/localize/templatestrings.fr.json | 3 ++- .../.template.config/localize/templatestrings.it.json | 3 ++- .../.template.config/localize/templatestrings.ja.json | 3 ++- .../.template.config/localize/templatestrings.ko.json | 3 ++- .../.template.config/localize/templatestrings.pl.json | 3 ++- .../localize/templatestrings.pt-BR.json | 3 ++- .../.template.config/localize/templatestrings.ru.json | 3 ++- .../.template.config/localize/templatestrings.tr.json | 3 ++- .../localize/templatestrings.zh-Hans.json | 3 ++- .../localize/templatestrings.zh-Hant.json | 3 ++- 28 files changed, 63 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.cs.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.cs.json index 96b6c6bace5..2d4b50ea6b6 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.cs.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.cs.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Šablona aktivity Androidu", "description": "Třída aktivity Androidu", - "symbols/namespace/description": "obor názvů pro vygenerovaný kód" + "symbols/namespace/description": "obor názvů pro vygenerovaný kód", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.de.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.de.json index 9ab59500642..a5adb998843 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.de.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.de.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Android-Aktivitätsvorlage", "description": "Eine Android-Aktivitätsklasse", - "symbols/namespace/description": "Namespace für den generierten Code" + "symbols/namespace/description": "Namespace für den generierten Code", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.en.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.en.json index 49e602ae83c..66a04db062c 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.en.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.en.json @@ -1,6 +1,7 @@ -{ - "author": "Microsoft", - "name": "Android Activity template", - "description": "An Android Activity class", - "symbols/namespace/description": "namespace for the generated code" +{ + "author": "Microsoft", + "name": "Android Activity template", + "description": "An Android Activity class", + "symbols/namespace/description": "namespace for the generated code", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.es.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.es.json index c2c4b99609a..3ba732ac7e0 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.es.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.es.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Plantilla de actividad de Android", "description": "Una clase de actividad de Android", - "symbols/namespace/description": "espacio de nombres para el código generado" + "symbols/namespace/description": "espacio de nombres para el código generado", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.fr.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.fr.json index d39d18bdcc1..e19f7434be5 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.fr.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.fr.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Modèle d’activité Android", "description": "Une classe d’activité Android", - "symbols/namespace/description": "espace de noms pour le code généré" + "symbols/namespace/description": "espace de noms pour le code généré", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.it.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.it.json index c21590dc729..f26bd97997a 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.it.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.it.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Modello di attività Android", "description": "Classe di attività Android", - "symbols/namespace/description": "spazio dei nomi per il codice generato" + "symbols/namespace/description": "spazio dei nomi per il codice generato", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ja.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ja.json index 040ce9e560a..1978f9b18d3 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ja.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ja.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Android アクティビティ テンプレート", "description": "Android アクティビティ クラス", - "symbols/namespace/description": "生成されたコードの名前空間" + "symbols/namespace/description": "生成されたコードの名前空間", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ko.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ko.json index 4b7836a4823..93569cf106e 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ko.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ko.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Android 활동 템플릿", "description": "Android 활동 클래스", - "symbols/namespace/description": "생성된 코드의 네임스페이스" + "symbols/namespace/description": "생성된 코드의 네임스페이스", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pl.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pl.json index f786efc3b6c..64cd628fce6 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pl.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pl.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Szablon Aktywność systemu Android", "description": "Klasa Aktywność systemu Android", - "symbols/namespace/description": "przestrzeń nazw wygenerowanego kodu." + "symbols/namespace/description": "przestrzeń nazw wygenerowanego kodu.", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pt-BR.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pt-BR.json index b24983ffa21..65709d2d53d 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pt-BR.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.pt-BR.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Modelo de Atividade do Android", "description": "Uma classe de Atividade do Android", - "symbols/namespace/description": "namespace do código gerado" + "symbols/namespace/description": "namespace do código gerado", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ru.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ru.json index 816867363e4..1d8a77cf250 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ru.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.ru.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Шаблон действий Android", "description": "Класс активности Android", - "symbols/namespace/description": "пространство имен для созданного кода" + "symbols/namespace/description": "пространство имен для созданного кода", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.tr.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.tr.json index 28816a8a679..b59d136b023 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.tr.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.tr.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Android Etkinlik şablonu", "description": "Android Etkinlik sınıfı", - "symbols/namespace/description": "oluşturulan kod için ad alanı" + "symbols/namespace/description": "oluşturulan kod için ad alanı", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hans.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hans.json index c1869379ee6..1ca1645ed23 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hans.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hans.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Android 活动模板", "description": "Android 活动类", - "symbols/namespace/description": "生成的代码的命名空间" + "symbols/namespace/description": "生成的代码的命名空间", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hant.json b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hant.json index 3ff5e84fc93..9289a5aca44 100644 --- a/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hant.json +++ b/src/Microsoft.Android.Templates/android-activity/.template.config/localize/templatestrings.zh-Hant.json @@ -2,5 +2,6 @@ "author": "Microsoft", "name": "Android 活動範本", "description": "Android 活動類別", - "symbols/namespace/description": "適用於產生之程式碼的命名空間" + "symbols/namespace/description": "適用於產生之程式碼的命名空間", + "postActions/openInEditor/description": "Opens Activity1.cs in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.cs.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.cs.json index 19468f6c06b..e7df33892b0 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.cs.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.cs.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Šablona rozložení pro Android", - "description": "Soubor rozložení Androidu (XML)" + "description": "Soubor rozložení Androidu (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.de.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.de.json index 9efa2e625d4..e06116083db 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.de.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.de.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Android-Layoutvorlage", - "description": "Eine Android-Layoutdatei (XML)" + "description": "Eine Android-Layoutdatei (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.en.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.en.json index 576c1260aa9..97dd8ac01e3 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.en.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.en.json @@ -1,5 +1,6 @@ -{ - "author": "Microsoft", - "name": "Android Layout template", - "description": "An Android layout (XML) file" +{ + "author": "Microsoft", + "name": "Android Layout template", + "description": "An Android layout (XML) file", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.es.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.es.json index a4ba036b699..6024c63403a 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.es.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.es.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Plantilla de diseño de Android", - "description": "Un archivo de diseño de Android (XML)" + "description": "Un archivo de diseño de Android (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.fr.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.fr.json index 0510754c32f..bcb4100be6c 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.fr.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.fr.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Modèle de disposition Android", - "description": "Fichier de disposition Android (XML)" + "description": "Fichier de disposition Android (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.it.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.it.json index 6312b8fe267..008fd522a22 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.it.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.it.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Modello di layout Android", - "description": "File di layout Android (XML)" + "description": "File di layout Android (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ja.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ja.json index 88961b84322..d73e996289b 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ja.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ja.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Android レイアウト テンプレート", - "description": "Android レイアウト (XML) ファイル" + "description": "Android レイアウト (XML) ファイル", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ko.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ko.json index b3453aea66d..08999834454 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ko.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ko.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Android 레이아웃 템플릿", - "description": "Android 레이아웃(XML) 파일" + "description": "Android 레이아웃(XML) 파일", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pl.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pl.json index d3d13274490..c696b001f50 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pl.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pl.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Szablon Układ systemu Android", - "description": "Plik układu systemu Android (XML)" + "description": "Plik układu systemu Android (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pt-BR.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pt-BR.json index 2352aa236ca..ba1b1470a48 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pt-BR.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.pt-BR.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Modelo de Layout do Android", - "description": "Um arquivo de layout do Android (XML)" + "description": "Um arquivo de layout do Android (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ru.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ru.json index 1649b2f1dfa..563979403c4 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ru.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.ru.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Шаблон макета Android", - "description": "Файл макета Android (XML)" + "description": "Файл макета Android (XML)", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.tr.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.tr.json index 9106f4a1483..8ccb7620316 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.tr.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.tr.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Android Düzeni şablonu", - "description": "Android düzeni (XML) dosyası" + "description": "Android düzeni (XML) dosyası", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hans.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hans.json index c7fb9a8681b..a56ebef7e32 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hans.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hans.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Android 布局模板", - "description": "Android 布局 (XML) 文件" + "description": "Android 布局 (XML) 文件", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file diff --git a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hant.json b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hant.json index 4d4caa80e3e..70cf426b1e4 100644 --- a/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hant.json +++ b/src/Microsoft.Android.Templates/android-layout/.template.config/localize/templatestrings.zh-Hant.json @@ -1,5 +1,6 @@ { "author": "Microsoft", "name": "Android 版面配置範本", - "description": "Android 配置 (XML) 檔案" + "description": "Android 配置 (XML) 檔案", + "postActions/openInEditor/description": "Opens Layout1.xml in the editor" } \ No newline at end of file From 2a0621efeecacd8f2d5139786d8861c935200eee Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 11 Jan 2024 18:34:24 +0100 Subject: [PATCH 062/143] Fix the `CheckPackageManagerAssemblyOrder` test --- .../Tasks/GeneratePackageManagerJava.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 767a3f3c7db..fc97a0c4111 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -344,13 +344,12 @@ void AddEnvironment () } } - var nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( - ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), - RegisteredTaskObjectLifetime.Build - ); - - if (nativeCodeGenStates == null) { - throw new InvalidOperationException ("Internal error: native code generation states not registered"); + Dictionary? nativeCodeGenStates = null; + if (enableMarshalMethods) { + nativeCodeGenStates = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal> ( + ProjectSpecificTaskObjectKey (GenerateJavaStubs.NativeCodeGenStateRegisterTaskKey), + RegisteredTaskObjectLifetime.Build + ); } bool haveRuntimeConfigBlob = !String.IsNullOrEmpty (RuntimeConfigBinFilePath) && File.Exists (RuntimeConfigBinFilePath); From ececd2dc6dcc828dcebc6343ff3254ce6e797b25 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 12 Jan 2024 10:50:43 +0100 Subject: [PATCH 063/143] Update apkdesc files --- .../BuildReleaseArm64SimpleDotNet.apkdesc | 6 +- .../BuildReleaseArm64XFormsDotNet.apkdesc | 294 +++++++++--------- 2 files changed, 150 insertions(+), 150 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc index f5aa405c5b7..e440cbace58 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64SimpleDotNet.apkdesc @@ -14,10 +14,10 @@ "Size": 61443 }, "lib/arm64-v8a/#Mono.Android.dll.so": { - "Size": 90523 + "Size": 90421 }, "lib/arm64-v8a/#Mono.Android.Runtime.dll.so": { - "Size": 5513 + "Size": 5146 }, "lib/arm64-v8a/#System.Console.dll.so": { "Size": 6543 @@ -29,7 +29,7 @@ "Size": 553334 }, "lib/arm64-v8a/#System.Runtime.dll.so": { - "Size": 2623 + "Size": 2547 }, "lib/arm64-v8a/#System.Runtime.InteropServices.dll.so": { "Size": 4022 diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc index 6dac615b231..6db0959d7f0 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Base/BuildReleaseArm64XFormsDotNet.apkdesc @@ -4,224 +4,224 @@ "AndroidManifest.xml": { "Size": 3904 }, - "assemblies/_Microsoft.Android.Resource.Designer.dll": { - "Size": 2281 + "classes.dex": { + "Size": 5908848 }, - "assemblies/FormsViewGroup.dll": { - "Size": 8099 + "lib/arm64-v8a/#_Microsoft.Android.Resource.Designer.dll.so": { + "Size": 2279 }, - "assemblies/Java.Interop.dll": { - "Size": 69705 + "lib/arm64-v8a/#FormsViewGroup.dll.so": { + "Size": 8089 }, - "assemblies/Mono.Android.dll": { - "Size": 456294 + "lib/arm64-v8a/#Java.Interop.dll.so": { + "Size": 69523 }, - "assemblies/Mono.Android.Runtime.dll": { - "Size": 5148 + "lib/arm64-v8a/#Mono.Android.dll.so": { + "Size": 454202 }, - "assemblies/mscorlib.dll": { - "Size": 4052 + "lib/arm64-v8a/#Mono.Android.Runtime.dll.so": { + "Size": 5146 }, - "assemblies/netstandard.dll": { - "Size": 5643 + "lib/arm64-v8a/#mscorlib.dll.so": { + "Size": 4045 }, - "assemblies/rc.bin": { - "Size": 1512 + "lib/arm64-v8a/#netstandard.dll.so": { + "Size": 5628 }, - "assemblies/System.Collections.Concurrent.dll": { - "Size": 11530 + "lib/arm64-v8a/#System.Collections.Concurrent.dll.so": { + "Size": 11516 }, - "assemblies/System.Collections.dll": { - "Size": 15430 + "lib/arm64-v8a/#System.Collections.dll.so": { + "Size": 15411 }, - "assemblies/System.Collections.NonGeneric.dll": { - "Size": 7452 + "lib/arm64-v8a/#System.Collections.NonGeneric.dll.so": { + "Size": 7443 }, - "assemblies/System.ComponentModel.dll": { - "Size": 1939 + "lib/arm64-v8a/#System.ComponentModel.dll.so": { + "Size": 1935 }, - "assemblies/System.ComponentModel.Primitives.dll": { - "Size": 2549 + "lib/arm64-v8a/#System.ComponentModel.Primitives.dll.so": { + "Size": 2547 }, - "assemblies/System.ComponentModel.TypeConverter.dll": { - "Size": 6033 + "lib/arm64-v8a/#System.ComponentModel.TypeConverter.dll.so": { + "Size": 6028 }, - "assemblies/System.Console.dll": { - "Size": 6578 + "lib/arm64-v8a/#System.Console.dll.so": { + "Size": 6575 }, - "assemblies/System.Core.dll": { - "Size": 1974 + "lib/arm64-v8a/#System.Core.dll.so": { + "Size": 1970 }, - "assemblies/System.Diagnostics.DiagnosticSource.dll": { - "Size": 9068 + "lib/arm64-v8a/#System.Diagnostics.DiagnosticSource.dll.so": { + "Size": 9060 }, - "assemblies/System.Diagnostics.TraceSource.dll": { - "Size": 6552 + "lib/arm64-v8a/#System.Diagnostics.TraceSource.dll.so": { + "Size": 6545 }, - "assemblies/System.dll": { - "Size": 2328 + "lib/arm64-v8a/#System.dll.so": { + "Size": 2326 }, - "assemblies/System.Drawing.dll": { - "Size": 1939 + "lib/arm64-v8a/#System.Drawing.dll.so": { + "Size": 1934 }, - "assemblies/System.Drawing.Primitives.dll": { - "Size": 11975 + "lib/arm64-v8a/#System.Drawing.Primitives.dll.so": { + "Size": 11961 }, - "assemblies/System.IO.Compression.Brotli.dll": { - "Size": 11196 + "lib/arm64-v8a/#System.IO.Compression.Brotli.dll.so": { + "Size": 11185 }, - "assemblies/System.IO.Compression.dll": { - "Size": 15880 + "lib/arm64-v8a/#System.IO.Compression.dll.so": { + "Size": 15862 }, - "assemblies/System.IO.IsolatedStorage.dll": { - "Size": 9875 + "lib/arm64-v8a/#System.IO.IsolatedStorage.dll.so": { + "Size": 9866 }, - "assemblies/System.Linq.dll": { - "Size": 19599 + "lib/arm64-v8a/#System.Linq.dll.so": { + "Size": 19573 }, - "assemblies/System.Linq.Expressions.dll": { - "Size": 165117 + "lib/arm64-v8a/#System.Linq.Expressions.dll.so": { + "Size": 164633 }, - "assemblies/System.Net.Http.dll": { - "Size": 67653 + "lib/arm64-v8a/#System.Net.Http.dll.so": { + "Size": 67537 }, - "assemblies/System.Net.Primitives.dll": { - "Size": 22432 + "lib/arm64-v8a/#System.Net.Primitives.dll.so": { + "Size": 22414 }, - "assemblies/System.Net.Requests.dll": { - "Size": 3602 + "lib/arm64-v8a/#System.Net.Requests.dll.so": { + "Size": 3591 }, - "assemblies/System.ObjectModel.dll": { - "Size": 8699 + "lib/arm64-v8a/#System.ObjectModel.dll.so": { + "Size": 8687 }, - "assemblies/System.Private.CoreLib.dll": { - "Size": 850036 + "lib/arm64-v8a/#System.Private.CoreLib.dll.so": { + "Size": 847568 }, - "assemblies/System.Private.DataContractSerialization.dll": { - "Size": 193991 + "lib/arm64-v8a/#System.Private.DataContractSerialization.dll.so": { + "Size": 193441 }, - "assemblies/System.Private.Uri.dll": { - "Size": 42860 + "lib/arm64-v8a/#System.Private.Uri.dll.so": { + "Size": 42797 }, - "assemblies/System.Private.Xml.dll": { - "Size": 216226 + "lib/arm64-v8a/#System.Private.Xml.dll.so": { + "Size": 215575 }, - "assemblies/System.Private.Xml.Linq.dll": { - "Size": 16639 + "lib/arm64-v8a/#System.Private.Xml.Linq.dll.so": { + "Size": 16618 }, - "assemblies/System.Runtime.dll": { - "Size": 2708 + "lib/arm64-v8a/#System.Runtime.dll.so": { + "Size": 2703 }, - "assemblies/System.Runtime.InteropServices.dll": { - "Size": 4028 + "lib/arm64-v8a/#System.Runtime.InteropServices.dll.so": { + "Size": 4022 }, - "assemblies/System.Runtime.Serialization.dll": { - "Size": 1865 + "lib/arm64-v8a/#System.Runtime.Serialization.dll.so": { + "Size": 1860 }, - "assemblies/System.Runtime.Serialization.Formatters.dll": { - "Size": 2484 + "lib/arm64-v8a/#System.Runtime.Serialization.Formatters.dll.so": { + "Size": 2482 }, - "assemblies/System.Runtime.Serialization.Primitives.dll": { - "Size": 3758 + "lib/arm64-v8a/#System.Runtime.Serialization.Primitives.dll.so": { + "Size": 3750 }, - "assemblies/System.Security.Cryptography.dll": { - "Size": 8111 + "lib/arm64-v8a/#System.Security.Cryptography.dll.so": { + "Size": 8102 }, - "assemblies/System.Text.RegularExpressions.dll": { - "Size": 159112 + "lib/arm64-v8a/#System.Text.RegularExpressions.dll.so": { + "Size": 158797 }, - "assemblies/System.Xml.dll": { - "Size": 1758 + "lib/arm64-v8a/#System.Xml.dll.so": { + "Size": 1755 }, - "assemblies/System.Xml.Linq.dll": { - "Size": 1774 + "lib/arm64-v8a/#System.Xml.Linq.dll.so": { + "Size": 1770 }, - "assemblies/UnnamedProject.dll": { - "Size": 5015 + "lib/arm64-v8a/#UnnamedProject.dll.so": { + "Size": 5006 }, - "assemblies/Xamarin.AndroidX.Activity.dll": { - "Size": 13842 + "lib/arm64-v8a/#Xamarin.AndroidX.Activity.dll.so": { + "Size": 13816 }, - "assemblies/Xamarin.AndroidX.AppCompat.AppCompatResources.dll": { - "Size": 6227 + "lib/arm64-v8a/#Xamarin.AndroidX.AppCompat.AppCompatResources.dll.so": { + "Size": 6217 }, - "assemblies/Xamarin.AndroidX.AppCompat.dll": { - "Size": 134494 + "lib/arm64-v8a/#Xamarin.AndroidX.AppCompat.dll.so": { + "Size": 133782 }, - "assemblies/Xamarin.AndroidX.CardView.dll": { - "Size": 6977 + "lib/arm64-v8a/#Xamarin.AndroidX.CardView.dll.so": { + "Size": 6959 }, - "assemblies/Xamarin.AndroidX.CoordinatorLayout.dll": { - "Size": 17886 + "lib/arm64-v8a/#Xamarin.AndroidX.CoordinatorLayout.dll.so": { + "Size": 17857 }, - "assemblies/Xamarin.AndroidX.Core.dll": { - "Size": 120018 + "lib/arm64-v8a/#Xamarin.AndroidX.Core.dll.so": { + "Size": 119399 }, - "assemblies/Xamarin.AndroidX.CursorAdapter.dll": { - "Size": 9002 + "lib/arm64-v8a/#Xamarin.AndroidX.CursorAdapter.dll.so": { + "Size": 8983 }, - "assemblies/Xamarin.AndroidX.DrawerLayout.dll": { - "Size": 15330 + "lib/arm64-v8a/#Xamarin.AndroidX.DrawerLayout.dll.so": { + "Size": 15299 }, - "assemblies/Xamarin.AndroidX.Fragment.dll": { - "Size": 45683 + "lib/arm64-v8a/#Xamarin.AndroidX.Fragment.dll.so": { + "Size": 45470 }, - "assemblies/Xamarin.AndroidX.Legacy.Support.Core.UI.dll": { - "Size": 6242 + "lib/arm64-v8a/#Xamarin.AndroidX.Legacy.Support.Core.UI.dll.so": { + "Size": 6233 }, - "assemblies/Xamarin.AndroidX.Lifecycle.Common.dll": { - "Size": 6574 + "lib/arm64-v8a/#Xamarin.AndroidX.Lifecycle.Common.dll.so": { + "Size": 6567 }, - "assemblies/Xamarin.AndroidX.Lifecycle.LiveData.Core.dll": { - "Size": 6749 + "lib/arm64-v8a/#Xamarin.AndroidX.Lifecycle.LiveData.Core.dll.so": { + "Size": 6739 }, - "assemblies/Xamarin.AndroidX.Lifecycle.ViewModel.dll": { - "Size": 6289 + "lib/arm64-v8a/#Xamarin.AndroidX.Lifecycle.ViewModel.dll.so": { + "Size": 6279 }, - "assemblies/Xamarin.AndroidX.Loader.dll": { - "Size": 13087 + "lib/arm64-v8a/#Xamarin.AndroidX.Loader.dll.so": { + "Size": 13068 }, - "assemblies/Xamarin.AndroidX.MultiDex.dll": { - "Size": 8354 + "lib/arm64-v8a/#Xamarin.AndroidX.MultiDex.dll.so": { + "Size": 8351 }, - "assemblies/Xamarin.AndroidX.RecyclerView.dll": { - "Size": 93990 + "lib/arm64-v8a/#Xamarin.AndroidX.RecyclerView.dll.so": { + "Size": 93516 }, - "assemblies/Xamarin.AndroidX.SavedState.dll": { - "Size": 4969 + "lib/arm64-v8a/#Xamarin.AndroidX.SavedState.dll.so": { + "Size": 4965 }, - "assemblies/Xamarin.AndroidX.SwipeRefreshLayout.dll": { - "Size": 13974 + "lib/arm64-v8a/#Xamarin.AndroidX.SwipeRefreshLayout.dll.so": { + "Size": 13946 }, - "assemblies/Xamarin.AndroidX.ViewPager.dll": { - "Size": 19073 + "lib/arm64-v8a/#Xamarin.AndroidX.ViewPager.dll.so": { + "Size": 19028 }, - "assemblies/Xamarin.Forms.Core.dll": { - "Size": 561410 + "lib/arm64-v8a/#Xamarin.Forms.Core.dll.so": { + "Size": 559493 }, - "assemblies/Xamarin.Forms.Platform.Android.dll": { - "Size": 372848 + "lib/arm64-v8a/#Xamarin.Forms.Platform.Android.dll.so": { + "Size": 371735 }, - "assemblies/Xamarin.Forms.Platform.dll": { - "Size": 17182 + "lib/arm64-v8a/#Xamarin.Forms.Platform.dll.so": { + "Size": 17172 }, - "assemblies/Xamarin.Forms.Xaml.dll": { - "Size": 63517 + "lib/arm64-v8a/#Xamarin.Forms.Xaml.dll.so": { + "Size": 63405 }, - "assemblies/Xamarin.Google.Android.Material.dll": { - "Size": 66417 + "lib/arm64-v8a/#Xamarin.Google.Android.Material.dll.so": { + "Size": 66088 }, - "assemblies/Xamarin.Google.Guava.ListenableFuture.dll": { - "Size": 10640 + "lib/arm64-v8a/#Xamarin.Google.Guava.ListenableFuture.dll.so": { + "Size": 10633 }, - "classes.dex": { - "Size": 5908848 + "lib/arm64-v8a/arc.bin.so": { + "Size": 1512 }, "lib/arm64-v8a/libmono-component-marshal-ilgen.so": { "Size": 87080 }, "lib/arm64-v8a/libmonodroid.so": { - "Size": 339864 + "Size": 340776 }, "lib/arm64-v8a/libmonosgen-2.0.so": { "Size": 3184512 @@ -236,7 +236,7 @@ "Size": 155568 }, "lib/arm64-v8a/libxamarin-app.so": { - "Size": 113968 + "Size": 115040 }, "META-INF/androidx.activity_activity.version": { "Size": 6 @@ -374,13 +374,13 @@ "Size": 1223 }, "META-INF/BNDLTOOL.SF": { - "Size": 89914 + "Size": 90407 }, "META-INF/com.google.android.material_material.version": { "Size": 6 }, "META-INF/MANIFEST.MF": { - "Size": 89787 + "Size": 90280 }, "META-INF/maven/com.google.guava/listenablefuture/pom.properties": { "Size": 96 @@ -2282,5 +2282,5 @@ "Size": 777972 } }, - "PackageSize": 9593384 + "PackageSize": 8881170 } \ No newline at end of file From 6bc842caf53e4dd2bae869f972dcf1e946ec1078 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 12 Jan 2024 11:55:53 +0100 Subject: [PATCH 064/143] Fix the `BuildAMassiveApp` test after a34988c a34988cf5623ec25e13dca39d1337e8e606f6fc0 removed support for the `Android.Support*` packages, which seems to have broken the `ACWMapGenerator` code in this PR and caused `BuildAMassiveApp` to fail (possibly also other tests relying on Xamarin.Forms 4.x). This commit bumps all of those tests to use `XF` and `XFMaps` 5.0.0.2515. --- .../Android/XamarinFormsAndroidApplicationProject.cs | 2 +- .../Android/XamarinFormsMapsApplicationProject.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs index 598f62a9139..2ed381d143c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs @@ -43,7 +43,7 @@ public XamarinFormsAndroidApplicationProject (string debugConfigurationName = "D { // Don't opt into ImplicitUsings RemoveProperty (KnownProperties.ImplicitUsings); - PackageReferences.Add (KnownPackages.XamarinForms_4_7_0_1142); + PackageReferences.Add (KnownPackages.XamarinFormsMaps_5_0_0_2515); AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\values\\colors.xml") { TextContent = () => colors_xml, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs index b83416c51d6..cb325b09946 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs @@ -17,7 +17,7 @@ static XamarinFormsMapsApplicationProject () public XamarinFormsMapsApplicationProject ([CallerMemberName] string packageName = "") : base (packageName: packageName) { - PackageReferences.Add (KnownPackages.XamarinFormsMaps_4_7_0_1142); + PackageReferences.Add (KnownPackages.XamarinFormsMaps_5_0_0_2515); PackageReferences.Add (KnownPackages.Xamarin_GooglePlayServices_Base); PackageReferences.Add (KnownPackages.Xamarin_GooglePlayServices_Basement); PackageReferences.Add (KnownPackages.Xamarin_GooglePlayServices_Maps); From eb84482527d474b0b49f3a130b17e65f8c15ad12 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Fri, 12 Jan 2024 16:31:37 +0100 Subject: [PATCH 065/143] Fix the `DuplicateManagedNames` test again... ...after the recent changes it broke again --- src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index cce58c94a64..f8ee5b45953 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -250,7 +250,7 @@ void Run (bool useMarshalMethods) var acwMapGen = new ACWMapGenerator (Log); if (!acwMapGen.Generate (templateCodeGenState, AcwMapFile)) { - return; + Log.LogDebugMessage ("ACW map generation failed"); } IList additionalProviders = MergeManifest (templateCodeGenState, MaybeGetArchAssemblies (userAssembliesPerArch, templateCodeGenState.TargetArch)); From 87e20670377bbb796623930bad377be66b3203c6 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 15 Jan 2024 12:25:34 +0100 Subject: [PATCH 066/143] Apply Android.Support changes --- .../Properties/Resources.resx | 5 + .../AndroidUpdateResourcesTest.cs | 46 +- .../BindingBuildTest.cs | 40 +- .../Xamarin.Android.Build.Tests/BuildTest.cs | 60 +-- .../Xamarin.Android.Build.Tests/BuildTest2.cs | 110 +---- .../BuildWithLibraryTests.cs | 8 +- .../DesignerTests.cs | 27 +- .../IncrementalBuildTest.cs | 41 +- .../ManifestTest.cs | 16 +- .../PackagingTest.cs | 36 +- .../Xamarin.Android.Build.Tests/WearTests.cs | 31 +- .../Android/KnownPackages.cs | 419 ++---------------- .../XamarinAndroidWearApplicationProject.cs | 24 +- .../XamarinFormsAndroidApplicationProject.cs | 6 +- .../XamarinFormsMapsApplicationProject.cs | 4 +- .../Resources/Wear/LayoutMain.axml | 11 - .../Resources/Wear/MainActivity.cs | 41 +- .../Xamarin.Android.Common.targets | 14 + .../LinkDescTest/MainActivityReplacement.cs | 2 +- .../Tests/InstallAndRunTests.cs | 6 - .../Tests/InstantRunTest.cs | 17 +- .../Tests/PerformanceTest.cs | 5 +- 22 files changed, 165 insertions(+), 804 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/LayoutMain.axml diff --git a/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx b/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx index 524a3f1fe3c..ae7fad2dc39 100644 --- a/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx +++ b/src/Xamarin.Android.Build.Tasks/Properties/Resources.resx @@ -1006,4 +1006,9 @@ To use a custom JDK path for a command line build, set the 'JavaSdkDirectory' MS {0} - The deprecated MSBuild property name {1} - The numeric version of .NET + + The Android Support libraries are no longer supported in .NET {0}, please migrate to AndroidX. See https://aka.ms/xamarin/androidx for more details. + The following are literal names and should not be translated: Android Support, AndroidX, .NET. +{0} - The numeric version of .NET + \ No newline at end of file diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs index f73f76da2a7..44fd9c2c3fd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AndroidUpdateResourcesTest.cs @@ -138,23 +138,17 @@ public void DesignTimeBuild ([Values(false, true)] bool isRelease, [Values (fals } [Test] - public void CheckEmbeddedSupportLibraryResources () + public void CheckEmbeddedAndroidXResources () { var proj = new XamarinAndroidApplicationProject () { IsRelease = true, PackageReferences = { - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, + KnownPackages.AndroidXAppCompat }, }; using (var b = CreateApkBuilder ()) { Assert.IsTrue (b.Build (proj), "First build should have succeeded."); - var Rdrawable = b.Output.GetIntermediaryPath (Path.Combine ("android", "bin", "classes", "android", "support", "v7", "appcompat", "R$drawable.class")); + var Rdrawable = b.Output.GetIntermediaryPath (Path.Combine ("android", "bin", "classes", "androidx", "appcompat", "R$drawable.class")); Assert.IsTrue (File.Exists (Rdrawable), $"{Rdrawable} should exist"); } } @@ -410,8 +404,7 @@ protected override void OnClick() } }" }); - proj.PackageReferences.Add (KnownPackages.AndroidSupportV4_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportV7AppCompat_27_0_2_1); + proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat); using (var libb = CreateDllBuilder (Path.Combine (projectPath, lib.ProjectName), cleanupOnDispose: false)) using (var b = CreateApkBuilder (Path.Combine (projectPath, proj.ProjectName), cleanupOnDispose: false)) { Assert.IsTrue (libb.Build (lib), "Library Build should have succeeded."); @@ -830,11 +823,7 @@ public void CheckFilesAreRemoved () { ", } - }, - PackageReferences = { - KnownPackages.SupportV7AppCompat_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - }, + } }; using (var builder = CreateApkBuilder ()) { Assert.IsTrue (builder.Build (proj), "Build should have succeeded"); @@ -1033,13 +1022,7 @@ public void BuildAppWithManagedResourceParserAndLibraries () new BuildItem.ProjectReference (@"..\Lib1\Lib1.csproj", libProj.ProjectName, libProj.ProjectGuid), }, PackageReferences = { - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, + KnownPackages.AndroidXAppCompat }, }; appProj.SetProperty ("AndroidUseManagedDesignTimeResourceGenerator", "True"); @@ -1224,20 +1207,7 @@ public void CustomViewAddResourceId () { var proj = new XamarinAndroidApplicationProject (); proj.LayoutMain = proj.LayoutMain.Replace ("", ""); - proj.PackageReferences.Add (KnownPackages.Android_Arch_Core_Common_26_1_0); - proj.PackageReferences.Add (KnownPackages.Android_Arch_Lifecycle_Common_26_1_0); - proj.PackageReferences.Add (KnownPackages.Android_Arch_Lifecycle_Runtime_26_1_0); - proj.PackageReferences.Add (KnownPackages.AndroidSupportV4_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportCompat_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportCoreUI_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportCoreUtils_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportDesign_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportFragment_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportMediaCompat_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportV7AppCompat_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportV7CardView_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportV7MediaRouter_27_0_2_1); - proj.PackageReferences.Add (KnownPackages.SupportV7RecyclerView_27_0_2_1); + proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat); using (var b = CreateApkBuilder (Path.Combine ("temp", TestName))) { Assert.IsTrue (b.Build (proj), "first build should have succeeded"); @@ -1257,7 +1227,7 @@ public void CustomViewAddResourceId () var r_java = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "android", "src", proj.PackageNameJavaIntermediatePath, "R.java"); FileAssert.Exists (r_java); var r_java_contents = File.ReadAllLines (r_java); - Assert.IsTrue (StringAssertEx.ContainsText (r_java_contents, textView1), $"android/support/compat/R.java should contain `{textView1}`!"); + Assert.IsTrue (StringAssertEx.ContainsText (r_java_contents, textView1), $"{r_java} should contain `{textView1}`!"); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs index 860c845eb67..66a14fe783b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs @@ -167,7 +167,6 @@ public void BuildAarBindigLibraryWithNuGetPackageOfJar (string classParser) var proj = new XamarinAndroidBindingProject () { IsRelease = true, }; - proj.PackageReferences.Add (KnownPackages.AndroidSupportV4_27_0_2_1); proj.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\android-crop-1.0.1.aar") { WebContent = "https://repo1.maven.org/maven2/com/soundcloud/android/android-crop/1.0.1/android-crop-1.0.1.aar" }); @@ -187,13 +186,12 @@ public void BuildAarBindigLibraryWithNuGetPackageOfJar (string classParser) [Test] [TestCaseSource (nameof (ClassParseOptions))] [NonParallelizable] - public void BuildLibraryZipBindigLibraryWithAarOfJar (string classParser) + public void BuildLibraryZipBindingLibraryWithAarOfJar (string classParser) { var proj = new XamarinAndroidBindingProject () { IsRelease = true, }; proj.AndroidClassParser = classParser; - proj.PackageReferences.Add (KnownPackages.AndroidSupportV4_27_0_2_1); proj.Jars.Add (new AndroidItem.LibraryProjectZip ("Jars\\aFileChooserBinaries.zip") { WebContentFileNameFromAzure = "aFileChooserBinaries.zip" }); @@ -202,41 +200,6 @@ public void BuildLibraryZipBindigLibraryWithAarOfJar (string classParser) Java.Lang.Object LoadInBackgroundImpl "; - proj.Sources.Add (new BuildItem (BuildActions.Compile, "Fixup.cs") { - TextContent = () => @"using System; -using System.Collections.Generic; -using Android.App; -using Android.Runtime; - -namespace Com.Ipaulpro.Afilechooser { - [Activity (Name = ""com.ipaulpro.afilechooser.FileChooserActivity"", - Icon = ""@drawable/ic_chooser"", - Exported = true)] - [IntentFilter (new string [] {""android.intent.action.GET_CONTENT""}, - Categories = new string [] { - ""android.intent.category.DEFAULT"", - //""android.intent.category.OPENABLE"" - }, - DataMimeType = ""*/*"")] - public partial class FileChooserActivity - { - } - - public partial class FileListFragment : global::Android.Support.V4.App.ListFragment, global::Android.Support.V4.App.LoaderManager.ILoaderCallbacks { - - public void OnLoadFinished (global::Android.Support.V4.Content.Loader p0, Java.Lang.Object p1) - { - OnLoadFinished (p0, (IList) new JavaList (p1.Handle, JniHandleOwnership.DoNotTransfer)); - } - } - public partial class FileLoader : Android.Support.V4.Content.AsyncTaskLoader { - public override Java.Lang.Object LoadInBackground () - { - return (Java.Lang.Object) LoadInBackgroundImpl (); - } - } -}" - }); using (var b = CreateDllBuilder ()) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); } @@ -467,7 +430,6 @@ public void RemoveEventHandlerResolution () Xamarin.ActionbarSherlockBinding.Views ", }; - binding.PackageReferences.Add (KnownPackages.AndroidSupportV4_27_0_2_1); using (var bindingBuilder = CreateDllBuilder (Path.Combine ("temp", "RemoveEventHandlerResolution", "Binding"))) { Assert.IsTrue (bindingBuilder.Build (binding), "binding build should have succeeded"); } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 6b734ca1447..03ccb901911 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -243,14 +243,6 @@ public void CheckAssemblyCounts (bool isRelease, bool aot) EmbedAssembliesIntoApk = true, AotAssemblies = aot, }; - proj.PackageReferences.Add (KnownPackages.AndroidXMigration); - proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat); - proj.PackageReferences.Add (KnownPackages.AndroidXAppCompatResources); - proj.PackageReferences.Add (KnownPackages.AndroidXBrowser); - proj.PackageReferences.Add (KnownPackages.AndroidXMediaRouter); - proj.PackageReferences.Add (KnownPackages.AndroidXLegacySupportV4); - proj.PackageReferences.Add (KnownPackages.AndroidXLifecycleLiveData); - proj.PackageReferences.Add (KnownPackages.XamarinGoogleAndroidMaterial); var abis = new [] { "armeabi-v7a", "x86" }; proj.SetRuntimeIdentifiers (abis); @@ -299,10 +291,6 @@ public void SmokeTestBuildWithSpecialCharacters ([Values (false, true)] bool for proj.IsRelease = true; proj.AotAssemblies = aot; - if (forms) { - proj.PackageReferences.Clear (); - proj.PackageReferences.Add (KnownPackages.XamarinForms_4_7_0_1142); - } using (var builder = CreateApkBuilder (Path.Combine (rootPath, proj.ProjectName))){ Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); } @@ -452,26 +440,6 @@ public void ApplicationIdPlaceholder () } } - [Test] - [Category ("XamarinBuildDownload")] - public void ExtraAaptManifest () - { - var proj = new XamarinAndroidApplicationProject (); - proj.MainActivity = proj.DefaultMainActivity.Replace ("base.OnCreate (bundle);", "base.OnCreate (bundle);\nCrashlytics.Crashlytics.HandleManagedExceptions();"); - proj.PackageReferences.Add (KnownPackages.Xamarin_Android_Crashlytics); - proj.PackageReferences.Add (KnownPackages.Xamarin_Android_Fabric); - proj.PackageReferences.Add (KnownPackages.Xamarin_Build_Download); - using (var builder = CreateApkBuilder (Path.Combine ("temp", TestName))) { - builder.Target = "Restore"; - Assert.IsTrue (builder.Build (proj), "Restore should have succeeded."); - builder.Target = "Build"; - Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); - var manifest = File.ReadAllText (Path.Combine (Root, builder.ProjectDirectory, "obj", "Debug", "android", "AndroidManifest.xml")); - Assert.IsTrue (manifest.Contains ($"android:authorities=\"{proj.PackageName}.crashlyticsinitprovider\""), "placeholder not replaced"); - Assert.IsFalse (manifest.Contains ("dollar_openBracket_applicationId_closeBracket"), "`aapt/AndroidManifest.xml` not ignored"); - } - } - [Test] public void AarContentExtraction () { @@ -721,7 +689,10 @@ public void BuildAfterUpgradingNuget () var proj = new XamarinAndroidApplicationProject (); proj.MainActivity = proj.DefaultMainActivity.Replace ("public class MainActivity : Activity", "public class MainActivity : AndroidX.AppCompat.App.AppCompatActivity"); - proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat); + proj.PackageReferences.Add (new Package { + Id = "Xamarin.AndroidX.AppCompat", + Version = "1.6.1.5", + }); using (var b = CreateApkBuilder (Path.Combine ("temp", TestContext.CurrentContext.Test.Name))) { //[TearDown] will still delete if test outcome successful, I need logs if assertions fail but build passes @@ -739,8 +710,8 @@ public void BuildAfterUpgradingNuget () FileAssert.Exists (build_props, "build.props should exist after first build."); proj.PackageReferences.Clear (); - //NOTE: we can get all the other dependencies transitively, yay! - proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat_1_6_0_1); + //NOTE: this should be newer than specified above + proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat); b.Save (proj, doNotCleanupOnUpdate: true); Assert.IsTrue (b.Build (proj), "second build should have succeeded."); Assert.IsFalse (b.Output.IsTargetSkipped ("_CleanIntermediateIfNeeded"), "`_CleanIntermediateIfNeeded` should have run for the second build!"); @@ -1172,6 +1143,7 @@ public MyWorker (Context c, WorkerParameters p) : base (c, p) { } " }); proj.PackageReferences.Add (KnownPackages.AndroidXWorkRuntime); + proj.PackageReferences.Add (KnownPackages.AndroidXLifecycleLiveData); using (var b = CreateApkBuilder (Path.Combine ("temp", TestName))) { Assert.IsTrue (b.Build (proj), "Build should have succeeded."); } @@ -1443,12 +1415,7 @@ public void CheckLintErrorsAndWarnings () { string disabledIssues = "StaticFieldLeak,ObsoleteSdkInt,AllowBackup,ExportedReceiver,RedundantLabel"; - var proj = new XamarinAndroidApplicationProject () { - PackageReferences = { - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportConstraintLayout_1_0_2_2, - }, - }; + var proj = new XamarinAndroidApplicationProject (); proj.SetProperty ("AndroidLintEnabled", true.ToString ()); proj.SetProperty ("AndroidLintDisabledIssues", disabledIssues); proj.SetProperty ("AndroidLintEnabledIssues", ""); @@ -1465,14 +1432,12 @@ public class MainActivity : Activity TextContent = () => { return @" "; } @@ -1628,25 +1593,18 @@ public void DuplicateValuesInResourceCaseMap () [Test] public void CheckLintResourceFileReferencesAreFixed () { - var proj = new XamarinAndroidApplicationProject () { - PackageReferences = { - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportConstraintLayout_1_0_2_2, - }, - }; + var proj = new XamarinAndroidApplicationProject (); proj.SetProperty ("AndroidLintEnabled", true.ToString ()); proj.AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\layout\\test.axml") { TextContent = () => { return @" new byte[10], MetadataValues = "Link=libs\\x86\\libtest.so", }, + new AndroidItem.AndroidNativeLibrary ("armeabi-v7a\\libRSSupport.so") { + BinaryContent = () => new byte[10], + }, }, }; var dll2 = new XamarinAndroidLibraryProject () { @@ -379,12 +382,9 @@ public void BuildWithNativeLibraries ([Values (true, false)] bool isRelease) new AndroidItem.AndroidNativeLibrary ("armeabi-v7a\\libRSSupport.so") { BinaryContent = () => new byte[10], }, - }, - PackageReferences = { - KnownPackages.Xamarin_Android_Support_v8_RenderScript_28_0_0_3, } }; - proj.SetAndroidSupportedAbis ("armeabi-v7a", "x86"); + proj.SetRuntimeIdentifiers (["armeabi-v7a", "x86"]); var path = Path.Combine (Root, "temp", string.Format ("BuildWithNativeLibraries_{0}", isRelease)); using (var b1 = CreateDllBuilder (Path.Combine (path, dll2.ProjectName))) { Assert.IsTrue (b1.Build (dll2), "Build should have succeeded."); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/DesignerTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/DesignerTests.cs index 236c5c574af..50a90784a83 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/DesignerTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/DesignerTests.cs @@ -31,16 +31,7 @@ public void CustomDesignerTargetSetupDependenciesForDesigner () }, }, }; - var proj = new XamarinAndroidApplicationProject () { - PackageReferences = { - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, - }, + var proj = new XamarinFormsAndroidApplicationProject () { References = { new BuildItem ("ProjectReference", "..\\Library1\\Library1.csproj") }, Imports = { new Import ("foo.targets") { @@ -152,7 +143,7 @@ public void IncrementalDesignTimeBuild () var resourcepathscache = Path.Combine (Root, b.ProjectDirectory, proj.IntermediateOutputPath, "designtime", "libraryprojectimports.cache"); FileAssert.Exists (resourcepathscache); var doc = XDocument.Load (resourcepathscache); - Assert.AreEqual (37, doc.Root.Element ("Jars").Elements ("Jar").Count (), "libraryprojectimports.cache did not contain expected jar files"); + Assert.AreEqual (54, doc.Root.Element ("Jars").Elements ("Jar").Count (), "libraryprojectimports.cache did not contain expected jar files"); } } @@ -212,17 +203,7 @@ public void IncrementalFullBuild () public void GetExtraLibraryLocationsForDesigner () { var target = "GetExtraLibraryLocationsForDesigner"; - var proj = new XamarinAndroidApplicationProject () { - PackageReferences = { - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, - }, - }; + var proj = new XamarinFormsAndroidApplicationProject (); string jar = "gson-2.7.jar"; proj.OtherBuildItems.Add (new BuildItem ("AndroidJavaLibrary", jar) { WebContent = $"https://repo1.maven.org/maven2/com/google/code/gson/gson/2.7/{jar}" @@ -231,7 +212,7 @@ public void GetExtraLibraryLocationsForDesigner () WebContent = "https://repo1.maven.org/maven2/com/soundcloud/android/android-crop/1.0.1/android-crop-1.0.1.aar" }); // Each NuGet package and AAR file are in libraryprojectimports.cache, AndroidJavaSource is not - int libraryProjectImportsJars = 14; + const int libraryProjectImportsJars = 55; using (var b = CreateApkBuilder (Path.Combine ("temp", TestName), false, false)) { // GetExtraLibraryLocationsForDesigner on new project Assert.IsTrue (b.RunTarget (proj, target, parameters: DesignerParameters), $"build should have succeeded for target `{target}` 1"); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs index 051d154d339..b4eae1ec907 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/IncrementalBuildTest.cs @@ -829,10 +829,10 @@ public void ResolveLibraryProjectImports () Assert.IsTrue (b.Build (proj), "second build should have succeeded."); FileAssert.Exists (cacheFile); var actual = ReadCache (cacheFile); - CollectionAssert.AreEqual (actual.Jars.Select (j => j.ItemSpec), - expected.Jars.Select (j => j.ItemSpec)); - CollectionAssert.AreEqual (actual.ResolvedResourceDirectories.Select (j => j.ItemSpec), - expected.ResolvedResourceDirectories.Select (j => j.ItemSpec)); + CollectionAssert.AreEqual (actual.Jars.Select (j => j.ItemSpec).OrderBy (j => j), + expected.Jars.Select (j => j.ItemSpec).OrderBy (j => j)); + CollectionAssert.AreEqual (actual.ResolvedResourceDirectories.Select (j => j.ItemSpec).OrderBy (j => j), + expected.ResolvedResourceDirectories.Select (j => j.ItemSpec).OrderBy (j => j)); // Add a new AAR file to the project var aar = new AndroidItem.AndroidAarLibrary ("Jars\\android-crop-1.0.1.aar") { @@ -1373,39 +1373,6 @@ public void AndroidAssetMissing () } } - [Test] - [NonParallelizable] - public void AndroidXMigrationBug () - { - var proj = new XamarinFormsAndroidApplicationProject (); - proj.PackageReferences.Add (KnownPackages.AndroidXMigration); - proj.PackageReferences.Add (KnownPackages.AndroidXAppCompat); - proj.PackageReferences.Add (KnownPackages.AndroidXAppCompatResources); - proj.PackageReferences.Add (KnownPackages.AndroidXBrowser); - proj.PackageReferences.Add (KnownPackages.AndroidXMediaRouter); - proj.PackageReferences.Add (KnownPackages.AndroidXLegacySupportV4); - proj.PackageReferences.Add (KnownPackages.AndroidXLifecycleLiveData); - proj.PackageReferences.Add (KnownPackages.XamarinGoogleAndroidMaterial); - - string source = "class Foo { }"; - proj.Sources.Add (new BuildItem.Source ("Foo.cs") { TextContent = () => source }); - - using (var b = CreateApkBuilder ()) { - Assert.IsTrue (b.Build (proj), "first build should have succeeded."); - source = source.Replace ("Foo", "Bar"); - proj.Touch ("Foo.cs"); - Assert.IsTrue (b.Build (proj, doNotCleanupOnUpdate: true), "second build should have succeeded."); - var targets = new [] { - "_CompileResources", - "_UpdateAndroidResgen", - "_GenerateAndroidResourceDir", - }; - foreach (var target in targets) { - Assert.IsTrue (b.Output.IsTargetSkipped (target), $"`{target}` should be skipped."); - } - } - } - [Test] public void ChangeSupportedAbis () { diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 27e3a35ba57..c31a78a6ded 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -737,13 +737,7 @@ public void MergeLibraryManifest () new BuildItem.ProjectReference ("..\\Binding1\\Binding1.csproj", lib.ProjectGuid) }, PackageReferences = { - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, + KnownPackages.AndroidXAppCompat }, }; proj.SetProperty ("AndroidManifestMerger", "legacy"); @@ -759,7 +753,7 @@ public void MergeLibraryManifest () using Android.Runtime; using Android.Views; using Android.Widget; -using Android.Support.V4.App; +using AndroidX.Fragment.App; using Android.Util; [Activity (Label = ""TestActivity1"")] [IntentFilter (new[]{Intent.ActionMain}, Categories = new[]{ ""com.xamarin.sample"" })] @@ -779,7 +773,7 @@ public class TestActivity1 : FragmentActivity { using Android.Runtime; using Android.Views; using Android.Widget; -using Android.Support.V4.App; +using AndroidX.Fragment.App; using Android.Util; [Activity (Label = ""TestActivity2"")] [IntentFilter (new[]{Intent.ActionMain}, Categories = new[]{ ""com.xamarin.sample"" })] @@ -800,8 +794,8 @@ public class TestActivity2 : FragmentActivity { "${applicationId}.FacebookInitProvider was not replaced with com.xamarin.manifest.FacebookInitProvider"); Assert.IsTrue (manifest.Contains ("com.xamarin.test.internal.FacebookInitProvider"), ".internal.FacebookInitProvider was not replaced with com.xamarin.test.internal.FacebookInitProvider"); - Assert.AreEqual (manifest.IndexOf ("meta-data", StringComparison.OrdinalIgnoreCase), - manifest.LastIndexOf ("meta-data", StringComparison.OrdinalIgnoreCase), "There should be only one meta-data element"); + Assert.AreEqual (manifest.IndexOf ("android.support.VERSION", StringComparison.OrdinalIgnoreCase), + manifest.LastIndexOf ("android.support.VERSION", StringComparison.OrdinalIgnoreCase), "There should be only one android.support.VERSION meta-data element"); var doc = XDocument.Parse (manifest); var ns = XNamespace.Get ("http://schemas.android.com/apk/res/android"); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs index b85f6147908..14d6301989b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/PackagingTest.cs @@ -262,25 +262,7 @@ public void ExplicitPackageNamingPolicy () public void CheckMetadataSkipItemsAreProcessedCorrectly () { var packages = new List () { - KnownPackages.Android_Arch_Core_Common_26_1_0, - KnownPackages.Android_Arch_Lifecycle_Common_26_1_0, - KnownPackages.Android_Arch_Lifecycle_Runtime_26_1_0, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportDesign_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, - KnownPackages.SupportV7CardView_27_0_2_1, - KnownPackages.SupportV7MediaRouter_27_0_2_1, - KnownPackages.SupportV7RecyclerView_27_0_2_1, - KnownPackages.VectorDrawable_27_0_2_1, - new Package () { Id = "Xamarin.Android.Support.Annotations", Version = "27.0.2.1" }, - new Package () { Id = "Xamarin.Android.Support.Transition", Version = "27.0.2.1" }, - new Package () { Id = "Xamarin.Android.Support.v7.Palette", Version = "27.0.2.1" }, - new Package () { Id = "Xamarin.Android.Support.Animated.Vector.Drawable", Version = "27.0.2.1" }, + KnownPackages.Xamarin_Jetbrains_Annotations, }; string metaDataTemplate = @" @@ -651,20 +633,8 @@ public void CheckIncludedFilesArePresent () [TestCase (-1, 200)] public void BuildApkWithZipFlushLimits (int filesLimit, int sizeLimit) { - var proj = new XamarinAndroidApplicationProject { - IsRelease = false, - PackageReferences = { - KnownPackages.SupportDesign_27_0_2_1, - KnownPackages.SupportV7CardView_27_0_2_1, - KnownPackages.AndroidSupportV4_27_0_2_1, - KnownPackages.SupportCoreUtils_27_0_2_1, - KnownPackages.SupportMediaCompat_27_0_2_1, - KnownPackages.SupportFragment_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.SupportV7AppCompat_27_0_2_1, - KnownPackages.SupportV7MediaRouter_27_0_2_1, - }, + var proj = new XamarinFormsAndroidApplicationProject { + IsRelease = false }; proj.SetProperty ("EmbedAssembliesIntoApk", "true"); if (filesLimit > 0) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/WearTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/WearTests.cs index e6ca9dec4f3..95cf507ac37 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/WearTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/WearTests.cs @@ -7,27 +7,6 @@ namespace Xamarin.Android.Build.Tests [TestFixture] public class WearTests : BaseTest { - [Test] - public void ResolveLibraryImportsWithReadonlyFiles () - { - //NOTE: doesn't need to be a full Android Wear app - var proj = new XamarinAndroidApplicationProject { - PackageReferences = { - KnownPackages.AndroidWear_2_2_0, - KnownPackages.Android_Arch_Core_Common_26_1_0, - KnownPackages.Android_Arch_Lifecycle_Common_26_1_0, - KnownPackages.Android_Arch_Lifecycle_Runtime_26_1_0, - KnownPackages.SupportCompat_27_0_2_1, - KnownPackages.SupportCoreUI_27_0_2_1, - KnownPackages.SupportPercent_27_0_2_1, - KnownPackages.SupportV7RecyclerView_27_0_2_1, - }, - }; - using (var b = CreateApkBuilder ()) { - Assert.IsTrue (b.Build (proj), "Build should have succeeded."); - } - } - [Test] public void BasicProject ([Values (true, false)] bool isRelease) { @@ -42,7 +21,6 @@ public void BasicProject ([Values (true, false)] bool isRelease) [Test] public void BundledWearApp () { - var target = "_UpdateAndroidResgen"; var path = Path.Combine ("temp", TestName); var app = new XamarinAndroidApplicationProject { ProjectName = "MyApp", @@ -71,7 +49,7 @@ public void WearProjectJavaBuildFailure () IsRelease = true, EnableDefaultItems = true, PackageReferences = { - new Package { Id = "Xamarin.AndroidX.Wear", Version = "1.2.0.5" }, + KnownPackages.XamarinAndroidXWear, new Package { Id = "Xamarin.Android.Wear", Version = "2.2.0" }, new Package { Id = "Xamarin.AndroidX.PercentLayout", Version = "1.0.0.14" }, new Package { Id = "Xamarin.AndroidX.Legacy.Support.Core.UI", Version = "1.0.0.14" }, @@ -81,12 +59,7 @@ public void WearProjectJavaBuildFailure () var builder = CreateApkBuilder (); builder.ThrowOnBuildFailure = false; Assert.IsFalse (builder.Build (proj), $"{proj.ProjectName} should fail."); - var text = $"java.lang.RuntimeException"; - Assert.IsTrue (StringAssertEx.ContainsText (builder.LastBuildOutput, text), $"Output did not contain '{text}'"); - text = $"is defined multiple times"; - Assert.IsTrue (StringAssertEx.ContainsText (builder.LastBuildOutput, text), $"Output did not contain '{text}'"); - text = $"is from 'androidx.core.core.aar'"; - Assert.IsTrue (StringAssertEx.ContainsText (builder.LastBuildOutput, text), $"Output did not contain '{text}'"); + Assert.IsTrue (StringAssertEx.ContainsText (builder.LastBuildOutput, "error XA1039"), "Should receive error XA1039"); } } } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/KnownPackages.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/KnownPackages.cs index cacbe31e528..39fa4132b58 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/KnownPackages.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/KnownPackages.cs @@ -4,339 +4,49 @@ namespace Xamarin.ProjectTools { public static class KnownPackages { - public static Package AndroidSupportV4_27_0_2_1 = new Package () { - Id = "Xamarin.Android.Support.v4", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.v4") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.v4.27.0.2.1\\lib\\MonoAndroid70\\Xamarin.Android.Support.v4.dll" } - } - }; - public static Package AndroidWear_2_2_0 = new Package () { - Id = "Xamarin.Android.Wear", - Version = "2.2.0", - TargetFramework = "MonoAndroid80", - References = { - new BuildItem.Reference ("Xamarin.Android.Wearable") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Wear.2.2.0\\lib\\MonoAndroid80\\Xamarin.Android.Wear.dll" } - } - }; - public static Package SupportV7RecyclerView_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.v7.RecyclerView", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.V7.RecyclerView") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.v7.RecyclerView.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.v7.RecyclerView.dll" - } - } - }; - public static Package SupportV7CardView_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.v7.Cardview", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.v7.CardView") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.v7.CardView.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.v7.CardView.dll" } - } - }; - public static Package SupportV7AppCompat_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.v7.AppCompat", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.v7.AppCompat") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.v7.AppCompat.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.v7.AppCompat.dll" } - } - }; - public static Package SupportCompat_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Compat", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Compat") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Compat.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Compat.dll" } - } - }; - public static Package SupportCoreUI_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Core.UI", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Core.UI") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Core.UI.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Core.UI.dll" } - } - }; - public static Package SupportCoreUtils_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Core.Utils", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Core.Utils") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Core.Utils.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Core.Utils.dll" } - } - }; - public static Package SupportFragment_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Fragment", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Fragment") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Fragment.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Fragment.dll" } - } - }; - public static Package SupportMediaCompat_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Media.Compat", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Media.Compat") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Media.Compat.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Media.Compat.dll" } - } - }; - public static Package SupportPercent_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Percent", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Percent") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Percent.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Percent.dll" } - } - }; - public static Package SupportV7MediaRouter_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.v7.MediaRouter", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.v7.MediaRouter") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.v7.MediaRouter.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.v7.MediaRouter.dll" } - } - }; - public static Package SupportConstraintLayout_1_0_2_2 = new Package { - Id = "Xamarin.Android.Support.Constraint.Layout", - Version = "1.0.2.2", - TargetFramework = "MonoAndroid70", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Constraint.Layout") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Constraint.Layout.1.0.2.2\\lib\\MonoAndroid70\\Xamarin.Android.Support.Constraint.Layout.dll" - } - } - }; - public static Package VectorDrawable_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Vector.Drawable", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Vector.Drawable") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Vector.Drawable.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Vector.Drawable.dll" } - }, - }; - public static Package SupportDesign_27_0_2_1 = new Package { - Id = "Xamarin.Android.Support.Design", - Version = "27.0.2.1", - TargetFramework = "MonoAndroid81", - References = { - new BuildItem.Reference ("Xamarin.Android.Support.Design") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.Design.27.0.2.1\\lib\\MonoAndroid81\\Xamarin.Android.Support.Design.dll" } - } - }; - public static Package XamarinFormsPCL_2_3_4_231 = new Package { - Id = "Xamarin.Forms", - Version = "2.3.4.231", - TargetFramework = "portable-net45+win+wp80+MonoAndroid10+xamarinios10+MonoTouch10", - References = { - new BuildItem.Reference ("Xamarin.Forms.Core") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\portable-win+net45+wp80+win81+wpa81+MonoAndroid10+MonoTouch10+Xamarin.iOS10\\Xamarin.Forms.Core.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Xaml") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\portable-win+net45+wp80+win81+wpa81+MonoAndroid10+MonoTouch10+Xamarin.iOS10\\Xamarin.Forms.Xaml.dll" - }, - } - }; - public static Package XamarinForms_2_3_4_231 = new Package { - Id = "Xamarin.Forms", - Version = "2.3.4.231", - TargetFramework = "MonoAndroid44", - References = { - new BuildItem.Reference ("Xamarin.Forms.Platform.Android") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\MonoAndroid10\\Xamarin.Forms.Platform.Android.dll" - }, - new BuildItem.Reference ("FormsViewGroup") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\MonoAndroid10\\FormsViewGroup.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Core") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\MonoAndroid10\\Xamarin.Forms.Core.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Xaml") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\MonoAndroid10\\Xamarin.Forms.Xaml.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Platform") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.2.3.4.231\\lib\\MonoAndroid10\\Xamarin.Forms.Platform.dll" - }, - } - }; - public static Package XamarinForms_4_0_0_425677 = new Package { - Id = "Xamarin.Forms", - Version = "4.0.0.425677", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.Forms.Platform.Android") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.0.0.425677\\lib\\MonoAndroid90\\Xamarin.Forms.Platform.Android.dll" - }, - new BuildItem.Reference ("FormsViewGroup") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.0.0.425677\\lib\\MonoAndroid90\\FormsViewGroup.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Core") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.0.0.425677\\lib\\MonoAndroid90\\Xamarin.Forms.Core.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Xaml") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.0.0.425677\\lib\\MonoAndroid90\\Xamarin.Forms.Xaml.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Platform") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.0.0.425677\\lib\\MonoAndroid90\\Xamarin.Forms.Platform.dll" - }, - } - }; - public static Package XamarinForms_4_7_0_1142 = new Package { - Id = "Xamarin.Forms", - Version = "4.7.0.1142", - TargetFramework = "MonoAndroid10.0", + public static Package XamarinAndroidXWear = new Package { + Id = "Xamarin.AndroidX.Wear", + Version = "1.2.0.5" }; - public static Package XamarinForms_5_0_0_2515 = new Package { + public static Package XamarinForms = new Package { Id = "Xamarin.Forms", - Version = "5.0.0.2515", - TargetFramework = "MonoAndroid10.0", - }; - public static Package XamarinFormsMaps_4_7_0_1142 = new Package { - Id = "Xamarin.Forms.Maps", - Version = "4.7.0.1142", - TargetFramework = "MonoAndroid10.0", + Version = "5.0.0.2622", }; - public static Package XamarinFormsMaps_5_0_0_2515 = new Package { + public static Package XamarinFormsMaps = new Package { Id = "Xamarin.Forms.Maps", - Version = "5.0.0.2515", - TargetFramework = "MonoAndroid10.0", + Version = "5.0.0.2622", }; - public static Package XamarinFormsMaps_4_0_0_425677 = new Package { - Id = "Xamarin.Forms.Maps", - Version = "4.0.0.425677", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.Forms.Maps.Android") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.Maps.4.0.0.425677\\lib\\MonoAndroid90\\Xamarin.Forms.Maps.Android.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Maps") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.Maps.4.0.0.425677\\lib\\MonoAndroid90\\Xamarin.Forms.Maps.dll" - }, - } - }; - public static Package XamarinForms_4_4_0_991265 = new Package { - Id = "Xamarin.Forms", - Version = "4.4.0.991265", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.Forms.Platform.Android") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.4.0.991265\\lib\\MonoAndroid90\\Xamarin.Forms.Platform.Android.dll" - }, - new BuildItem.Reference ("FormsViewGroup") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.4.0.991265\\lib\\MonoAndroid90\\FormsViewGroup.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Core") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.4.0.991265\\lib\\MonoAndroid90\\Xamarin.Forms.Core.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Xaml") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.4.0.991265\\lib\\MonoAndroid90\\Xamarin.Forms.Xaml.dll" - }, - new BuildItem.Reference ("Xamarin.Forms.Platform") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Forms.4.4.0.991265\\lib\\MonoAndroid90\\Xamarin.Forms.Platform.dll" - }, - } - }; - public static Package AndroidXMigration = new Package { - Id = "Xamarin.AndroidX.Migration", - Version = "1.0.6.1", - TargetFramework = "MonoAndroid10", + public static Package AndroidXConstraintLayout = new Package { + Id = "Xamarin.AndroidX.ConstraintLayout", + Version = "2.1.4.9", }; public static Package AndroidXAppCompat = new Package { Id = "Xamarin.AndroidX.AppCompat", - Version = "1.1.0.1", - TargetFramework = "MonoAndroid10", - }; - public static Package AndroidXAppCompat_1_6_0_1 = new Package { - Id = "Xamarin.AndroidX.AppCompat", - Version = "1.6.0.1", - TargetFramework = "MonoAndroid10", + Version = "1.6.1.6", }; public static Package AndroidXBrowser = new Package { Id = "Xamarin.AndroidX.Browser", - Version = "1.2.0.1", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.AndroidX.Browser") { - MetadataValues = "HintPath=..\\packages\\Xamarin.AndroidX.Browser.1.0.0\\lib\\MonoAndroid90\\Xamarin.AndroidX.Browser.dll" - }, - } - }; - public static Package AndroidXMediaRouter = new Package { - Id = "Xamarin.AndroidX.MediaRouter", - Version = "1.1.0.1", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.AndroidX.MediaRouter") { - MetadataValues = "HintPath=..\\packages\\Xamarin.AndroidX.MediaRouter.1.1.0\\lib\\MonoAndroid90\\Xamarin.AndroidX.MediaRouter.dll" - }, - } + Version = "1.5.0.3", }; public static Package AndroidXLegacySupportV4 = new Package { Id = "Xamarin.AndroidX.Legacy.Support.V4", - Version = "1.0.0.1", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.AndroidX.Legacy.Support.V4") { - MetadataValues = "HintPath=..\\packages\\Xamarin.AndroidX.Legacy.Support.V4.1.0.0\\lib\\MonoAndroid90\\Xamarin.AndroidX.Legacy.Support.V4.dll" - }, - } - }; - public static Package AndroidXLifecycleLiveData = new Package { - Id = "Xamarin.AndroidX.Lifecycle.LiveData", - Version = "2.2.0.1", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.AndroidX.Lifecycle.LiveData") { - MetadataValues = "HintPath=..\\packages\\Xamarin.AndroidX.Lifecycle.LiveData.2.1.0\\lib\\MonoAndroid90\\Xamarin.AndroidX.Lifecycle.LiveData.dll" - }, - } + Version = "1.0.0.22", }; public static Package AndroidXAppCompatResources = new Package { Id = "Xamarin.AndroidX.AppCompat.AppCompatResources", - Version = "1.1.0.1", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.AndroidX.AppCompat.AppCompatResources") { - MetadataValues = "HintPath=..\\packages\\Xamarin.AndroidX.AppCompat.AppCompatResources.1.1.0.1\\lib\\MonoAndroid90\\Xamarin.AndroidX.AppCompat.AppCompatResources.dll" - }, - } + Version = "1.6.1.7", + }; + public static Package AndroidXLifecycleLiveData = new Package { + Id = "Xamarin.AndroidX.Lifecycle.LiveData", + Version = "2.6.2.3", }; public static Package AndroidXWorkRuntime = new Package { Id = "Xamarin.AndroidX.Work.Runtime", - Version = "2.3.4.3", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference("Xamarin.AndroidX.Work.Runtime") { - MetadataValues = "HintPath=..\\packages\\Xamarin.AndroidX.Work.Runtime.2.3.4.3\\lib\\MonoAndroid90\\Xamarin.AndroidX.Work.Runtime.dll" - } - } + Version = "2.9.0", }; public static Package XamarinGoogleAndroidMaterial = new Package { Id = "Xamarin.Google.Android.Material", - Version = "1.0.0.1", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("Xamarin.Google.Android.Material") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Google.Android.Material.1.0.0\\lib\\MonoAndroid90\\Xamarin.Google.Android.Material.dll" - }, - } + Version = "1.10.0.2", }; public static Package CocosSharp_PCL_Shared_1_5_0_0 = new Package { Id = "CocosSharp.PCL.Shared", @@ -364,16 +74,6 @@ public static class KnownPackages }, } }; - public static Package Xamarin_Android_Support_v8_RenderScript_28_0_0_3 = new Package { - Id = "Xamarin.Android.Support.v8.RenderScript", - Version = "28.0.0.3", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference ("MonoGame.Framework") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Support.v8.RenderScript.28.0.0.3\\lib\\MonoAndroid90\\Xamarin.Android.Support.v8.RenderScript.dll" - }, - } - }; public static Package FSharp_Core_Latest = new Package { Id = "FSharp.Core", Version = "4.7.1", @@ -425,79 +125,9 @@ public static class KnownPackages } } }; - public static Package Android_Arch_Core_Common_26_1_0 = new Package { - Id = "Xamarin.Android.Arch.Core.Common", - Version = "26.1.0", - TargetFramework = "MonoAndroid80", - References = { - new BuildItem.Reference("Xamarin.Android.Arch.Core.Common") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Arch.Core.Common.26.1.0\\lib\\MonoAndroid80\\Xamarin.Android.Arch.Core.Common.dll" - } - } - }; - public static Package Android_Arch_Lifecycle_Common_26_1_0 = new Package { - Id = "Xamarin.Android.Arch.Lifecycle.Common", - Version = "26.1.0", - TargetFramework = "MonoAndroid80", - References = { - new BuildItem.Reference("Xamarin.Android.Arch.Lifecycle.Common") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Arch.Lifecycle.Common.26.1.0\\lib\\MonoAndroid80\\Xamarin.Android.Arch.Lifecycle.Common.dll" - } - } - }; - public static Package Android_Arch_Lifecycle_Runtime_26_1_0 = new Package { - Id = "Xamarin.Android.Arch.Lifecycle.Runtime", - Version = "26.1.0", - TargetFramework = "MonoAndroid80", - References = { - new BuildItem.Reference("Xamarin.Android.Arch.Lifecycle.Runtime") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Arch.Lifecycle.Runtime.26.1.0\\lib\\MonoAndroid80\\Xamarin.Android.Arch.Lifecycle.Runtime.dll" - } - } - }; - public static Package Android_Arch_Work_Runtime = new Package { - Id = "Xamarin.Android.Arch.Work.Runtime", - Version = "1.0.0", - TargetFramework = "MonoAndroid90", - References = { - new BuildItem.Reference("Xamarin.Android.Arch.Work.Runtime") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Arch.Work.Runtime.1.0.0\\lib\\MonoAndroid90\\Xamarin.Android.Arch.Work.Runtime.dll" - } - } - }; - public static Package Xamarin_Android_Crashlytics = new Package { - Id = "Xamarin.Android.Crashlytics", - Version = "2.9.4.4", - TargetFramework = "MonoAndroid60", - References = { - new BuildItem.Reference("Xamarin.Android.Crashlytics") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Crashlytics.2.9.4.4\\lib\\MonoAndroid60\\Xamarin.Android.Crashlytics.dll" - } - } - }; - public static Package Xamarin_Android_Fabric = new Package { - Id = "Xamarin.Android.Fabric", - Version = "1.4.3.4", - TargetFramework = "MonoAndroid60", - References = { - new BuildItem.Reference("Xamarin.Android.Fabric") { - MetadataValues = "HintPath=..\\packages\\Xamarin.Android.Fabric.1.4.3.4\\lib\\MonoAndroid60\\Xamarin.Android.Fabric.dll" - } - } - }; public static Package Xamarin_Build_Download = new Package { Id = "Xamarin.Build.Download", - Version = "0.11.2", - }; - - public static Package Xamarin_Build_Download_0_11_3 = new Package { - Id = "Xamarin.Build.Download", - Version = "0.11.3", - }; - // NOTE: old version required for some tests - public static Package Xamarin_Build_Download_0_4_11 = new Package { - Id = "Xamarin.Build.Download", - Version = "0.4.11", + Version = "0.11.4", }; public static Package NuGet_Build_Packaging = new Package { Id = "NuGet.Build.Packaging", @@ -571,7 +201,7 @@ public static class KnownPackages }; public static Package ZXing_Net_Mobile = new Package { Id = "ZXing.Net.Mobile", - Version = "2.4.1", + Version = "3.0.0-beta5", // version with AndroidX TargetFramework = "MonoAndroid10", }; public static Package Xamarin_Legacy_OpenTK = new Package { @@ -584,6 +214,10 @@ public static class KnownPackages Version = "0.0.1-alpha", TargetFramework = "MonoAndroid10", }; + public static Package Xamarin_Jetbrains_Annotations = new Package { + Id = "Xamarin.Jetbrains.Annotations", + Version = "24.1.0.1", + }; public static Package Mono_AotProfiler_Android = new Package { Id = "Mono.AotProfiler.Android", Version = "7.0.0-preview1", @@ -602,4 +236,3 @@ public static class KnownPackages }; } } - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinAndroidWearApplicationProject.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinAndroidWearApplicationProject.cs index 93bf7c01a4f..7b32bcc983e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinAndroidWearApplicationProject.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinAndroidWearApplicationProject.cs @@ -3,13 +3,18 @@ using System.IO; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Linq; namespace Xamarin.ProjectTools { + /// + /// Migrated from Android.Support to AndroidX + /// see: https://android-developers.googleblog.com/2016/04/build-beautifully-for-android-wear.html + /// public class XamarinAndroidWearApplicationProject : XamarinAndroidApplicationProject { static readonly string default_strings_xml, default_main_activity; - static readonly string default_layout_main, default_layout_rect_main, default_layout_round_main; + static readonly string default_layout_rect_main, default_layout_round_main; static XamarinAndroidWearApplicationProject () { @@ -17,8 +22,6 @@ static XamarinAndroidWearApplicationProject () default_main_activity = sr.ReadToEnd (); using (var sr = new StreamReader (typeof(XamarinAndroidApplicationProject).Assembly.GetManifestResourceStream ("Xamarin.ProjectTools.Resources.Wear.Strings.xml"))) default_strings_xml = sr.ReadToEnd (); - using (var sr = new StreamReader (typeof(XamarinAndroidApplicationProject).Assembly.GetManifestResourceStream ("Xamarin.ProjectTools.Resources.Wear.LayoutMain.axml"))) - default_layout_main = sr.ReadToEnd (); using (var sr = new StreamReader (typeof(XamarinAndroidApplicationProject).Assembly.GetManifestResourceStream ("Xamarin.ProjectTools.Resources.Wear.LayoutRectMain.axml"))) default_layout_rect_main = sr.ReadToEnd (); using (var sr = new StreamReader (typeof(XamarinAndroidApplicationProject).Assembly.GetManifestResourceStream ("Xamarin.ProjectTools.Resources.Wear.LayoutRoundMain.axml"))) @@ -28,16 +31,23 @@ static XamarinAndroidWearApplicationProject () public XamarinAndroidWearApplicationProject (string debugConfigurationName = "Debug", string releaseConfigurationName = "Release", [CallerMemberName] string packageName = "") : base (debugConfigurationName, releaseConfigurationName, packageName) { - PackageReferences.Add (KnownPackages.AndroidWear_2_2_0); + PackageReferences.Add (KnownPackages.XamarinAndroidXWear); + + // uses-sdk:minSdkVersion 21 cannot be smaller than version 23 declared in library androidx.wear.wear.aar as the library might be using APIs not available in 21 + SupportedOSPlatformVersion = "23"; MainActivity = default_main_activity; StringsXml = default_strings_xml; - LayoutMain = default_layout_main; LayoutRectMain = default_layout_rect_main; LayoutRoundMain = default_layout_round_main; - AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\layout\\RectangleMain.axml") { TextContent = () => LayoutRectMain }); - AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\layout\\RoundMain.axml") { TextContent = () => LayoutRoundMain }); + // Remove Resources\layout\Main.axml + var main = AndroidResources.FirstOrDefault (a => a.Include () == "Resources\\layout\\Main.axml"); + if (main != null) + AndroidResources.Remove (main); + + AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\layout-notround\\activity_main.axml") { TextContent = () => LayoutRectMain }); + AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\layout-round\\activity_main.axml") { TextContent = () => LayoutRoundMain }); } public string LayoutRectMain { get; set; } diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs index 2ed381d143c..66a1b08f19f 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsAndroidApplicationProject.cs @@ -43,7 +43,11 @@ public XamarinFormsAndroidApplicationProject (string debugConfigurationName = "D { // Don't opt into ImplicitUsings RemoveProperty (KnownProperties.ImplicitUsings); - PackageReferences.Add (KnownPackages.XamarinFormsMaps_5_0_0_2515); + PackageReferences.Add (KnownPackages.XamarinForms); + + // Workarounds for Guava.ListenableFuture + // See: https://github.com/xamarin/AndroidX/issues/535 + PackageReferences.Add (KnownPackages.AndroidXBrowser); AndroidResources.Add (new AndroidItem.AndroidResource ("Resources\\values\\colors.xml") { TextContent = () => colors_xml, diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs index cb325b09946..b068e6decc2 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Android/XamarinFormsMapsApplicationProject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Runtime.CompilerServices; @@ -17,7 +17,7 @@ static XamarinFormsMapsApplicationProject () public XamarinFormsMapsApplicationProject ([CallerMemberName] string packageName = "") : base (packageName: packageName) { - PackageReferences.Add (KnownPackages.XamarinFormsMaps_5_0_0_2515); + PackageReferences.Add (KnownPackages.XamarinFormsMaps); PackageReferences.Add (KnownPackages.Xamarin_GooglePlayServices_Base); PackageReferences.Add (KnownPackages.Xamarin_GooglePlayServices_Basement); PackageReferences.Add (KnownPackages.Xamarin_GooglePlayServices_Maps); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/LayoutMain.axml b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/LayoutMain.axml deleted file mode 100644 index 72954bdb6ee..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/LayoutMain.axml +++ /dev/null @@ -1,11 +0,0 @@ - - diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/MainActivity.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/MainActivity.cs index 09aa236dc7e..1b0017e1b3b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/MainActivity.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.ProjectTools/Resources/Wear/MainActivity.cs @@ -1,13 +1,12 @@ -using System; +using System; using Android.App; using Android.Content; using Android.OS; using Android.Runtime; -using Android.Support.V4.App; -using Android.Support.Wearable.Views; using Android.Views; using Android.Widget; +using AndroidX.Core.App; //${USINGS} namespace ${ROOT_NAMESPACE} @@ -22,26 +21,22 @@ protected override void OnCreate (Bundle bundle) base.OnCreate (bundle); // Set our view from the "main" layout resource - SetContentView (Resource.Layout.Main); - - var v = FindViewById (Resource.Id.watch_view_stub); - v.LayoutInflated += delegate { - - // Get our button from the layout resource, - // and attach an event to it - Button button = FindViewById