Skip to content

Commit

Permalink
[tmt] Update to work with current libxamarin-app.so (#8694)
Browse files Browse the repository at this point in the history
Context: 46f10fe
Context? 6836818

`tools/tmt` (46f10fe) is a utility to print typemap entries contained
within an application.

`tools/tmt` no longer supports dumping typemap entries; it was
possibly broken in 6836818:

	% ./dotnet-local.sh build tools/tmt/*.csproj
	% ./dotnet-local.sh build -c Release samples/HelloWorld/HelloWorld/HelloWorld.DotNet.csproj
	% bin/Debug/bin/tmt samples/HelloWorld/HelloWorld/bin/Release/net9.0-android/*-Signed.apk
	No type maps found in 'samples/HelloWorld/HelloWorld/bin/Release/net9.0-android/com.xamarin.android.helloworld-Signed.apk'

Update the `tools/tmt` utility to support the current format of
typemaps within `libxamarin-app.so`, and update it to generate nicely
formatted Markdown report files instead of the text file output:

	% bin/Debug/bin/tmt samples/HelloWorld/HelloWorld/bin/Release/net9.0-android/*-Signed.apk
	samples/HelloWorld/HelloWorld/bin/Release/net9.0-android/com.xamarin.android.helloworld-Signed.apk!lib/arm64-v8a/libxamarin-app.so:
	  File Type: Xamarin App Release DSO
	  Format version: 2
	  Map kind: Release
	  Map architecture: ARM64
	  Managed to Java entries: 56
	  Java to Managed entries: 46 (without duplicates)
	…

`typemap-v2-Release-ARM64.md` will be created (among other files)
which contains the actual typemap data, in Markdown tabular form:

> # Java to Managed

> | Java type name                                     | Managed type name                                                | Type token ID         | Is Generic?      | MVID                                 |
> | -------------------------------------------------- | ---------------------------------------------------------------- | --------------------- | ---------------- | ------------------------------------ |
> | android/app/Activity                               | Android.App.Activity, Mono.Android                               | 0x02000042 (33554498) | no               | 33da2efb-61bb-4fd5-b529-2dee309a3d65 |
> …
> | java/lang/Object                                   | Java.Lang.Object, Mono.Android                                   | 0x0200008B (33554571) | no               | 33da2efb-61bb-4fd5-b529-2dee309a3d65 |
> …

> # Managed to Java

> | Managed type name                                                       | Java type name                                     | Type token ID         | Is Generic? | Is Duplicate? | MVID                                 |
> | ----------------------------------------------------------------------- | -------------------------------------------------- | --------------------- | ----------- | ------------- | ------------------------------------ |
> | HelloLibrary.LibraryActivity, HelloLibrary.DotNet                       | mono/samples/hello/LibraryActivity                 | 0x02000002 (33554434) | no          | false         | ca140934-068f-47d0-a861-6179233e49aa |
> …
  • Loading branch information
grendello authored Feb 14, 2024
1 parent 5472eec commit 71b6fcc
Show file tree
Hide file tree
Showing 25 changed files with 2,470 additions and 722 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ sealed class XamarinAndroidBundledAssembly
}

// Keep in sync with FORMAT_TAG in src/monodroid/jni/xamarin-app.hh
const ulong FORMAT_TAG = 0x015E6972616D58;
const ulong FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version

SortedDictionary <string, string>? environmentVariables;
SortedDictionary <string, string>? systemProperties;
Expand Down
31 changes: 31 additions & 0 deletions src/Xamarin.Android.Build.Tasks/Utilities/TypeMapHelper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
using System;
using System.IO.Hashing;
using System.Text;

namespace Xamarin.Android.Tasks;

static class TypeMapHelper
{
/// <summary>
/// Hash the given Java type name for use in java-to-managed typemap array.
/// </summary>
public static ulong HashJavaName (string name, bool is64Bit)
{
if (name.Length == 0) {
return UInt64.MaxValue;
}

// Native code (EmbeddedAssemblies::typemap_java_to_managed in embedded-assemblies.cc) will operate on wchar_t cast to a byte array, we need to do
// the same
return HashBytes (Encoding.Unicode.GetBytes (name), is64Bit);
}

static ulong HashBytes (byte[] bytes, bool is64Bit)
{
if (is64Bit) {
return XxHash64.HashToUInt64 (bytes);
}

return (ulong)XxHash32.HashToUInt32 (bytes);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -403,32 +403,12 @@ void HashJavaNames (ConstructionState cs)
TypeMapJava entry = cs.JavaMap[i].Instance;

// The cast is safe, xxHash will return a 32-bit value which (for convenience) was upcast to 64-bit
entry.JavaNameHash32 = (uint)HashName (entry.JavaName, is64Bit: false);
entry.JavaNameHash32 = (uint)TypeMapHelper.HashJavaName (entry.JavaName, is64Bit: false);
hashes32.Add (entry.JavaNameHash32);

entry.JavaNameHash64 = HashName (entry.JavaName, is64Bit: true);
entry.JavaNameHash64 = TypeMapHelper.HashJavaName (entry.JavaName, is64Bit: true);
hashes64.Add (entry.JavaNameHash64);
}

ulong HashName (string name, bool is64Bit)
{
if (name.Length == 0) {
return UInt64.MaxValue;
}

// Native code (EmbeddedAssemblies::typemap_java_to_managed in embedded-assemblies.cc) will operate on wchar_t cast to a byte array, we need to do
// the same
return HashBytes (Encoding.Unicode.GetBytes (name), is64Bit);
}

ulong HashBytes (byte[] bytes, bool is64Bit)
{
if (is64Bit) {
return XxHash64.HashToUInt64 (bytes);
}

return (ulong)XxHash32.HashToUInt32 (bytes);
}
}
}
}
2 changes: 1 addition & 1 deletion src/monodroid/jni/xamarin-app.hh
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
#include "monodroid.h"
#include "xxhash.hh"

static constexpr uint64_t FORMAT_TAG = 0x015E6972616D58;
static constexpr uint64_t FORMAT_TAG = 0x00025E6972616D58; // 'Xmari^XY' where XY is the format version
static constexpr uint32_t COMPRESSED_DATA_MAGIC = 0x5A4C4158; // 'XALZ', little-endian
static constexpr uint32_t ASSEMBLY_STORE_MAGIC = 0x41424158; // 'XABA', little-endian
static constexpr uint32_t ASSEMBLY_STORE_FORMAT_VERSION = 1; // Increase whenever an incompatible change is made to the
Expand Down
11 changes: 11 additions & 0 deletions tools/assembly-store-reader/AssemblyStoreExplorer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,17 @@ public AssemblyStoreExplorer (string storePath, Action<AssemblyStoreExplorerLogL
ProcessStores ();
}

public AssemblyStoreExplorer (ZipArchive archive, string basePathInArchive, Action<AssemblyStoreExplorerLogLevel, string>? customLogger = null, bool keepStoreInMemory = false)
{
logger = customLogger;
this.keepStoreInMemory = keepStoreInMemory;
StorePath = "<in-memory-archive>";
StoreSetName = StorePath;
ReadStoreSetFromArchive (archive, basePathInArchive);

ProcessStores ();
}

void Logger (AssemblyStoreExplorerLogLevel level, string message)
{
if (level == AssemblyStoreExplorerLogLevel.Error) {
Expand Down
50 changes: 40 additions & 10 deletions tools/tmt/AnELF.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,28 +74,35 @@ public bool HasSymbol (string symbolName)
return GetSymbol (symbolName) != null;
}

public byte[] GetData (string symbolName)
public (byte[] data, ISymbolEntry? symbol) GetData (string symbolName)
{
Log.Debug ($"Looking for symbol: {symbolName}");
ISymbolEntry? symbol = GetSymbol (symbolName);
if (symbol == null)
return EmptyArray;
if (symbol == null) {
return (EmptyArray, null);
}

if (Is64Bit) {
var symbol64 = symbol as SymbolEntry<ulong>;
if (symbol64 == null)
throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 64-bit symbol");
return GetData (symbol64);
return (GetData (symbol64), symbol);
}

var symbol32 = symbol as SymbolEntry<uint>;
if (symbol32 == null)
throw new InvalidOperationException ($"Symbol '{symbolName}' is not a valid 32-bit symbol");

return GetData (symbol32);
return (GetData (symbol32), symbol);
}

public abstract byte[] GetData (ulong symbolValue, ulong size);
public abstract byte[] GetDataFromPointer (ulong pointerValue, ulong size);

public string GetASCIIZFromPointer (ulong pointerValue)
{
return GetASCIIZ (GetDataFromPointer (pointerValue, 0), 0);
}

public string GetASCIIZ (ulong symbolValue)
{
Expand Down Expand Up @@ -134,14 +141,27 @@ protected byte[] GetData (ISymbolEntry symbol, ulong size, ulong offset)
return GetData (symbol.PointedSection, size, offset);
}

void LogSectionInfo<T> (Section<T> section) where T: struct
{
Log.Debug ($" section offset in file: 0x{section.Offset:x}; section size: {section.Size}; alignment: {section.Alignment}");
}

protected byte[] GetData (ISection section, ulong size, ulong offset)
{
Log.Debug ($"AnELF.GetData: section == {section.Name}; size == {size}; offset == {offset:X}");
Log.Debug ($"AnELF.GetData: section == '{section.Name}'; requested data size == {size}; offset in section == 0x{offset:x}");
byte[] data = section.GetContents ();

Log.Debug ($" data length: {data.Length} (long: {data.LongLength})");
Log.Debug ($" offset: {offset}; size: {size}");
if (section is Section<ulong> sec64) {
LogSectionInfo (sec64);
} else if (section is Section<uint> sec32) {
LogSectionInfo (sec32);
} else {
throw new NotSupportedException ($"Are we in the 128-bit future yet? Unsupported section type {section.GetType ()}");
}

Log.Debug ($" section data length: {data.Length} (long: {data.LongLength})");
if ((ulong)data.LongLength < (offset + size)) {
Log.Debug ($" not enough data in section");
return EmptyArray;
}

Expand All @@ -156,9 +176,18 @@ protected byte[] GetData (ISection section, ulong size, ulong offset)
return ret;
}

/// <summary>
/// Find a relocation corresponding to a pointer at offset <paramref name="pointerOffset"/> into
/// the specified <paramref name="symbol"/>. Returns an `ulong`, which needs to be cast to `uint`
/// for 32-pointers (it can be done safely as the upper 32-bits will be 0 in such cases)
/// </summary>
public abstract ulong DeterminePointerAddress (ISymbolEntry symbol, ulong pointerOffset);
public abstract ulong DeterminePointerAddress (ulong symbolValue, ulong pointerOffset);

public uint GetUInt32 (string symbolName)
{
return GetUInt32 (GetData (symbolName), 0, symbolName);
(byte[] data, _) = GetData (symbolName);
return GetUInt32 (data, 0, symbolName);
}

public uint GetUInt32 (ulong symbolValue)
Expand All @@ -177,7 +206,8 @@ protected uint GetUInt32 (byte[] data, ulong offset, string symbolName)

public ulong GetUInt64 (string symbolName)
{
return GetUInt64 (GetData (symbolName), 0, symbolName);
(byte[] data, _) = GetData (symbolName);
return GetUInt64 (data, 0, symbolName);
}

public ulong GetUInt64 (ulong symbolValue)
Expand Down
137 changes: 105 additions & 32 deletions tools/tmt/ApkManagedTypeResolver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

using K4os.Compression.LZ4;
using Mono.Cecil;
using Xamarin.Android.AssemblyStore;
using Xamarin.Tools.Zip;

namespace tmt
Expand All @@ -12,15 +13,44 @@ class ApkManagedTypeResolver : ManagedTypeResolver
{
const uint CompressedDataMagic = 0x5A4C4158; // 'XALZ', little-endian

Dictionary<string, ZipEntry> assemblies;
ZipArchive apk;
readonly Dictionary<string, ZipEntry>? individualAssemblies;
readonly Dictionary<string, AssemblyStoreAssembly>? blobAssemblies;
readonly ZipArchive apk;
readonly AssemblyStoreExplorer? assemblyStoreExplorer;

public ApkManagedTypeResolver (ZipArchive apk, string assemblyEntryPrefix)
{
this.apk = apk;
assemblies = new Dictionary<string, ZipEntry> (StringComparer.Ordinal);

foreach (ZipEntry entry in apk) {
if (apk.ContainsEntry ($"{assemblyEntryPrefix}assemblies.blob")) {
blobAssemblies = new Dictionary<string, AssemblyStoreAssembly> (StringComparer.Ordinal);
assemblyStoreExplorer = new AssemblyStoreExplorer (apk, assemblyEntryPrefix, keepStoreInMemory: true);
LoadAssemblyBlobs (apk, assemblyEntryPrefix, assemblyStoreExplorer);
} else {
individualAssemblies = new Dictionary<string, ZipEntry> (StringComparer.Ordinal);
LoadIndividualAssemblies (apk, assemblyEntryPrefix);
}
}

void LoadAssemblyBlobs (ZipArchive apkArchive, string assemblyEntryPrefix, AssemblyStoreExplorer explorer)
{
foreach (AssemblyStoreAssembly assembly in explorer.Assemblies) {
string assemblyName = assembly.Name;
string dllName = assembly.DllName;

if (!String.IsNullOrEmpty (assembly.Store.Arch)) {
assemblyName = $"{assembly.Store.Arch}/{assemblyName}";
dllName = $"{assembly.Store.Arch}/{dllName}";
}

blobAssemblies!.Add (assemblyName, assembly);
blobAssemblies!.Add (dllName, assembly);
}
}

void LoadIndividualAssemblies (ZipArchive apkArchive, string assemblyEntryPrefix)
{
foreach (ZipEntry entry in apkArchive) {
if (!entry.FullName.StartsWith (assemblyEntryPrefix, StringComparison.Ordinal)) {
continue;
}
Expand All @@ -29,61 +59,104 @@ public ApkManagedTypeResolver (ZipArchive apk, string assemblyEntryPrefix)
continue;
}

assemblies.Add (Path.GetFileNameWithoutExtension (entry.FullName), entry);
assemblies.Add (entry.FullName, entry);
string relativeName = entry.FullName.Substring (assemblyEntryPrefix.Length);
string? dir = Path.GetDirectoryName (relativeName);
string name = Path.GetFileNameWithoutExtension (relativeName);
if (!String.IsNullOrEmpty (dir)) {
name = $"{dir}/{name}";
}

individualAssemblies!.Add (name, entry);
individualAssemblies.Add (entry.FullName, entry);
}
}

protected override string? FindAssembly (string assemblyName)
{
if (assemblies.Count == 0) {
return null;
if (individualAssemblies != null) {
if (individualAssemblies.Count == 0) {
return null;
}

if (!individualAssemblies.TryGetValue (assemblyName, out ZipEntry? entry) || entry == null) {
return null;
}

return entry.FullName;
}

if (!assemblies.TryGetValue (assemblyName, out ZipEntry? entry) || entry == null) {
if (blobAssemblies == null || !blobAssemblies.TryGetValue (assemblyName, out AssemblyStoreAssembly? assembly) || assembly == null) {
return null;
}

return entry.FullName;
return assembly.Name;
}

protected override AssemblyDefinition ReadAssembly (string assemblyPath)
Stream GetAssemblyStream (string assemblyPath)
{
if (!assemblies.TryGetValue (assemblyPath, out ZipEntry? entry) || entry == null) {
MemoryStream? stream = null;
if (individualAssemblies != null) {
if (!individualAssemblies.TryGetValue (assemblyPath, out ZipEntry? entry) || entry == null) {
// Should "never" happen - if the assembly wasn't there, FindAssembly should have returned `null`
throw new InvalidOperationException ($"Should not happen: assembly '{assemblyPath}' not found in the APK archive.");
}

stream = new MemoryStream ();
entry.Extract (stream);
return PrepStream (stream);
}

if (blobAssemblies == null) {
throw new InvalidOperationException ("Internal error: blobAssemblies shouldn't be null");
}

if (blobAssemblies == null || !blobAssemblies.TryGetValue (assemblyPath, out AssemblyStoreAssembly? assembly) || assembly == null) {
// Should "never" happen - if the assembly wasn't there, FindAssembly should have returned `null`
throw new InvalidOperationException ($"Should not happen: assembly {assemblyPath} not found in the APK archive.");
throw new InvalidOperationException ($"Should not happen: assembly '{assemblyPath}' not found in the assembly blob.");
}

stream = new MemoryStream ();
assembly.ExtractImage (stream);

return PrepStream (stream);

Stream PrepStream (Stream stream)
{
stream.Seek (0, SeekOrigin.Begin);
return stream;
}
}

protected override AssemblyDefinition ReadAssembly (string assemblyPath)
{
byte[]? assemblyBytes = null;
var stream = new MemoryStream ();
entry.Extract (stream);
stream.Seek (0, SeekOrigin.Begin);
Stream stream = GetAssemblyStream (assemblyPath);

//
// LZ4 compressed assembly header format:
// uint magic; // 0x5A4C4158; 'XALZ', little-endian
// uint descriptor_index; // Index into an internal assembly descriptor table
// uint uncompressed_length; // Size of assembly, uncompressed
//
using (var reader = new BinaryReader (stream)) {
uint magic = reader.ReadUInt32 ();
if (magic == CompressedDataMagic) {
reader.ReadUInt32 (); // descriptor index, ignore
uint decompressedLength = reader.ReadUInt32 ();

int inputLength = (int)(stream.Length - 12);
byte[] sourceBytes = Utilities.BytePool.Rent (inputLength);
reader.Read (sourceBytes, 0, inputLength);

assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength);
int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength);
if (decoded != (int)decompressedLength) {
throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})");
}
Utilities.BytePool.Return (sourceBytes);
using var reader = new BinaryReader (stream);
uint magic = reader.ReadUInt32 ();
if (magic == CompressedDataMagic) {
reader.ReadUInt32 (); // descriptor index, ignore
uint decompressedLength = reader.ReadUInt32 ();

int inputLength = (int)(stream.Length - 12);
byte[] sourceBytes = Utilities.BytePool.Rent (inputLength);
reader.Read (sourceBytes, 0, inputLength);

assemblyBytes = Utilities.BytePool.Rent ((int)decompressedLength);
int decoded = LZ4Codec.Decode (sourceBytes, 0, inputLength, assemblyBytes, 0, (int)decompressedLength);
if (decoded != (int)decompressedLength) {
throw new InvalidOperationException ($"Failed to decompress LZ4 data of {assemblyPath} (decoded: {decoded})");
}
Utilities.BytePool.Return (sourceBytes);
}


if (assemblyBytes != null) {
stream.Close ();
stream.Dispose ();
Expand Down
Loading

0 comments on commit 71b6fcc

Please sign in to comment.