Skip to content

Commit

Permalink
Improve MachO support to read LC_BUILD_VERSION and static libraries. (#…
Browse files Browse the repository at this point in the history 
…5685)

* MachO.cs: Support reading LC_BUILD_VERSION

Newer SDKs set this instead of LC_VERSION_MIN_*

* MachO.cs: Add support for reading Mach-O files inside ar archives.

* [tests] Augment ProductTests.MinOSVersion to test static libraries as well.

* Adjust enum field names to match our naming scheme.
  • Loading branch information
@rolfbjarne
rolfbjarne authored Mar 1, 2019
1 parent 507d37e commit 5905242
Show file tree
Hide file tree
Showing 3 changed files with 197 additions and 29 deletions.
10 changes: 10 additions & 0 deletions docs/website/mtouch-errors.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1585,6 +1585,16 @@ An error occurred while processing the MachO file in question.

Please make sure the file is a valid Mach-O dynamic library.

The format of a file can be verified using the `file` command from a terminal:

file -arch all -l /path/to/file

### MT1605: Invalid entry * in the static library *: *

An error occurred while processing the MachO file in question.

Please make sure the file is a valid Mach-O static library.

The format of a file can be verified using the `file` command from a terminal:

file -arch all -l /path/to/file
Expand Down
73 changes: 51 additions & 22 deletions tests/common/ProductTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,35 +38,54 @@ public void MonoVersion ()
}

[Test]
[TestCase (Profile.macOSSystem, MachO.LoadCommands.MinMacOSX)]
[TestCase (Profile.macOSFull, MachO.LoadCommands.MinMacOSX)]
[TestCase (Profile.macOSMobile, MachO.LoadCommands.MinMacOSX)]
[TestCase (Profile.iOS, MachO.LoadCommands.MiniPhoneOS, false)]
[TestCase (Profile.iOS, MachO.LoadCommands.MiniPhoneOS, true)]
[TestCase (Profile.watchOS, MachO.LoadCommands.MinwatchOS, false)]
[TestCase (Profile.watchOS, MachO.LoadCommands.MinwatchOS, true)]
[TestCase (Profile.tvOS, MachO.LoadCommands.MintvOS, false)]
[TestCase (Profile.tvOS, MachO.LoadCommands.MintvOS, true)]
public void MinOSVersion (Profile profile, MachO.LoadCommands load_command, bool device = false)
[TestCase (Profile.macOSSystem, MachO.LoadCommands.MinMacOSX, MachO.Platform.MacOS)]
[TestCase (Profile.macOSFull, MachO.LoadCommands.MinMacOSX, MachO.Platform.MacOS)]
[TestCase (Profile.macOSMobile, MachO.LoadCommands.MinMacOSX, MachO.Platform.MacOS)]
[TestCase (Profile.iOS, MachO.LoadCommands.MiniPhoneOS, MachO.Platform.IOSSimulator, false)]
[TestCase (Profile.iOS, MachO.LoadCommands.MiniPhoneOS, MachO.Platform.IOS, true)]
[TestCase (Profile.watchOS, MachO.LoadCommands.MinwatchOS, MachO.Platform.WatchOSSimulator, false)]
[TestCase (Profile.watchOS, MachO.LoadCommands.MinwatchOS, MachO.Platform.WatchOS, true)]
[TestCase (Profile.tvOS, MachO.LoadCommands.MintvOS, MachO.Platform.TvOSSimulator, false)]
[TestCase (Profile.tvOS, MachO.LoadCommands.MintvOS, MachO.Platform.TvOS, true)]
public void MinOSVersion (Profile profile, MachO.LoadCommands load_command, MachO.Platform platform, bool device = false)
{
if (device)
Configuration.AssertDeviceAvailable ();

var dylibs = Directory.GetFiles (Configuration.GetSdkPath (profile, device), "*.dylib", SearchOption.AllDirectories)
.Where ((v) => !v.Contains ("dylib.dSYM/Contents/Resources/DWARF")); // Don't include *.dylib from inside .dSYMs.
var machoFiles = Directory.GetFiles (Configuration.GetSdkPath (profile, device), "*", SearchOption.AllDirectories)
.Where ((v) => {
if (v.Contains ("dylib.dSYM/Contents/Resources/DWARF")) {
// Don't include *.dylib from inside .dSYMs.
return false;
} else if (v.Contains ("libxammac-classic") || v.Contains ("libxammac-system-classic")) {
// We don't care about XM Classic, those are binary dependencies.
return false;
}
var ext = Path.GetExtension (v);
return ext == ".a" || ext == ".dylib";
}
);

var failed = new List<string> ();
foreach (var dylib in dylibs) {
var fatfile = MachO.Read (dylib);
foreach (var machoFile in machoFiles) {
var fatfile = MachO.Read (machoFile);
foreach (var slice in fatfile) {
var any_load_command = false;
foreach (var lc in slice.load_commands) {
Version lc_min_version;
var mincmd = lc as MinCommand;
if (mincmd == null)
continue;
// Console.WriteLine ($" {mincmd.Command} version: {mincmd.version}=0x{mincmd.version.ToString ("x")}={mincmd.Version} sdk: {mincmd.sdk}=0x{mincmd.sdk.ToString ("x")}={mincmd.Sdk}");
if (mincmd != null){
Assert.AreEqual (load_command, mincmd.Command, "Unexpected min load command");
lc_min_version = mincmd.Version;
} else {
// starting from iOS SDK 12 the LC_BUILD_VERSION is used instead
var buildver = lc as BuildVersionCommand;
if (buildver == null)
continue;

Assert.AreEqual (load_command, mincmd.Command, "Unexpected min load command");
Assert.AreEqual (platform, buildver.Platform, "Unexpected build version command");
lc_min_version = buildver.MinOS;
}

Version version;
Version alternate_version = null;
Expand All @@ -76,10 +95,12 @@ public void MinOSVersion (Profile profile, MachO.LoadCommands load_command, bool
break;
case MachO.LoadCommands.MiniPhoneOS:
version = SdkVersions.MiniOSVersion;
if (device) {
if (slice.IsDynamicLibrary && device) {
if (version.Major < 7)
version = new Version (7, 0, 0); // dylibs are supported starting with iOS 7.
alternate_version = new Version (8, 0, 0); // some iOS dylibs also have min OS 8.0 (if they're used as frameworks as well).
} else if (slice.Architecture == MachO.Architectures.ARM64) {
alternate_version = new Version (7, 0, 0); // our arm64 slices has min iOS 7.0.
}
break;
case MachO.LoadCommands.MintvOS:
Expand All @@ -92,20 +113,28 @@ public void MinOSVersion (Profile profile, MachO.LoadCommands load_command, bool
throw new NotImplementedException (load_command.ToString ());
}

version = new Version (version.Major, version.Minor, version.Build < 0 ? 0 : version.Build);
version = version.WithBuild ();
if (alternate_version == null)
alternate_version = version;

if (version != mincmd.Version && alternate_version != mincmd.Version)
failed.Add ($"Unexpected minOS version (expected {version}, alternatively {alternate_version}, found {mincmd.Version}) in {dylib}.");
failed.Add ($"Unexpected minOS version (expected {version}, alternatively {alternate_version}, found {mincmd.Version}) in {machoFile} ({slice.Filename}).");
any_load_command = true;
}
if (!any_load_command)
failed.Add ($"No minOS version found in {dylib}.");
failed.Add ($"No minOS version found in {machoFile}.");
}
}
CollectionAssert.IsEmpty (failed, "Failures");
}
}

static class VersionExtensions
{
public static Version WithBuild (this Version version)
{
return new Version (version.Major, version.Minor, version.Build < 0 ? 0 : version.Build);
}
}
}

143 changes: 136 additions & 7 deletions tools/common/MachO.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Text;

#if MLAUNCH
using Xamarin.Launcher;
Expand Down Expand Up @@ -133,8 +134,18 @@ public enum LoadCommands : uint
MintvOS = 0x2f,//#define LC_VERSION_MIN_TVOS 0x2F /* build for AppleTV min OS version */
MinwatchOS = 0x30,//#define LC_VERSION_MIN_WATCHOS 0x30 /* build for Watch min OS version */
//#define LC_NOTE 0x31 /* arbitrary data included within a Mach-O file */
//#define LC_BUILD_VERSION 0x32 /* build for platform min OS version */
BuildVersion = 0x32,//#define LC_BUILD_VERSION 0x32 /* build for platform min OS version */
}

public enum Platform : uint {
MacOS = 1,
IOS = 2,
TvOS = 3,
WatchOS = 4,
BridgeOS = 5,
IOSSimulator = 7,
TvOSSimulator = 8,
WatchOSSimulator = 9,
}

internal static uint FromBigEndian (uint number)
Expand Down Expand Up @@ -189,7 +200,7 @@ static object ReadFile (BinaryReader reader, string filename)
default:
if (StaticLibrary.IsStaticLibrary (reader)) {
var sl = new StaticLibrary ();
sl.Read (reader);
sl.Read (filename, reader, reader.BaseStream.Length);
return sl;
}
throw new Exception (string.Format ("File format not recognized: {0} (magic: 0x{1})", filename, magic.ToString ("X")));
Expand All @@ -210,8 +221,13 @@ public static IEnumerable<MachOFile> Read (string filename)
var file = ReadFile (filename);
var fatfile = file as FatFile;
if (fatfile != null) {
foreach (var ff in fatfile.entries)
yield return ff.entry;
foreach (var ff in fatfile.entries) {
if (ff.entry != null)
yield return ff.entry;
if (ff.static_library != null)
foreach (var obj in ff.static_library.ObjectFiles)
yield return obj;
}
} else {
var mf = file as MachOFile;
if (mf != null)
Expand Down Expand Up @@ -456,9 +472,73 @@ public static bool IsDynamicFramework (string filename)

public class StaticLibrary
{
internal void Read (BinaryReader reader)
List<MachOFile> object_files = new List<MachOFile> ();

public IEnumerable<MachOFile> ObjectFiles { get { return object_files; } }

static string ReadString (BinaryReader reader, int length)
{
var bytes = reader.ReadBytes (length);
for (var i = 0; i < bytes.Length; i++) {
if (bytes [i] == 0) {
length = i;
break;
}
}
return Encoding.ASCII.GetString (bytes, 0, length);
}

static long ReadDecimal (BinaryReader reader, int length)
{
var str = ReadString (reader, length);
str = str.TrimEnd (' ');
return long.Parse (str);
}

static long ReadOctal (BinaryReader reader, int length)
{
var str = ReadString (reader, length);
str = str.TrimEnd (' ');
return Convert.ToInt64 (str, 8);
}

internal void Read (string filename, BinaryReader reader, long size)
{
IsStaticLibrary (reader, throw_if_error: true);

var pos = reader.BaseStream.Position;
reader.BaseStream.Position += 8; // header

byte [] bytes;
while (reader.BaseStream.Position < pos + size) {
var fileIdentifier = ReadString (reader, 16);
var fileModificationTimestamp = ReadDecimal (reader, 12);
var ownerId = ReadDecimal (reader, 6);
var groupId = ReadDecimal (reader, 6);
var fileMode = ReadOctal (reader, 8);
var fileSize = ReadDecimal (reader, 10);
bytes = reader.ReadBytes (2); // ending characters
if (bytes [0] != 0x60 && bytes [1] != 0x0A)
throw ErrorHelper.CreateError (1605, $"Invalid entry '{fileIdentifier}' in the static library '{filename}', entry header doesn't end with 0x60 0x0A (found '0x{bytes [0].ToString ("x")} 0x{bytes [1].ToString ("x")}')");

if (fileIdentifier.StartsWith ("#1/", StringComparison.Ordinal)) {
var nameLength = int.Parse (fileIdentifier.Substring (3).TrimEnd (' '));
fileIdentifier = ReadString (reader, nameLength);
fileSize -= nameLength;
}

var nextPosition = reader.BaseStream.Position + fileSize;
if (MachOFile.IsMachOLibrary (null, reader)) {
var file = new MachOFile (fileIdentifier);
file.Read (reader);
object_files.Add (file);
}
// byte position is always even after each file.
if (nextPosition % 1 == 1)
nextPosition++;
reader.BaseStream.Position = nextPosition;
}

}

public static bool IsStaticLibrary (BinaryReader reader, bool throw_if_error = false)
Expand All @@ -470,7 +550,7 @@ public static bool IsStaticLibrary (BinaryReader reader, bool throw_if_error = f
reader.BaseStream.Position = pos;

if (throw_if_error && !rv)
throw ErrorHelper.CreateError (1601, "Not a Mach-O dynamic library (unknown header '0x{0}'): {1}.", System.Text.Encoding.ASCII.GetString (bytes, 0, 7));
throw ErrorHelper.CreateError (1601, "Not a Mach-O static library (unknown header '{0}', expected '!<arch>').", System.Text.Encoding.ASCII.GetString (bytes, 0, 7));

return rv;
}
Expand Down Expand Up @@ -502,6 +582,8 @@ public class MachOFile

public List<LoadCommand> load_commands;

public string Filename { get { return filename; } }

public MachOFile (FatEntry parent)
{
fat_parent = parent;
Expand Down Expand Up @@ -657,6 +739,23 @@ internal void Read (BinaryReader reader)
minCmd.sdk = reader.ReadUInt32 ();
lc = minCmd;
break;
case MachO.LoadCommands.BuildVersion:
var buildVer = new BuildVersionCommand ();
buildVer.cmd = reader.ReadUInt32 ();
buildVer.cmdsize = reader.ReadUInt32 ();
buildVer.platform = reader.ReadUInt32 ();
buildVer.minos = reader.ReadUInt32 ();
buildVer.sdk = reader.ReadUInt32 ();
buildVer.ntools = reader.ReadUInt32 ();
buildVer.tools = new BuildVersionCommand.BuildToolVersion[buildVer.ntools];
for (int j = 0; j < buildVer.ntools; j++) {
var buildToolVer = new BuildVersionCommand.BuildToolVersion ();
buildToolVer.tool = reader.ReadUInt32 ();
buildToolVer.version = reader.ReadUInt32 ();
buildVer.tools[j] = buildToolVer;
}
lc = buildVer;
break;
default:
lc = new LoadCommand ();
lc.cmd = reader.ReadUInt32 ();
Expand Down Expand Up @@ -812,7 +911,7 @@ internal void ReadEntry (BinaryReader reader)
entry.Read (reader);
} else if (StaticLibrary.IsStaticLibrary (reader)) {
static_library = new StaticLibrary ();
static_library.Read (reader);
static_library.Read (parent?.Filename, reader, size);
} else {
throw ErrorHelper.CreateError (1603, "Unknown format for fat entry at position {0} in {1}.", offset, parent.Filename);
}
Expand Down Expand Up @@ -884,4 +983,34 @@ public Version Sdk {
get { return DeNibble (sdk); }
}
}

public class BuildVersionCommand : LoadCommand {
public uint platform;
public uint minos; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */
public uint sdk; /* X.Y.Z is encoded in nibbles xxxx.yy.zz */
public uint ntools;
public BuildToolVersion[] tools;

public class BuildToolVersion {
public uint tool;
public uint version;
}

Version DeNibble (uint value)
{
return new Version ((int)(value >> 16), (int)((value >> 8) & 0xFF), (int)(value & 0xFF));
}

public Version MinOS {
get { return DeNibble (minos); }
}

public Version Sdk {
get { return DeNibble (sdk); }
}

public MachO.Platform Platform {
get { return (MachO.Platform)platform; }
}
}
}

3 comments on commit 5905242

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❌ Build was (probably) aborted

🔥 Jenkins job (on internal Jenkins) failed in stage(s) 'Test run' 🔥 : org.jenkinsci.plugins.workflow.steps.FlowInterruptedException

✅ Build succeeded
✅ Packages: xamarin.ios-12.7.0.197.pkg xamarin.mac-5.7.0.197.pkg
✅ API Diff (from stable)
⚠️ API Diff (from PR only) (🔥 breaking changes 🔥)
✅ Generator Diff (no change)
🔥 Test run failed 🔥

Test results

# Test run in progress: Built: 39, Succeeded: 288, Ignored: 1936, TimedOut: 5

Failed tests

  • monotouch-test/iOS Unified 64-bits - simulator/Debug: TimedOut
  • introspection/iOS Unified 64-bits - simulator/Debug: TimedOut
  • monotouch-test/iOS Unified 64-bits - simulator/Debug (static registrar): TimedOut
  • monotouch-test/iOS Unified 64-bits - simulator/Release (all optimizations): TimedOut
  • monotouch-test/iOS Unified 64-bits - simulator/Debug (all optimizations): TimedOut

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❌ Build was (probably) aborted

🔥 Jenkins job (on internal Jenkins) failed in stage(s) 'Test run, Test run' 🔥

✅ Build succeeded
✅ Packages: xamarin.ios-12.7.0.197.pkg xamarin.mac-5.7.0.197.pkg
✅ API Diff (from stable)
⚠️ API Diff (from PR only) (🔥 breaking changes 🔥)
✅ Generator Diff (no change)
🔥 Test run failed 🔥

Test results

2 tests failed, 0 tests skipped, 330 tests passed.

Failed tests

  • [NUnit] Mono CorlibTests/watchOS - simulator/Debug: Crashed
  • [xUnit] Mono CorlibXunit/watchOS - simulator/Debug: Crashed

@dalexsoto
Copy link
Member

@dalexsoto dalexsoto commented on 5905242 Mar 4, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setting state to success where context is continuous-integration/jenkins/branch.

No blocking issues found

Please sign in to comment.