Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks] Reimplement NdkUtils (#6083)
Browse files Browse the repository at this point in the history 
Context: #5964
Context: #5964 (comment)
Context: #5996
Context: #6073

The `NdkUtils` class is used by Xamarin.Andrid.Build.Tasks to find
tooling shipped with the Android NDK, and has grown increasingly
complicated over the years due to a number of incompatibilities
between various NDK versions. The code became hard to follow and
untidy.

Address this issue by replacing the single static `NdkUtils` class
with a hierarchy of dynamically instantiated classes rooted in a new
base class, `NdkTools`.

`NdkUtils` had to be initialized for each thread that needed to access
its methods, which led to various issues with concurrency and lack of
proper initialization since the initialization had to be done wherever
`NdkUtils` was first accessed, meaning that every task using it had to
do it; see accc846 & [this comment][0].

`NdkTools` doesn't require such initialization, instead it provides an
`NdkTools.Create()` factory method takes the path to the NDK and
returns an instance of an `NdkTools` subclass (or `null` if an error
occurs), which the can be safely used by the caller.  Callers need not
concern themselves with what is the actual type of the returned
instance, they access only methods and properties defined in the
`NdkTools` base abstract class.

The hierarchy of `NdkTools` derivatives is structured and named after
the breaking changes in the NDK.  For instance, NDK versions before 16
used the GNU compilers, while release 16 and above use the clang
compilers - this is reflected in existence of two classes derived from
`NdkTools`, `NoClang` for NDKs older than r16 and `WithClang` for the
newer ones.  The other breaking changes are the addition of unified
headers in r19, removal of the `platforms` directory in r22 and removal
of GNU Binutils in r23.

NDK r23 is recognized in this commit but it is NOT supported.
Support for r23 is being worked on in PR #6073, which will be merged
once r23 is out of beta.

[0]: #5964 (comment)
  • Loading branch information
@grendello
grendello authored Jul 15, 2021
1 parent 1d53c00 commit 71ae556
Show file tree
Hide file tree
Showing 20 changed files with 1,000 additions and 863 deletions.
54 changes: 22 additions & 32 deletions src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,6 @@ public class Aot : AndroidAsyncTask
AotMode AotMode;
SequencePointsMode sequencePointsMode;

public override bool RunTask ()
{
// NdkUtil must always be initialized - once per thread
if (!NdkUtil.Init (LogCodedError, AndroidNdkDirectory))
return false;

return base.RunTask ();
}

public static bool GetAndroidAotMode(string androidAotMode, out AotMode aotMode)
{
aotMode = AotMode.Normal;
Expand Down Expand Up @@ -138,7 +129,7 @@ public static bool TryGetSequencePointsMode (string value, out SequencePointsMod
return false;
}

static string GetNdkToolchainLibraryDir(string binDir, string archDir = null)
static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, string archDir = null)
{
var baseDir = Path.GetFullPath(Path.Combine(binDir, ".."));

Expand All @@ -154,7 +145,7 @@ static string GetNdkToolchainLibraryDir(string binDir, string archDir = null)
goto no_toolchain_error;
}

if (NdkUtil.UsingClangNDK)
if (ndk.UsesClang)
return libPath;

gccLibDir = Directory.EnumerateDirectories(libPath).ToList();
Expand All @@ -171,9 +162,9 @@ static string GetNdkToolchainLibraryDir(string binDir, string archDir = null)
throw new Exception("Could not find a valid NDK compiler toolchain library path");
}

static string GetNdkToolchainLibraryDir (string binDir, AndroidTargetArch arch)
static string GetNdkToolchainLibraryDir (NdkTools ndk, string binDir, AndroidTargetArch arch)
{
return GetNdkToolchainLibraryDir (binDir, NdkUtil.GetArchDirName (arch));
return GetNdkToolchainLibraryDir (ndk, binDir, ndk.GetArchDirName (arch));
}

static string QuoteFileName(string fileName)
Expand All @@ -183,7 +174,7 @@ static string QuoteFileName(string fileName)
return builder.ToString();
}

int GetNdkApiLevel(string androidNdkPath, string androidApiLevel, AndroidTargetArch arch)
int GetNdkApiLevel (NdkTools ndk, string androidApiLevel, AndroidTargetArch arch)
{
var manifest = AndroidAppManifest.Load (ManifestFile.ItemSpec, MonoAndroidHelper.SupportedVersions);

Expand All @@ -209,15 +200,15 @@ int GetNdkApiLevel(string androidNdkPath, string androidApiLevel, AndroidTargetA
else if (level == 23) level = 21;

// API levels below level 21 do not provide support for 64-bit architectures.
if (NdkUtil.IsNdk64BitArch(arch) && level < 21) {
if (ndk.IsNdk64BitArch (arch) && level < 21) {
level = 21;
}

// We perform a downwards API level lookup search since we might not have hardcoded the correct API
// mapping above and we do not want to crash needlessly.
for (; level >= 5; level--) {
try {
NdkUtil.GetNdkPlatformLibPath (androidNdkPath, arch, level);
ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level);
break;
} catch (InvalidOperationException ex) {
// Path not found, continue searching...
Expand All @@ -230,10 +221,9 @@ int GetNdkApiLevel(string androidNdkPath, string androidApiLevel, AndroidTargetA

public async override System.Threading.Tasks.Task RunTaskAsync ()
{
// NdkUtil must always be initialized - once per thread
if (!NdkUtil.Init (LogCodedError, AndroidNdkDirectory)) {
LogDebugMessage ("Failed to initialize NdkUtil");
return;
NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log);
if (ndk == null) {
return; // NdkTools.Create will log appropriate error
}

bool hasValidAotMode = GetAndroidAotMode (AndroidAotMode, out AotMode);
Expand All @@ -251,7 +241,7 @@ public async override System.Threading.Tasks.Task RunTaskAsync ()

var nativeLibs = new List<string> ();

await this.WhenAllWithLock (GetAotConfigs (),
await this.WhenAllWithLock (GetAotConfigs (ndk),
(config, lockObject) => {
if (!config.Valid) {
Cancel ();
Expand All @@ -277,7 +267,7 @@ await this.WhenAllWithLock (GetAotConfigs (),
LogDebugTaskItems (" NativeLibrariesReferences: ", NativeLibrariesReferences);
}

IEnumerable<Config> GetAotConfigs ()
IEnumerable<Config> GetAotConfigs (NdkTools ndk)
{
if (!Directory.Exists (AotOutputDirectory))
Directory.CreateDirectory (AotOutputDirectory);
Expand Down Expand Up @@ -325,7 +315,7 @@ IEnumerable<Config> GetAotConfigs ()
throw new Exception ("Unsupported Android target architecture ABI: " + abi);
}

if (EnableLLVM && !NdkUtil.ValidateNdkPlatform (LogMessage, LogCodedError, AndroidNdkDirectory, arch, enableLLVM:EnableLLVM)) {
if (EnableLLVM && !ndk.ValidateNdkPlatform (LogMessage, LogCodedError, arch, enableLLVM:EnableLLVM)) {
yield return Config.Invalid;
yield break;
}
Expand All @@ -341,8 +331,8 @@ IEnumerable<Config> GetAotConfigs ()

int level = 0;
string toolPrefix = EnableLLVM
? NdkUtil.GetNdkToolPrefix (AndroidNdkDirectory, arch, level = GetNdkApiLevel (AndroidNdkDirectory, AndroidApiLevel, arch))
: Path.Combine (AndroidBinUtilsDirectory, $"{NdkUtil.GetArchDirName (arch)}-");
? ndk.GetNdkToolPrefixForAOT (arch, level = GetNdkApiLevel (ndk, AndroidApiLevel, arch))
: Path.Combine (AndroidBinUtilsDirectory, $"{ndk.GetArchDirName (arch)}-");
var toolchainPath = toolPrefix.Substring(0, toolPrefix.LastIndexOf(Path.DirectorySeparatorChar));
var ldFlags = string.Empty;
if (EnableLLVM) {
Expand All @@ -353,25 +343,25 @@ IEnumerable<Config> GetAotConfigs ()

string androidLibPath = string.Empty;
try {
androidLibPath = NdkUtil.GetNdkPlatformLibPath(AndroidNdkDirectory, arch, level);
androidLibPath = ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level);
} catch (InvalidOperationException ex) {
Diagnostic.Error (5101, ex.Message);
}

string toolchainLibDir;
if (NdkUtil.UsingClangNDK)
toolchainLibDir = GetNdkToolchainLibraryDir (toolchainPath, arch);
if (ndk.UsesClang)
toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath, arch);
else
toolchainLibDir = GetNdkToolchainLibraryDir (toolchainPath);
toolchainLibDir = GetNdkToolchainLibraryDir (ndk, toolchainPath);

var libs = new List<string>();
if (NdkUtil.UsingClangNDK) {
if (ndk.UsesClang) {
libs.Add ($"-L{toolchainLibDir.TrimEnd ('\\')}");
libs.Add ($"-L{androidLibPath.TrimEnd ('\\')}");

if (arch == AndroidTargetArch.Arm) {
// Needed for -lunwind to work
string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", NdkUtil.GetArchDirName (arch));
string compilerLibDir = Path.Combine (toolchainPath, "..", "sysroot", "usr", "lib", ndk.GetArchDirName (arch));
libs.Add ($"-L{compilerLibDir.TrimEnd ('\\')}");
}
}
Expand All @@ -385,7 +375,7 @@ IEnumerable<Config> GetAotConfigs ()

string ldName = String.Empty;
if (EnableLLVM) {
ldName = NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "ld", level);
ldName = ndk.GetToolPath (NdkToolKind.Linker, arch, level);
if (!String.IsNullOrEmpty (ldName)) {
ldName = Path.GetFileName (ldName);
if (ldName.IndexOf ('-') >= 0) {
Expand Down
8 changes: 6 additions & 2 deletions src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,8 +635,12 @@ private void AddNativeLibraries (ArchiveFileList files, string [] supportedAbis)
return;
}

NdkUtil.Init (AndroidNdkDirectory);
string clangDir = NdkUtil.GetClangDeviceLibraryPath (AndroidNdkDirectory);
NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log);
if (ndk == null) {
return; // NdkTools.Create will log appropriate error
}

string clangDir = ndk.GetClangDeviceLibraryPath ();
if (String.IsNullOrEmpty (clangDir)) {
LogSanitizerError ($"Unable to find the clang compiler directory. Is NDK installed?");
return;
Expand Down
2 changes: 1 addition & 1 deletion src/Xamarin.Android.Build.Tasks/Tasks/LinkApplicationSharedLibraries.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ IEnumerable<Config> GetLinkerConfigs ()
linkerArgs.Add (QuoteFileName (file));
}

string ld = MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, $"{NdkUtil.GetNdkToolchainPrefix (arch, false)}ld");
string ld = MonoAndroidHelper.GetExecutablePath (AndroidBinUtilsDirectory, $"{NdkTools.GetBinutilsToolchainPrefix (arch)}ld");
yield return new Config {
LinkerPath = Path.Combine (AndroidBinUtilsDirectory, ld),
LinkerOptions = String.Join (" ", linkerArgs),
Expand Down
34 changes: 18 additions & 16 deletions src/Xamarin.Android.Build.Tasks/Tasks/MakeBundleNativeCodeExternal.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ public class MakeBundleNativeCodeExternal : AndroidTask

[Required]
public ITaskItem[] Assemblies { get; set; }

// Which ABIs to include native libs for
[Required]
public string [] SupportedAbis { get; set; }

[Required]
public string TempOutputPath { get; set; }

Expand All @@ -57,11 +57,13 @@ public MakeBundleNativeCodeExternal ()

public override bool RunTask ()
{
if (!NdkUtil.Init (Log, AndroidNdkDirectory))
return false;
NdkTools? ndk = NdkTools.Create (AndroidNdkDirectory, Log);
if (ndk == null) {
return false; // NdkTools.Create will log appropriate error
}

try {
return DoExecute ();
return DoExecute (ndk);
} catch (XamarinAndroidException e) {
Log.LogCodedError (string.Format ("XA{0:0000}", e.Code), e.MessageWithoutCode);
if (MonoAndroidHelper.LogInternalExceptions)
Expand All @@ -72,7 +74,7 @@ public override bool RunTask ()
return !Log.HasLoggedErrors;
}

bool DoExecute ()
bool DoExecute (NdkTools ndk)
{
var results = new List<ITaskItem> ();
string bundlepath = Path.Combine (TempOutputPath, "bundles");
Expand Down Expand Up @@ -102,11 +104,11 @@ bool DoExecute ()
break;
}

if (!NdkUtil.ValidateNdkPlatform (Log, AndroidNdkDirectory, arch, enableLLVM: false)) {
if (!ndk.ValidateNdkPlatform (arch, enableLLVM: false)) {
return false;
}

int level = NdkUtil.GetMinimumApiLevelFor (arch, AndroidNdkDirectory);
int level = ndk.GetMinimumApiLevelFor (arch);
var outpath = Path.Combine (bundlepath, abi);
if (!Directory.Exists (outpath))
Directory.CreateDirectory (outpath);
Expand Down Expand Up @@ -140,10 +142,10 @@ bool DoExecute ()
CreateNoWindow = true,
WindowStyle = ProcessWindowStyle.Hidden,
};
string windowsCompilerSwitches = NdkUtil.GetCompilerTargetParameters (AndroidNdkDirectory, arch, level);
var compilerNoQuotes = NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "gcc", level);
string windowsCompilerSwitches = ndk.GetCompilerTargetParameters (arch, level);
var compilerNoQuotes = ndk.GetToolPath (NdkToolKind.CompilerC, arch, level);
var compiler = $"\"{compilerNoQuotes}\" {windowsCompilerSwitches}".Trim ();
var gas = '"' + NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "as", level) + '"';
var gas = '"' + ndk.GetToolPath (NdkToolKind.Assembler, arch, level) + '"';
psi.EnvironmentVariables ["CC"] = compiler;
psi.EnvironmentVariables ["AS"] = gas;
Log.LogDebugMessage ("CC=" + compiler);
Expand All @@ -167,7 +169,7 @@ bool DoExecute ()

clb = new CommandLineBuilder ();

// See NdkUtils.GetNdkTool for reasons why
// See NdkToolsWithClangWithPlatforms.ctor for reasons why
if (!String.IsNullOrEmpty (windowsCompilerSwitches))
clb.AppendTextUnquoted (windowsCompilerSwitches);

Expand All @@ -188,14 +190,14 @@ bool DoExecute ()
clb.AppendFileNameIfNotNull (IncludePath);
}

string asmIncludePath = NdkUtil.GetNdkAsmIncludePath (AndroidNdkDirectory, arch, level);
string asmIncludePath = ndk.GetDirectoryPath (NdkToolchainDir.AsmInclude, arch, level);
if (!String.IsNullOrEmpty (asmIncludePath)) {
clb.AppendSwitch ("-I");
clb.AppendFileNameIfNotNull (asmIncludePath);
}

clb.AppendSwitch ("-I");
clb.AppendFileNameIfNotNull (NdkUtil.GetNdkPlatformIncludePath (AndroidNdkDirectory, arch, level));
clb.AppendFileNameIfNotNull (ndk.GetDirectoryPath (NdkToolchainDir.PlatformInclude, arch, level));
clb.AppendFileNameIfNotNull (Path.Combine (outpath, "temp.c"));
Log.LogDebugMessage ("[CC] " + compiler + " " + clb);
if (MonoAndroidHelper.RunProcess (compilerNoQuotes, clb.ToString (), OnCcOutputData, OnCcErrorData) != 0) {
Expand All @@ -216,13 +218,13 @@ bool DoExecute ()
clb.AppendSwitch ("-o");
clb.AppendFileNameIfNotNull (Path.Combine (outpath, BundleSharedLibraryName));
clb.AppendSwitch ("-L");
clb.AppendFileNameIfNotNull (NdkUtil.GetNdkPlatformLibPath (AndroidNdkDirectory, arch, level));
clb.AppendFileNameIfNotNull (ndk.GetDirectoryPath (NdkToolchainDir.PlatformLib, arch, level));
clb.AppendSwitch ("-lc");
clb.AppendSwitch ("-lm");
clb.AppendSwitch ("-ldl");
clb.AppendSwitch ("-llog");
clb.AppendSwitch ("-lz"); // Compress
string ld = NdkUtil.GetNdkTool (AndroidNdkDirectory, arch, "ld", level);
string ld = ndk.GetToolPath (NdkToolKind.Linker, arch, level);
Log.LogMessage (MessageImportance.Normal, "[LD] " + ld + " " + clb);
if (MonoAndroidHelper.RunProcess (ld, clb.ToString (), OnLdOutputData, OnLdErrorData) != 0) {
Log.LogCodedError ("XA5201", Properties.Resources.XA5201, proc.ExitCode);
Expand Down
Loading

0 comments on commit 71ae556

Please sign in to comment.