[NDK] Locate and select only compatible NDK versions (#103)

Context: actions/runner-images#2420
Context: dotnet/android#5499
Context: dotnet/android#5526
Context: android/ndk#1427
Context: https://developer.android.com/studio/command-line/variables#envar

Xamarin.Android is not (yet) compatible with the recently released
Android NDK r22 version.  Azure build images have recently rolled out
an update which includes NDK r22 and, thus, it breaks builds for
customers using any form of Xamarin.Android AOT build.

In an attempt to detect broken/incompatible NDK versions as well as
to select the "best one", this commit adds code to scan the known NDK
locations in search of the preferred version.

Given an `AndroidSdkInfo` creation of:

	var info = new AndroidSdkInfo (logger:logger,
	        androidSdkPath: sdkPath,
	        androidNdkPath: ndkPath,
	        javaSdkPath: javaPath);
	var usedNdkPath = info.AndroidNdkPath;

If `ndkPath` is not `null` and otherwise valid, then `usedNdkPath`
is `ndkPath`.

If `ndkPath` is `null` or is otherwise invalid (missing `ndk-stack`,
etc.), then we search for an `info.AndroidNdkPath` value as follows:

 1. If `androidSdkPath` is not `null` and valid, then we check for
    Android SDK-relative NDK locations, in:

      * `{androidSdkPath}/ndk/*`
      * `{androidSdkPath}/ndk-bundle`

    For each found SDK-relative NDK directory, we filter out NDKs for
    which we cannot determine the package version, as well as those
    which are "too old" (< `MinimumCompatibleNDKMajorVersion`) or
    "too new" (> `MaximumCompatibleNDKMajorVersion`), currently r22.

    We prefer the NDK location with the highest version number. 

 2. If `androidSdkPath` is not `null` and valid and if there are no
    Android SDK-relative NDK locations, then we use the user-selected
    "preferred NDK location".  See also
    `AndroidSdkInfo.SetPreferredAndroidNdkPath()`.

 3. If `androidSdkPath` is not `null` and valid and if the preferred
    NDK location isn't set or is invalid, then we check directories
    specified in `$PATH`, and use the directory which contains
    `ndk-stack`.

 4. If `androidSdkPath` is not `null` and valid and `$PATH` didn't
    contain `ndk-stack`, then we continue looking for NDK locations
    within the Android SDK locations specified by the `$ANDROID_HOME`
    and `$ANDROID_SDK_ROOT` environment variables.
    As with (1), these likewise look for e.g. `${ANDROID_HOME}/ndk/*`
    or `${ANDROID_SDK_ROOT}/ndk-bundle` directories and select the
    NDK with the highest supported version.

 5. If `androidSdkPath` is `null`, then *first* we try to find a
    valid Android SDK directory, using on Unix:

     a. The preferred Android SDK directory; see also
        `AndroidSdkInfo.SetPreferredAndroidSdkPath().

     b. The `$ANDROID_HOME` and `ANDROID_SDK_ROOT`
        environment variables.

     c. Directories within `$PATH` that contain `adb`.

    Once an Android SDK is found, steps (1)…(4) are performed.

In (1) and (4), we now look for the Android SDK packages containing
the NDK.  There are two kinds of such packages:

  * `ndk-bundle` is the older package which allows for installation of
    only one NDK inside the SDK directory
  * `ndk/*` is a newer package which allows for installation of several
    NDK versions in parallel.  Each subdirectory of `ndk` is an `X.Y.Z`
    version number of the NDK.

In each of these directories we look for the `source.properties` file
from which we then extract the NDK version and then we sort thus
discovered NDK instances using their version as the key, in the
descending order.  The latest compatible (currently: less than 22 and
more than 15) version is selected and its path returned to the caller.

src/Xamarin.Android.Tools.AndroidSdk/Sdks/AndroidSdkBase.cs

@@ -9,6 +9,15 @@ namespace Xamarin.Android.Tools
 {
 	abstract class AndroidSdkBase
 	{
+		// When this changes, update the test: Xamarin.Android.Tools.Tests.AndroidSdkInfoTests.Ndk_MultipleNdkVersionsInSdk
+		const int MinimumCompatibleNDKMajorVersion = 16;
+		const int MaximumCompatibleNDKMajorVersion = 21;
+
+		static readonly char[] SourcePropertiesKeyValueSplit = new char[] { '=' };
+
+		// Per https://developer.android.com/studio/command-line/variables#envar
+		protected static readonly string[] AndroidSdkEnvVars = {"ANDROID_HOME", "ANDROID_SDK_ROOT"};
+
 		string[]? allAndroidSdks;
 
 		public string[] AllAndroidSdks {
@@ -97,8 +106,9 @@ public virtual void Initialize (string? androidSdkPath = null, string? androidNd
 			if (pathValidator (ctorParam))
 				return ctorParam;
 			foreach (var path in getAllPaths ()) {
-				if (pathValidator (path))
+				if (pathValidator (path)) {
 					return path;
+				}
 			}
 			return null;
 		}
@@ -108,7 +118,7 @@ public virtual void Initialize (string? androidSdkPath = null, string? androidNd
 			if (ValidateAndroidNdkLocation (ctorParam))
 				return ctorParam;
 			if (AndroidSdkPath != null) {
-				string bundle = Path.Combine (AndroidSdkPath, "ndk-bundle");
+				string bundle = FindBestNDK (AndroidSdkPath);
 				if (Directory.Exists (bundle) && ValidateAndroidNdkLocation (bundle))
 					return bundle;
 			}
@@ -125,6 +135,18 @@ public virtual void Initialize (string? androidSdkPath = null, string? androidNd
 		protected abstract IEnumerable<string> GetAllAvailableAndroidSdks ();
 		protected abstract string GetShortFormPath (string path);
 
+		protected IEnumerable<string> GetSdkFromEnvironmentVariables ()
+		{
+			foreach (string envVar in AndroidSdkEnvVars) {
+				string ev = Environment.GetEnvironmentVariable (envVar);
+				if (String.IsNullOrEmpty (ev)) {
+					continue;
+				}
+
+				yield return ev;
+			}
+		}
+
 		protected virtual IEnumerable<string> GetAllAvailableAndroidNdks ()
 		{
 			// Look in PATH
@@ -139,7 +161,67 @@ protected virtual IEnumerable<string> GetAllAvailableAndroidNdks ()
 			foreach (var sdk in GetAllAvailableAndroidSdks ()) {
 				if (sdk == AndroidSdkPath)
 					continue;
-				yield return Path.Combine (sdk, "ndk-bundle");
+				yield return FindBestNDK (sdk);
+			}
+		}
+
+		string FindBestNDK (string androidSdkPath)
+		{
+			var ndkInstances = new SortedDictionary<Version, string> (Comparer<Version>.Create ((Version l, Version r) => r.CompareTo (l)));
+
+			foreach (string ndkPath in Directory.EnumerateDirectories (androidSdkPath, "ndk*", SearchOption.TopDirectoryOnly)) {
+				if (String.Compare ("ndk-bundle", Path.GetFileName (ndkPath), StringComparison.OrdinalIgnoreCase) == 0) {
+					LoadNDKVersion (ndkPath);
+					continue;
+				}
+
+				if (String.Compare ("ndk", Path.GetFileName (ndkPath), StringComparison.OrdinalIgnoreCase) != 0) {
+					continue;
+				}
+
+				foreach (string versionedNdkPath in Directory.EnumerateDirectories (ndkPath, "*", SearchOption.TopDirectoryOnly)) {
+					LoadNDKVersion (versionedNdkPath);
+				}
+			}
+
+			if (ndkInstances.Count == 0) {
+				return String.Empty;
+			}
+
+			var kvp = ndkInstances.First ();
+			Logger (TraceLevel.Verbose, $"Best NDK selected: v{kvp.Key} in {kvp.Value}");
+			return kvp.Value;
+
+			void LoadNDKVersion (string path)
+			{
+				string propsFilePath = Path.Combine (path, "source.properties");
+				if (!File.Exists (propsFilePath)) {
+					Logger (TraceLevel.Verbose, $"Skipping NDK in '{path}': no source.properties, cannot determine version");
+					return;
+				}
+
+				foreach (string line in File.ReadLines (propsFilePath)) {
+					string[] parts = line.Split (SourcePropertiesKeyValueSplit, 2, StringSplitOptions.RemoveEmptyEntries);
+					if (parts.Length != 2) {
+						continue;
+					}
+
+					if (String.Compare ("Pkg.Revision", parts[0].Trim (), StringComparison.Ordinal) != 0) {
+						continue;
+					}
+
+					if (!Version.TryParse (parts[1].Trim (), out Version? ndkVer) || ndkVer == null || ndkInstances.ContainsKey (ndkVer)) {
+						continue;
+					}
+
+					if (ndkVer.Major < MinimumCompatibleNDKMajorVersion || ndkVer.Major > MaximumCompatibleNDKMajorVersion) {
+						Logger (TraceLevel.Verbose, $"Skipping NDK in '{path}': version {ndkVer} is out of the accepted range (major version must be between {MinimumCompatibleNDKMajorVersion} and {MaximumCompatibleNDKMajorVersion}");
+						continue;
+					}
+
+					ndkInstances.Add (ndkVer, path);
+					return;
+				}
 			}
 		}
 

src/Xamarin.Android.Tools.AndroidSdk/Sdks/AndroidSdkUnix.cs

@@ -94,6 +94,10 @@ protected override IEnumerable<string> GetAllAvailableAndroidSdks ()
 			if (!string.IsNullOrEmpty (preferedSdkPath))
 				yield return preferedSdkPath!;
 
+			foreach (string dir in GetSdkFromEnvironmentVariables ()) {
+				yield return dir;
+			}
+
 			// Look in PATH
 			foreach (var adb in ProcessUtils.FindExecutablesInPath (Adb)) {
 				var path = Path.GetDirectoryName (adb);

src/Xamarin.Android.Tools.AndroidSdk/Sdks/AndroidSdkWindows.cs

@@ -230,6 +230,10 @@ protected override IEnumerable<string> GetAllAvailableAndroidNdks ()
 				if (CheckRegistryKeyForExecutable (root, regKey, MDREG_ANDROID_NDK, wow, ".", NdkStack))
 					yield return RegistryEx.GetValueString (root, regKey, MDREG_ANDROID_NDK, wow) ?? "";
 
+			foreach (string dir in GetSdkFromEnvironmentVariables ()) {
+				yield return dir;
+			}
+
 			/*
 			// Check for the key written by the Xamarin installer
 			if (CheckRegistryKeyForExecutable (RegistryEx.CurrentUser, XAMARIN_ANDROID_INSTALLER_PATH, XAMARIN_ANDROID_INSTALLER_KEY, wow, "platform-tools", Adb))

tests/Xamarin.Android.Tools.AndroidSdk-Tests/AndroidSdkInfoTests.cs

@@ -13,6 +13,8 @@ namespace Xamarin.Android.Tools.Tests
 	[TestFixture]
 	public class AndroidSdkInfoTests
 	{
+		const string NdkVersion = "21.0.6113669";
+
 		string      UnixConfigDirOverridePath;
 		string      PreferredJdksOverridePath;
 
@@ -64,21 +66,73 @@ public void Constructor_Paths ()
 			}
 		}
 
+		[Test]
+		public void Ndk_MultipleNdkVersionsInSdk ()
+		{
+			// Must match like-named constants in AndroidSdkBase
+			const int MinimumCompatibleNDKMajorVersion = 16;
+			const int MaximumCompatibleNDKMajorVersion = 21;
+
+			CreateSdks(out string root, out string jdk, out string ndk, out string sdk);
+
+			Action<TraceLevel, string> logger = (level, message) => {
+				Console.WriteLine($"[{level}] {message}");
+			};
+
+			var ndkVersions = new List<string> {
+				"16.1.4479499",
+				"17.2.4988734",
+				"18.1.5063045",
+				"19.2.5345600",
+				"20.0.5594570",
+				"20.1.5948944",
+				"21.0.6113669",
+				"21.1.6352462",
+				"21.2.6472646",
+				"21.3.6528147",
+				"22.0.7026061",
+			};
+			string expectedVersion = "21.3.6528147";
+			string expectedNdkPath = Path.Combine (sdk, "ndk", expectedVersion);
+
+			try {
+				MakeNdkDir (Path.Combine (sdk, "ndk-bundle"), NdkVersion);
+
+				foreach (string ndkVer in ndkVersions) {
+					MakeNdkDir (Path.Combine (sdk, "ndk", ndkVer), ndkVer);
+				}
+
+				var info = new AndroidSdkInfo (logger, androidSdkPath: sdk, androidNdkPath: null, javaSdkPath: jdk);
+
+				Assert.AreEqual (expectedNdkPath, info.AndroidNdkPath, "AndroidNdkPath not found inside sdk!");
+
+				string ndkVersion = Path.GetFileName (info.AndroidNdkPath);
+				if (!Version.TryParse (ndkVersion, out Version ver)) {
+					Assert.Fail ($"Unable to parse '{ndkVersion}' as a valid version.");
+				}
+
+				Assert.True (ver.Major >= MinimumCompatibleNDKMajorVersion, $"NDK version must be at least {MinimumCompatibleNDKMajorVersion}");
+				Assert.True (ver.Major <= MaximumCompatibleNDKMajorVersion, $"NDK version must be at most {MinimumCompatibleNDKMajorVersion}");
+			} finally {
+				Directory.Delete (root, recursive: true);
+			}
+		}
+
 		[Test]
 		public void Ndk_PathInSdk()
 		{
 			CreateSdks(out string root, out string jdk, out string ndk, out string sdk);
 
-			var logs = new StringWriter();
 			Action<TraceLevel, string> logger = (level, message) => {
-				logs.WriteLine($"[{level}] {message}");
+				Console.WriteLine($"[{level}] {message}");
 			};
 
 			try
 			{
 				var extension = OS.IsWindows ? ".cmd" : "";
 				var ndkPath = Path.Combine(sdk, "ndk-bundle");
 				Directory.CreateDirectory(ndkPath);
+				File.WriteAllText(Path.Combine (ndkPath, "source.properties"), $"Pkg.Revision = {NdkVersion}");
 				Directory.CreateDirectory(Path.Combine(ndkPath, "toolchains"));
 				File.WriteAllText(Path.Combine(ndkPath, $"ndk-stack{extension}"), "");
 
@@ -106,6 +160,8 @@ public void Constructor_SetValuesFromPath ()
 			};
 			var oldPath = Environment.GetEnvironmentVariable ("PATH");
 			var oldJavaHome = Environment.GetEnvironmentVariable ("JAVA_HOME");
+			var oldAndroidHome = Environment.GetEnvironmentVariable ("ANDROID_HOME");
+			var oldAndroidSdkRoot = Environment.GetEnvironmentVariable ("ANDROID_SDK_ROOT");
 			try {
 				var paths   = new List<string> () {
 					Path.Combine (jdk, "bin"),
@@ -117,6 +173,12 @@ public void Constructor_SetValuesFromPath ()
 				if (!string.IsNullOrEmpty (oldJavaHome)) {
 					Environment.SetEnvironmentVariable ("JAVA_HOME", string.Empty);
 				}
+				if (!string.IsNullOrEmpty (oldAndroidHome)) {
+					Environment.SetEnvironmentVariable ("ANDROID_HOME", string.Empty);
+				}
+				if (!string.IsNullOrEmpty (oldAndroidSdkRoot)) {
+					Environment.SetEnvironmentVariable ("ANDROID_SDK_ROOT", string.Empty);
+				}
 
 				var info    = new AndroidSdkInfo (logger);
 
@@ -129,6 +191,12 @@ public void Constructor_SetValuesFromPath ()
 				if (!string.IsNullOrEmpty (oldJavaHome)) {
 					Environment.SetEnvironmentVariable ("JAVA_HOME", oldJavaHome);
 				}
+				if (!string.IsNullOrEmpty (oldAndroidHome)) {
+					Environment.SetEnvironmentVariable ("ANDROID_HOME", oldAndroidHome);
+				}
+				if (!string.IsNullOrEmpty (oldAndroidSdkRoot)) {
+					Environment.SetEnvironmentVariable ("ANDROID_SDK_ROOT", oldAndroidSdkRoot);
+				}
 				Directory.Delete (root, recursive: true);
 			}
 		}
@@ -243,7 +311,7 @@ static void CreateSdks (out string root, out string jdk, out string ndk, out str
 			Directory.CreateDirectory (jdk);
 
 			CreateFauxAndroidSdkDirectory (sdk, "26.0.0");
-			CreateFauxAndroidNdkDirectory (ndk);
+			CreateFauxAndroidNdkDirectory (ndk, NdkVersion);
 			CreateFauxJavaSdkDirectory (jdk, "1.8.0", out var _, out var _);
 		}
 
@@ -311,8 +379,9 @@ struct ApiInfo {
 			public  string      Id;
 		}
 
-		static void CreateFauxAndroidNdkDirectory (string androidNdkDirectory)
+		static void CreateFauxAndroidNdkDirectory (string androidNdkDirectory, string ndkVersion)
 		{
+			File.WriteAllText (Path.Combine (androidNdkDirectory, "source.properties"), $"Pkg.Revision = {ndkVersion}");
 			File.WriteAllText (Path.Combine (androidNdkDirectory, "ndk-stack"),     "");
 			File.WriteAllText (Path.Combine (androidNdkDirectory, "ndk-stack.cmd"), "");
 
@@ -474,5 +543,14 @@ public void GetBuildToolsPaths_StableVersionsFirst ()
 				Directory.Delete (root, recursive: true);
 			}
 		}
+
+		void MakeNdkDir (string rootPath, string version)
+		{
+			var extension = OS.IsWindows ? ".cmd" : String.Empty;
+			Directory.CreateDirectory(rootPath);
+			File.WriteAllText(Path.Combine (rootPath, "source.properties"), $"Pkg.Revision = {version}");
+			Directory.CreateDirectory(Path.Combine(rootPath, "toolchains"));
+			File.WriteAllText(Path.Combine(rootPath, $"ndk-stack{extension}"), String.Empty);
+		}
 	}
 }