From be47b3ecbb377727b1f9ecca24a973b42dd3ef7e Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 18 Feb 2020 21:54:02 -0600 Subject: [PATCH] [Xamarin.Android.Build.Tasks] implement a MemoryStreamPool (#4251) Context: https://docs.microsoft.com/dotnet/api/system.buffers.arraypool-1 When using `MemoryStream`, it is better to reuse a `MemoryStream` instance rather than creating new ones, such as in a loop: using (var memoryStream = new MemoryStream ()) foreach (var foo in bar) { //Reset for reuse memoryStream.SetLength (0); // Use the stream } } The `SetLength(0)` call preserves the underlying `byte[]` and just sets the `_length` and `_position` fields. Subsequent writes don't need to allocate another `byte[]`: https://github.com/mono/mono/blob/eb85a108a33ba86ffd184689b62ac1f7250c9818/mcs/class/referencesource/mscorlib/system/io/memorystream.cs#L527-L548 To make this pattern even better, we can model after `System.Buffers.ArrayPool` and reuse `MemoryStream` objects throughout the entire build. var memoryStream = MemoryStreamPool.Shared.Rent (); try { // Use the stream } finally { MemoryStreamPool.Shared.Return (memoryStream); } Note that you will have to take special care to not dispose the `MemoryStream`. If we `Return()` a disposed `MemoryStream`, that would be bad news! In many cases a `StreamWriter` or `TextWriter` are wrapped around a `MemoryStream`, so we implement a convenience method: using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { // Use the writer writer.Flush (); MonoAndroidHelper.CopyIfStreamChanged (writer.BaseStream, path); } In this case `writer` is a special `StreamWriter` that returns the underlying `MemoryStream` when the writer is disposed. It also takes care of *not* disposing the `MemoryStream` for you. ~~ Results ~~ The reuse of `MemoryStream` impacts the entire build. Testing on macOS, a build of the Xamarin.Forms integration project: ./bin/Release/bin/xabuild tests/Xamarin.Forms-Performance-Integration/Droid/Xamarin.Forms.Performance.Integration.Droid.csproj /restore Some of the specific targets that would be affected: * Before: 1618 ms _GenerateJavaStubs 1 calls 1656 ms _ResolveLibraryProjectImports 1 calls 40 ms _GeneratePackageManagerJava 1 calls * After: 1568 ms _GenerateJavaStubs 1 calls 1376 ms _ResolveLibraryProjectImports 1 calls 36 ms _GeneratePackageManagerJava 1 calls This seems like we could save ~334ms on incremental builds where these targets run. An example would be an incremental build where `MainActivity.cs` changed. If I compare the memory usage with the Mono profiler: * Before: Allocation summary Bytes Count Average Type name 166246184 37846 4392 System.Byte[] 82480 1031 80 System.IO.MemoryStream * After: Allocation summary Bytes Count Average Type name 157191784 37794 4159 System.Byte[] 77520 969 80 System.IO.MemoryStream It seems like we created ~62 fewer `MemoryStream` in this build and saved ~9,059,360 bytes of allocations. After a few runs, I found I could drop the numbers on some of the times in `MSBuildDeviceIntegration.csv`. The times seemed to improve a bit for incremental builds where a lot is happening, such as when the `.csproj` changes. Some of the time improved is likely other changes around `` or `` such as 97d250be or 1e96c795. ~~ General Refactoring ~~ Any usage of `new Utf8Encoding(false)` I moved to a `static` `MonoAndroidHelper.UTF8withoutBOM` field. The method `Generator.CreateJavaSources()` had eight parameters! I moved it to be an instance method on ``, which could use the properties of the task, reducing it to one parameter. --- .../Generator/Generator.cs | 100 ---------------- src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs | 2 +- .../Tasks/ClassParse.cs | 2 +- .../Tasks/CompileToDalvik.cs | 2 +- .../Tasks/GenerateJavaStubs.cs | 112 ++++++++++++++---- .../Tasks/GenerateLibraryResources.cs | 7 +- .../Tasks/GeneratePackageManagerJava.cs | 91 +++++++------- .../Tasks/JavaCompileToolTask.cs | 2 +- .../Tasks/ManifestMerger.cs | 5 +- .../Xamarin.Android.Build.Tests/BuildTest.cs | 4 +- .../Utilities/MemoryStreamPoolTests.cs | 57 +++++++++ .../Utilities/MonoAndroidHelperTests.cs | 20 ++++ .../Utilities/Files.cs | 7 +- .../Utilities/ManifestDocument.cs | 30 +++-- .../Utilities/MemoryStreamPool.cs | 64 ++++++++++ .../Utilities/MonoAndroidHelper.cs | 9 +- .../Utilities/ObjectPool.cs | 37 ++++++ .../Utilities/TypeMapGenerator.cs | 16 ++- .../TypeMappingNativeAssemblyGenerator.cs | 20 ++-- .../Utilities/XDocumentExtensions.cs | 6 +- .../MSBuildDeviceIntegration.csv | 20 ++-- 21 files changed, 385 insertions(+), 228 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Tasks/Generator/Generator.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MemoryStreamPoolTests.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/MemoryStreamPool.cs create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/ObjectPool.cs diff --git a/src/Xamarin.Android.Build.Tasks/Generator/Generator.cs b/src/Xamarin.Android.Build.Tasks/Generator/Generator.cs deleted file mode 100644 index bcbcc0161c6..00000000000 --- a/src/Xamarin.Android.Build.Tasks/Generator/Generator.cs +++ /dev/null @@ -1,100 +0,0 @@ -using Java.Interop.Tools.Diagnostics; -using Java.Interop.Tools.JavaCallableWrappers; -using Microsoft.Build.Utilities; -using Mono.Cecil; -using System; -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Text; -using Xamarin.Android.Tools; - -namespace Xamarin.Android.Tasks -{ - class Generator - { - public static bool CreateJavaSources (TaskLoggingHelper log, IEnumerable javaTypes, string outputPath, - string applicationJavaClass, string androidSdkPlatform, bool useSharedRuntime, bool generateOnCreateOverrides, bool hasExportReference) - { - string monoInit = GetMonoInitSource (androidSdkPlatform, useSharedRuntime); - - bool ok = true; - using (var memoryStream = new MemoryStream ()) - using (var writer = new StreamWriter (memoryStream)) { - foreach (var t in javaTypes) { - //Reset for reuse - memoryStream.SetLength (0); - - try { - var jti = new JavaCallableWrapperGenerator (t, log.LogWarning) { - GenerateOnCreateOverrides = generateOnCreateOverrides, - ApplicationJavaClass = applicationJavaClass, - MonoRuntimeInitialization = monoInit, - }; - - jti.Generate (writer); - writer.Flush (); - - var path = jti.GetDestinationPath (outputPath); - MonoAndroidHelper.CopyIfStreamChanged (memoryStream, path); - if (jti.HasExport && !hasExportReference) - Diagnostic.Error (4210, Properties.Resources.XA4210); - } catch (XamarinAndroidException xae) { - ok = false; - log.LogError ( - subcategory: "", - errorCode: "XA" + xae.Code, - helpKeyword: string.Empty, - file: xae.SourceFile, - lineNumber: xae.SourceLine, - columnNumber: 0, - endLineNumber: 0, - endColumnNumber: 0, - message: xae.MessageWithoutCode, - messageArgs: new object [0] - ); - } catch (DirectoryNotFoundException ex) { - ok = false; - if (OS.IsWindows) { - Diagnostic.Error (5301, Properties.Resources.XA5301, t.FullName, ex); - } else { - Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); - } - } catch (Exception ex) { - ok = false; - Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); - } - } - } - return ok; - } - - static string GetMonoInitSource (string androidSdkPlatform, bool useSharedRuntime) - { - // Lookup the mono init section from MonoRuntimeProvider: - // Mono Runtime Initialization {{{ - // }}} - var builder = new StringBuilder (); - var runtime = useSharedRuntime ? "Shared" : "Bundled"; - var api = ""; - if (int.TryParse (androidSdkPlatform, out int apiLevel) && apiLevel < 21) { - api = ".20"; - } - var assembly = Assembly.GetExecutingAssembly (); - using (var s = assembly.GetManifestResourceStream ($"MonoRuntimeProvider.{runtime}{api}.java")) - using (var reader = new StreamReader (s)) { - bool copy = false; - string line; - while ((line = reader.ReadLine ()) != null) { - if (string.CompareOrdinal ("\t\t// Mono Runtime Initialization {{{", line) == 0) - copy = true; - if (copy) - builder.AppendLine (line); - if (string.CompareOrdinal ("\t\t// }}}", line) == 0) - break; - } - } - return builder.ToString (); - } - } -} diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs b/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs index 109eba737ac..ac39663494a 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/Aot.cs @@ -435,7 +435,7 @@ bool RunAotCompiler (string assembliesPath, string aotCompiler, string aotOption var stdout_completed = new ManualResetEvent (false); var stderr_completed = new ManualResetEvent (false); - using (var sw = new StreamWriter (responseFile, append: false, encoding: new UTF8Encoding (encoderShouldEmitUTF8Identifier: false))) { + using (var sw = new StreamWriter (responseFile, append: false, encoding: MonoAndroidHelper.UTF8withoutBOM)) { sw.WriteLine (aotOptions + " " + QuoteFileName (assembly)); } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ClassParse.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ClassParse.cs index 8cdbc49caa7..977d08dacc6 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ClassParse.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ClassParse.cs @@ -28,7 +28,7 @@ public class ClassParse : AndroidTask public override bool RunTask () { using (var output = new StreamWriter (OutputFile, append: false, - encoding: new UTF8Encoding (encoderShouldEmitUTF8Identifier: false))) { + encoding: MonoAndroidHelper.UTF8withoutBOM)) { Bytecode.Log.OnLog = LogEventHandler; var classPath = new Bytecode.ClassPath () { ApiSource = "class-parse", diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/CompileToDalvik.cs b/src/Xamarin.Android.Build.Tasks/Tasks/CompileToDalvik.cs index 3fc3ac097c1..0b81c372a5c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/CompileToDalvik.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/CompileToDalvik.cs @@ -117,7 +117,7 @@ protected override string GenerateCommandLineCommands () cmd.AppendSwitchIfNotNull ("--output ", Path.GetDirectoryName (ClassesOutputDirectory)); using (var sw = new StreamWriter (path: inputListFile, append: false, - encoding: new UTF8Encoding (encoderShouldEmitUTF8Identifier: false))) { + encoding: MonoAndroidHelper.UTF8withoutBOM)) { // .jar files if (AlternativeJarFiles != null && AlternativeJarFiles.Any ()) { Log.LogDebugMessage (" processing AlternativeJarFiles..."); diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 5664b18aaf5..f5c52610d17 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -5,7 +5,7 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Xml.Linq; +using System.Reflection; using System.Text; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -171,7 +171,7 @@ void Run (DirectoryAssemblyResolver res) // Step 2 - Generate type maps // Type mappings need to use all the assemblies, always. - WriteTypeMappings (res, allJavaTypes); + WriteTypeMappings (allJavaTypes); var javaTypes = new List (); foreach (TypeDefinition td in allJavaTypes) { @@ -183,15 +183,7 @@ void Run (DirectoryAssemblyResolver res) } // Step 3 - Generate Java stub code - var success = Generator.CreateJavaSources ( - Log, - javaTypes, - Path.Combine (OutputDirectory, "src"), - ApplicationJavaClass, - AndroidSdkPlatform, - UseSharedRuntime, - int.Parse (AndroidSdkPlatform) <= 10, - hasExportReference); + var success = CreateJavaSources (javaTypes); if (!success) return; @@ -202,9 +194,7 @@ void Run (DirectoryAssemblyResolver res) var managedConflicts = new Dictionary> (0, StringComparer.Ordinal); var javaConflicts = new Dictionary> (0, StringComparer.Ordinal); - // Allocate a MemoryStream with a reasonable guess at its capacity - using (var stream = new MemoryStream (javaTypes.Count * 32)) - using (var acw_map = new StreamWriter (stream)) { + using (var acw_map = MemoryStreamPool.Shared.CreateStreamWriter (Encoding.Default)) { foreach (TypeDefinition type in javaTypes) { string managedKey = type.FullName.Replace ('/', '.'); string javaKey = JavaNativeTypeManager.ToJniName (type).Replace ('/', '.'); @@ -246,7 +236,7 @@ void Run (DirectoryAssemblyResolver res) } acw_map.Flush (); - MonoAndroidHelper.CopyIfStreamChanged (stream, AcwMapFile); + MonoAndroidHelper.CopyIfStreamChanged (acw_map.BaseStream, AcwMapFile); } foreach (var kvp in managedConflicts) { @@ -281,11 +271,9 @@ void Run (DirectoryAssemblyResolver res) var additionalProviders = manifest.Merge (Log, allJavaTypes, ApplicationJavaClass, EmbedAssemblies, BundledWearApplicationName, MergedManifestDocuments); - using (var stream = new MemoryStream ()) { - manifest.Save (Log, stream); - - // Only write the new manifest if it actually changed - MonoAndroidHelper.CopyIfStreamChanged (stream, MergedAndroidManifestOutput); + // Only write the new manifest if it actually changed + if (manifest.SaveIfChanged (Log, MergedAndroidManifestOutput)) { + Log.LogDebugMessage ($"Saving: {MergedAndroidManifestOutput}"); } // Create additional runtime provider java sources. @@ -316,6 +304,88 @@ void Run (DirectoryAssemblyResolver res) template => template.Replace ("// REGISTER_APPLICATION_AND_INSTRUMENTATION_CLASSES_HERE", regCallsWriter.ToString ())); } + bool CreateJavaSources (IEnumerable javaTypes) + { + string outputPath = Path.Combine (OutputDirectory, "src"); + string monoInit = GetMonoInitSource (AndroidSdkPlatform, UseSharedRuntime); + bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); + bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; + + bool ok = true; + foreach (var t in javaTypes) { + using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { + try { + var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning) { + GenerateOnCreateOverrides = generateOnCreateOverrides, + ApplicationJavaClass = ApplicationJavaClass, + MonoRuntimeInitialization = monoInit, + }; + + jti.Generate (writer); + writer.Flush (); + + var path = jti.GetDestinationPath (outputPath); + MonoAndroidHelper.CopyIfStreamChanged (writer.BaseStream, path); + if (jti.HasExport && !hasExportReference) + Diagnostic.Error (4210, Properties.Resources.XA4210); + } catch (XamarinAndroidException xae) { + ok = false; + Log.LogError ( + subcategory: "", + errorCode: "XA" + xae.Code, + helpKeyword: string.Empty, + file: xae.SourceFile, + lineNumber: xae.SourceLine, + columnNumber: 0, + endLineNumber: 0, + endColumnNumber: 0, + message: xae.MessageWithoutCode, + messageArgs: new object [0] + ); + } catch (DirectoryNotFoundException ex) { + ok = false; + if (OS.IsWindows) { + Diagnostic.Error (5301, Properties.Resources.XA5301, t.FullName, ex); + } else { + Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); + } + } catch (Exception ex) { + ok = false; + Diagnostic.Error (4209, Properties.Resources.XA4209, t.FullName, ex); + } + } + } + return ok; + } + + static string GetMonoInitSource (string androidSdkPlatform, bool useSharedRuntime) + { + // Lookup the mono init section from MonoRuntimeProvider: + // Mono Runtime Initialization {{{ + // }}} + var builder = new StringBuilder (); + var runtime = useSharedRuntime ? "Shared" : "Bundled"; + var api = ""; + if (int.TryParse (androidSdkPlatform, out int apiLevel) && apiLevel < 21) { + api = ".20"; + } + var assembly = Assembly.GetExecutingAssembly (); + using (var s = assembly.GetManifestResourceStream ($"MonoRuntimeProvider.{runtime}{api}.java")) + using (var reader = new StreamReader (s)) { + bool copy = false; + string line; + while ((line = reader.ReadLine ()) != null) { + if (string.CompareOrdinal ("\t\t// Mono Runtime Initialization {{{", line) == 0) + copy = true; + if (copy) + builder.AppendLine (line); + if (string.CompareOrdinal ("\t\t// }}}", line) == 0) + break; + } + } + return builder.ToString (); + } + string GetResource (string resource) { using (var stream = GetType ().Assembly.GetManifestResourceStream (resource)) @@ -330,7 +400,7 @@ void SaveResource (string resource, string filename, string destDir, Func types) + void WriteTypeMappings (List types) { var tmg = new TypeMapGenerator ((string message) => Log.LogDebugMessage (message), SupportedAbis); if (!tmg.Generate (types, TypemapOutputDirectory, GenerateNativeAssembly)) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateLibraryResources.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateLibraryResources.cs index 564bbfeaf39..f1d13b35c8e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateLibraryResources.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateLibraryResources.cs @@ -114,8 +114,7 @@ void GenerateJava (Package package) } var lines = LoadValues (package); - using (var memory = new MemoryStream ()) - using (var writer = new StreamWriter (memory, Encoding)) { + using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { // This code is based on the Android gradle plugin // https://android.googlesource.com/platform/tools/base/+/908b391a9c006af569dfaff08b37f8fdd6c4da89/build-system/builder/src/main/java/com/android/builder/internal/SymbolWriter.java @@ -173,7 +172,7 @@ void GenerateJava (Package package) writer.Flush (); var r_java = Path.Combine (output_directory, package.Name.Replace ('.', Path.DirectorySeparatorChar), "R.java"); - if (MonoAndroidHelper.CopyIfStreamChanged (memory, r_java)) { + if (MonoAndroidHelper.CopyIfStreamChanged (writer.BaseStream, r_java)) { LogDebugMessage ($"Writing: {r_java}"); } else { LogDebugMessage ($"Up to date: {r_java}"); @@ -227,8 +226,6 @@ bool SetValue (string key, string[] line, string r_txt) } return false; } - - static readonly Encoding Encoding = new UTF8Encoding (encoderShouldEmitUTF8Identifier: false); static readonly char [] Delimiter = new [] { ' ' }; class Index diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs index 2764f083cc2..8a597557228 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GeneratePackageManagerJava.cs @@ -98,8 +98,7 @@ public override bool RunTask () Func fileNameEq = (a,b) => a.Equals (b, StringComparison.OrdinalIgnoreCase); assemblies = assemblies.Where (a => fileNameEq (a.ItemSpec, mainFileName)).Concat (assemblies.Where (a => !fileNameEq (a.ItemSpec, mainFileName))).ToList (); - using (var stream = new MemoryStream ()) - using (var pkgmgr = new StreamWriter (stream)) { + using (var pkgmgr = MemoryStreamPool.Shared.CreateStreamWriter ()) { pkgmgr.WriteLine ("package mono;"); // Write all the user assemblies @@ -133,7 +132,7 @@ public override bool RunTask () // Only copy to the real location if the contents actually changed var dest = Path.GetFullPath (Path.Combine (OutputDirectory, "MonoPackageManager_Resources.java")); - MonoAndroidHelper.CopyIfStreamChanged (stream, dest); + MonoAndroidHelper.CopyIfStreamChanged (pkgmgr.BaseStream, dest); } AddEnvironment (); @@ -250,52 +249,48 @@ void AddEnvironment () throw new InvalidOperationException ($"Unsupported BoundExceptionType value '{BoundExceptionType}'"); } - using (var ms = new MemoryStream ()) { - var utf8Encoding = new UTF8Encoding (false); - foreach (string abi in SupportedAbis) { - ms.SetLength (0); - NativeAssemblerTargetProvider asmTargetProvider; - string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}"); - string asmFilePath = $"{baseAsmFilePath}.s"; - switch (abi.Trim ()) { - case "armeabi-v7a": - asmTargetProvider = new ARMNativeAssemblerTargetProvider (false); - break; - - case "arm64-v8a": - asmTargetProvider = new ARMNativeAssemblerTargetProvider (true); - break; - - case "x86": - asmTargetProvider = new X86NativeAssemblerTargetProvider (false); - break; - - case "x86_64": - asmTargetProvider = new X86NativeAssemblerTargetProvider (true); - break; - - default: - throw new InvalidOperationException ($"Unknown ABI {abi}"); - } - - var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, baseAsmFilePath, environmentVariables, systemProperties) { - IsBundledApp = IsBundledApplication, - UsesMonoAOT = usesMonoAOT, - UsesMonoLLVM = EnableLLVM, - UsesAssemblyPreload = usesAssemblyPreload, - MonoAOTMode = aotMode.ToString ().ToLowerInvariant (), - AndroidPackageName = AndroidPackageName, - BrokenExceptionTransitions = brokenExceptionTransitions, - PackageNamingPolicy = pnp, - BoundExceptionType = boundExceptionType, - InstantRunEnabled = InstantRunEnabled, - }; - - using (var sw = new StreamWriter (ms, utf8Encoding, bufferSize: 8192, leaveOpen: true)) { - asmgen.Write (sw); - MonoAndroidHelper.CopyIfStreamChanged (ms, asmFilePath); - } + foreach (string abi in SupportedAbis) { + NativeAssemblerTargetProvider asmTargetProvider; + string baseAsmFilePath = Path.Combine (EnvironmentOutputDirectory, $"environment.{abi.ToLowerInvariant ()}"); + string asmFilePath = $"{baseAsmFilePath}.s"; + switch (abi.Trim ()) { + case "armeabi-v7a": + asmTargetProvider = new ARMNativeAssemblerTargetProvider (false); + break; + + case "arm64-v8a": + asmTargetProvider = new ARMNativeAssemblerTargetProvider (true); + break; + + case "x86": + asmTargetProvider = new X86NativeAssemblerTargetProvider (false); + break; + + case "x86_64": + asmTargetProvider = new X86NativeAssemblerTargetProvider (true); + break; + + default: + throw new InvalidOperationException ($"Unknown ABI {abi}"); + } + var asmgen = new ApplicationConfigNativeAssemblyGenerator (asmTargetProvider, baseAsmFilePath, environmentVariables, systemProperties) { + IsBundledApp = IsBundledApplication, + UsesMonoAOT = usesMonoAOT, + UsesMonoLLVM = EnableLLVM, + UsesAssemblyPreload = usesAssemblyPreload, + MonoAOTMode = aotMode.ToString ().ToLowerInvariant (), + AndroidPackageName = AndroidPackageName, + BrokenExceptionTransitions = brokenExceptionTransitions, + PackageNamingPolicy = pnp, + BoundExceptionType = boundExceptionType, + InstantRunEnabled = InstantRunEnabled, + }; + + using (var sw = MemoryStreamPool.Shared.CreateStreamWriter ()) { + asmgen.Write (sw); + sw.Flush (); + MonoAndroidHelper.CopyIfStreamChanged (sw.BaseStream, asmFilePath); } } diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/JavaCompileToolTask.cs b/src/Xamarin.Android.Build.Tasks/Tasks/JavaCompileToolTask.cs index 3813fc222bb..2cc66341dbd 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/JavaCompileToolTask.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/JavaCompileToolTask.cs @@ -61,7 +61,7 @@ private void GenerateResponseFile () TemporarySourceListFile = Path.GetTempFileName (); using (var sw = new StreamWriter (path:TemporarySourceListFile, append:false, - encoding:new UTF8Encoding (encoderShouldEmitUTF8Identifier:false))) { + encoding: MonoAndroidHelper.UTF8withoutBOM)) { WriteOptionsToResponseFile (sw); // Include any user .java files diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/ManifestMerger.cs b/src/Xamarin.Android.Build.Tasks/Tasks/ManifestMerger.cs index ca2f6459eab..acc7b3cea2c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/ManifestMerger.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/ManifestMerger.cs @@ -40,10 +40,13 @@ public override bool Execute () if (!result) return result; var m = new ManifestDocument (tempFile); - using (var ms = new MemoryStream ()) { + var ms = MemoryStreamPool.Shared.Rent (); + try { m.Save (Log, ms); MonoAndroidHelper.CopyIfStreamChanged (ms, OutputManifestFile); return result; + } finally { + MemoryStreamPool.Shared.Return (ms); } } finally { if (File.Exists (tempFile)) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index 4dfb630351d..9d028c757cf 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -13,6 +13,7 @@ using Microsoft.Build.Framework; using Mono.Cecil; using NUnit.Framework; +using Xamarin.Android.Tasks; using Xamarin.Android.Tools; using Xamarin.ProjectTools; @@ -3507,8 +3508,7 @@ public override void OnReceive(Context context, Intent intent) { } rules.Add ("-dontwarn java.lang.invoke.LambdaMetafactory"); } //FIXME: We aren't de-BOM'ing proguard files? - var encoding = new UTF8Encoding (encoderShouldEmitUTF8Identifier: false); - var bytes = encoding.GetBytes (string.Join (Environment.NewLine, rules)); + var bytes = MonoAndroidHelper.UTF8withoutBOM.GetBytes (string.Join (Environment.NewLine, rules)); proj.OtherBuildItems.Add (new BuildItem ("ProguardConfiguration", "okhttp3.pro") { BinaryContent = () => bytes, }); diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MemoryStreamPoolTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MemoryStreamPoolTests.cs new file mode 100644 index 00000000000..48086af18d7 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MemoryStreamPoolTests.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NUnit.Framework; +using Xamarin.Android.Tasks; + +namespace Xamarin.Android.Build.Tests.Utilities +{ + [TestFixture] + public class MemoryStreamPoolTests + { + MemoryStreamPool pool; + + [SetUp] + public void SetUp () + { + pool = new MemoryStreamPool (); + } + + [Test] + public void Reuse () + { + var expected = pool.Rent (); + expected.Write (new byte [] { 1, 2, 3 }, 0, 3); + pool.Return (expected); + var actual = pool.Rent (); + Assert.AreSame (expected, actual); + Assert.AreEqual (0, actual.Length); + } + + [Test] + public void PutDisposed () + { + var stream = new MemoryStream (); + stream.Dispose (); + Assert.Throws (() => pool.Return (stream)); + } + + [Test] + public void CreateStreamWriter () + { + var pool = new MemoryStreamPool (); + var expected = pool.Rent (); + using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { + writer.WriteLine ("foobar"); + } + pool.Return (expected); + + var actual = pool.Rent (); + Assert.AreSame (expected, actual); + Assert.AreEqual (0, actual.Length); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MonoAndroidHelperTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MonoAndroidHelperTests.cs index 12df8ad34ca..fc4ceac3b68 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MonoAndroidHelperTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/MonoAndroidHelperTests.cs @@ -163,5 +163,25 @@ public void CleanBOM_Readonly () var preamble = encoding.GetPreamble (); Assert.AreEqual (before.Length, after.Length + preamble.Length, "BOM should be removed!"); } + + [Test] + public void CopyIfStreamChanged_MemoryStreamPool () + { + var pool = new MemoryStreamPool (); + var expected = pool.Rent (); + pool.Return (expected); + + using (var writer = pool.CreateStreamWriter ()) { + writer.WriteLine ("bar"); + writer.Flush (); + + Assert.IsTrue (MonoAndroidHelper.CopyIfStreamChanged (writer.BaseStream, temp), "Should write on new file."); + FileAssert.Exists (temp); + } + + var actual = pool.Rent (); + Assert.AreSame (expected, actual); + Assert.AreEqual (0, actual.Length); + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/Files.cs b/src/Xamarin.Android.Build.Tasks/Utilities/Files.cs index d4b9d505923..a02d5dda92d 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/Files.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/Files.cs @@ -320,8 +320,9 @@ public static bool ExtractAll (ZipArchive zip, string destination, Action files = new HashSet (); - using (var memoryStream = new MemoryStream ()) { + var files = new HashSet (); + var memoryStream = MemoryStreamPool.Shared.Rent (); + try { foreach (var entry in zip) { progressCallback?.Invoke (i++, total); if (entry.IsDirectory) @@ -341,6 +342,8 @@ public static bool ExtractAll (ZipArchive zip, string destination, Action subclasses, i } } + public bool SaveIfChanged (TaskLoggingHelper log, string filename) + { + MemoryStream stream = MemoryStreamPool.Shared.Rent (); + try { + Save (log, stream); + return MonoAndroidHelper.CopyIfStreamChanged (stream, filename); + } finally { + MemoryStreamPool.Shared.Return (stream); + } + } + public void Save (TaskLoggingHelper log, string filename) => Save (m => log.LogWarning (m), filename); public void Save (Action logWarning, string filename) { - using (var file = new StreamWriter (filename, append: false, encoding: new UTF8Encoding (false))) + using (var file = new StreamWriter (filename, append: false, encoding: MonoAndroidHelper.UTF8withoutBOM)) Save (logWarning, file); } @@ -895,18 +906,23 @@ public void Save (TaskLoggingHelper log, Stream stream) => public void Save (Action logWarning, Stream stream) { - using (var file = new StreamWriter (stream, new UTF8Encoding (false), bufferSize: 1024, leaveOpen: true)) + using (var file = new StreamWriter (stream, MonoAndroidHelper.UTF8withoutBOM, bufferSize: 1024, leaveOpen: true)) Save (logWarning, file); } public void Save (Action logWarning, TextWriter stream) { RemoveDuplicateElements (); - var ms = new MemoryStream (); - doc.Save (ms); - ms.Flush (); - ms.Position = 0; - var s = new StreamReader (ms).ReadToEnd (); + string s; + var ms = MemoryStreamPool.Shared.Rent (); + try { + doc.Save (ms); + ms.Flush (); + ms.Position = 0; + s = new StreamReader (ms).ReadToEnd (); + } finally { + MemoryStreamPool.Shared.Return (ms); + } if (ApplicationName != null) s = s.Replace ("${applicationId}", ApplicationName); if (Placeholders != null) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MemoryStreamPool.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MemoryStreamPool.cs new file mode 100644 index 00000000000..668612bff21 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MemoryStreamPool.cs @@ -0,0 +1,64 @@ +using System.IO; +using System.Text; + +namespace Xamarin.Android.Tasks +{ + /// + /// A class for pooling and reusing MemoryStream objects. + /// + /// Based on: + /// https://docs.microsoft.com/dotnet/standard/collections/thread-safe/how-to-create-an-object-pool + /// https://docs.microsoft.com/dotnet/api/system.buffers.arraypool-1 + /// + class MemoryStreamPool : ObjectPool + { + /// + /// Static instance across the entire process. Use this most of the time. + /// + public static readonly MemoryStreamPool Shared = new MemoryStreamPool (); + + public MemoryStreamPool () : base (() => new MemoryStream ()) { } + + public override void Return (MemoryStream stream) + { + // We want to throw here before base.Return() if it was disposed + stream.SetLength (0); + base.Return (stream); + } + + /// + /// Creates a StreamWriter that uses the underlying MemoryStreamPool. Calling Dispose() will Return() the MemoryStream. + /// By default uses MonoAndroidHelper.UTF8withoutBOM for the encoding. + /// + public StreamWriter CreateStreamWriter () => CreateStreamWriter (MonoAndroidHelper.UTF8withoutBOM); + + /// + /// Creates a StreamWriter that uses the underlying MemoryStreamPool. Calling Dispose() will Return() the MemoryStream. + /// + public StreamWriter CreateStreamWriter (Encoding encoding) => new ReturningWriter (this, Rent (), encoding); + + class ReturningWriter : StreamWriter + { + readonly MemoryStreamPool pool; + readonly MemoryStream stream; + bool returned; + + public ReturningWriter (MemoryStreamPool pool, MemoryStream stream, Encoding encoding) : base (stream, encoding, bufferSize: 8 * 1024, leaveOpen: true) + { + this.pool = pool; + this.stream = stream; + } + + protected override void Dispose (bool disposing) + { + base.Dispose (disposing); + + //NOTE: Dispose() can be called multiple times on a StreamWriter + if (disposing && !returned) { + returned = true; + pool.Return (stream); + } + } + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 0cc1af3b8d8..d66a9acadf8 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -6,6 +6,7 @@ using System.Reflection.Metadata; using System.Reflection.PortableExecutable; using System.Security.Cryptography; +using System.Text; using Xamarin.Android.Tools; using Xamarin.Tools.Zip; @@ -26,7 +27,8 @@ public class MonoAndroidHelper public static AndroidVersions SupportedVersions; public static AndroidSdkInfo AndroidSdk; - readonly static byte[] Utf8Preamble = System.Text.Encoding.UTF8.GetPreamble (); + public static readonly Encoding UTF8withoutBOM = new UTF8Encoding (encoderShouldEmitUTF8Identifier: false); + readonly static byte[] Utf8Preamble = Encoding.UTF8.GetPreamble (); public static int RunProcess (string name, string args, DataReceivedEventHandler onOutput, DataReceivedEventHandler onError, Dictionary environmentVariables = null) { @@ -589,14 +591,13 @@ public static Dictionary> LoadCustomViewMapFile (IBuildE public static bool SaveCustomViewMapFile (IBuildEngine4 engine, string mapFile, Dictionary> map) { engine?.RegisterTaskObject (mapFile, map, RegisteredTaskObjectLifetime.Build, allowEarlyCollection: false); - using (var stream = new MemoryStream ()) - using (var writer = new StreamWriter (stream)) { + using (var writer = MemoryStreamPool.Shared.CreateStreamWriter (Encoding.Default)) { foreach (var i in map.OrderBy (x => x.Key)) { foreach (var v in i.Value.OrderBy (x => x)) writer.WriteLine ($"{i.Key};{v}"); } writer.Flush (); - return CopyIfStreamChanged (stream, mapFile); + return CopyIfStreamChanged (writer.BaseStream, mapFile); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ObjectPool.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ObjectPool.cs new file mode 100644 index 00000000000..0a079145873 --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ObjectPool.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Concurrent; + +namespace Xamarin.Android.Tasks +{ + /// + /// A class for pooling and reusing objects. See MemoryStreamPool. + /// + /// Based on: + /// https://docs.microsoft.com/dotnet/standard/collections/thread-safe/how-to-create-an-object-pool + /// https://docs.microsoft.com/dotnet/api/system.buffers.arraypool-1 + /// + class ObjectPool + { + readonly ConcurrentBag bag = new ConcurrentBag(); + readonly Func generator; + + public ObjectPool (Func generator) + { + if (generator == null) + throw new ArgumentNullException (nameof (generator)); + this.generator = generator; + } + + public virtual T Rent () + { + if (bag.TryTake (out T item)) + return item; + return generator (); + } + + public virtual void Return (T item) + { + bag.Add (item); + } + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs index f397f57019c..3d75ed2b657 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMapGenerator.cs @@ -82,7 +82,7 @@ public TypeMapGenerator (Action logger, string[] supportedAbis) throw new ArgumentNullException (nameof (supportedAbis)); this.supportedAbis = supportedAbis; - outputEncoding = new UTF8Encoding (false); + outputEncoding = MonoAndroidHelper.UTF8withoutBOM; moduleMagicString = outputEncoding.GetBytes (TypeMapMagicString); typemapIndexMagicString = outputEncoding.GetBytes (TypeMapIndexMagicString); } @@ -240,14 +240,12 @@ public bool Generate (List javaTypes, string outputDirectory, bo var generator = new TypeMappingNativeAssemblyGenerator (asmTargetProvider, data, Path.Combine (outputDirectory, "typemaps"), sharedBitsWritten, sharedIncludeUsesAbiPrefix); - using (var ms = new MemoryStream ()) { - using (var sw = new StreamWriter (ms, outputEncoding)) { - generator.Write (sw); - sw.Flush (); - MonoAndroidHelper.CopyIfStreamChanged (ms, generator.MainSourceFile); - if (!sharedIncludeUsesAbiPrefix) - sharedBitsWritten = true; - } + using (var sw = MemoryStreamPool.Shared.CreateStreamWriter (outputEncoding)) { + generator.Write (sw); + sw.Flush (); + MonoAndroidHelper.CopyIfStreamChanged (sw.BaseStream, generator.MainSourceFile); + if (!sharedIncludeUsesAbiPrefix) + sharedBitsWritten = true; } } return true; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs index 790fe071af5..c94ec0af478 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingNativeAssemblyGenerator.cs @@ -52,22 +52,18 @@ protected override void WriteSymbols (StreamWriter output) output.WriteLine (); if (!sharedBitsWritten && haveAssemblyNames) { - using (var ms = new MemoryStream ()) { - using (var sharedOutput = new StreamWriter (ms, output.Encoding)) { - WriteAssemblyNames (sharedOutput); - sharedOutput.Flush (); - MonoAndroidHelper.CopyIfStreamChanged (ms, SharedIncludeFile); - } + using (var sharedOutput = MemoryStreamPool.Shared.CreateStreamWriter (output.Encoding)) { + WriteAssemblyNames (sharedOutput); + sharedOutput.Flush (); + MonoAndroidHelper.CopyIfStreamChanged (sharedOutput.BaseStream, SharedIncludeFile); } } if (haveModules) { - using (var ms = new MemoryStream ()) { - using (var mapOutput = new StreamWriter (ms, output.Encoding)) { - WriteMapModules (output, mapOutput, "map_modules"); - mapOutput.Flush (); - MonoAndroidHelper.CopyIfStreamChanged (ms, TypemapsIncludeFile); - } + using (var mapOutput = MemoryStreamPool.Shared.CreateStreamWriter (output.Encoding)) { + WriteMapModules (output, mapOutput, "map_modules"); + mapOutput.Flush (); + MonoAndroidHelper.CopyIfStreamChanged (mapOutput.BaseStream, TypemapsIncludeFile); } } else { WriteMapModules (output, null, "map_modules"); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/XDocumentExtensions.cs b/src/Xamarin.Android.Build.Tasks/Utilities/XDocumentExtensions.cs index 0f0273b0bfb..8098d460890 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/XDocumentExtensions.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/XDocumentExtensions.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Xml.XPath; using System.Xml.Linq; using Microsoft.Build.Framework; @@ -58,12 +59,11 @@ public static string ToFullString (this XElement element) public static bool SaveIfChanged (this XDocument document, string fileName) { - using (var stream = new MemoryStream ()) - using (var sw = new StreamWriter (stream)) + using (var sw = MemoryStreamPool.Shared.CreateStreamWriter (Encoding.Default)) using (var xw = new Monodroid.LinePreservedXmlWriter (sw)) { xw.WriteNode (document.CreateNavigator (), false); xw.Flush (); - return MonoAndroidHelper.CopyIfStreamChanged (stream, fileName); + return MonoAndroidHelper.CopyIfStreamChanged (sw.BaseStream, fileName); } } } diff --git a/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv b/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv index 4f19d360e8f..0269d98b09d 100644 --- a/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv +++ b/tests/msbuild-times-reference/MSBuildDeviceIntegration.csv @@ -2,14 +2,14 @@ # First non-comment row is human description of columns Test Name,Time in ms (int) # Data -Build_No_Changes,3350 -Build_CSharp_Change,4500 -Build_AndroidResource_Change,4250 -Build_AndroidManifest_Change,4500 -Build_Designer_Change,3750 +Build_No_Changes,3250 +Build_CSharp_Change,4450 +Build_AndroidResource_Change,4150 +Build_AndroidManifest_Change,4350 +Build_Designer_Change,3600 Build_JLO_Change,8700 -Build_CSProj_Change,9800 -Build_XAML_Change,9600 -Build_XAML_Change_RefAssembly,6350 -Install_CSharp_Change,6000 -Install_XAML_Change,7500 +Build_CSProj_Change,9500 +Build_XAML_Change,9400 +Build_XAML_Change_RefAssembly,6000 +Install_CSharp_Change,5500 +Install_XAML_Change,7000