Serializer benchmarks

src/benchmarks/Benchmarks.csproj

@@ -6,10 +6,17 @@
     <PlatformTarget>AnyCPU</PlatformTarget>
     <DebugType>pdbonly</DebugType>
     <DebugSymbols>true</DebugSymbols>
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.10.14.514" />
+    <Compile Remove="img\**" />
+    <EmbeddedResource Remove="img\**" />
+    <None Remove="img\**" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="BenchmarkDotNet" Version="0.10.14.516" />
     <PackageReference Include="CommandLineParser" Version="2.2.1" />
     <PackageReference Include="Jil" Version="2.15.4" />
     <PackageReference Include="MessagePack" Version="1.7.3.4" />

src/benchmarks/Program.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
+using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Diagnosers;
@@ -23,15 +24,12 @@ class Program
     {
         static void Main(string[] args)
             => Parser.Default.ParseArguments<Options>(args)
-                .WithParsed(RunOptionsAndReturnExitCode)
+                .WithParsed(RunBenchmarks)
                 .WithNotParsed(errors => { }); // ignore the errors, the parser prints nice error message
 
-        private static void RunOptionsAndReturnExitCode(Options options)
+        private static void RunBenchmarks(Options options)
             => BenchmarkSwitcher
-                .FromTypes
-                (
-                    SerializerBenchmarks.GetTypes()
-                )
+                .FromAssemblyAndTypes(typeof(Program).Assembly, SerializerBenchmarks.GetTypes())
                 .Run(config: GetConfig(options));
 
         private static IConfig GetConfig(Options options)
@@ -71,7 +69,17 @@ private static IEnumerable<Job> GetJobs(Options options, Job baseJob)
             if (options.RunCoreRt)
                 yield return baseJob.With(Runtime.CoreRT).With(CoreRtToolchain.LatestMyGetBuild);
             if (!string.IsNullOrEmpty(options.CoreRtVersion))
-                yield return baseJob.With(Runtime.CoreRT).With(CoreRtToolchain.CreateBuilder().UseCoreRtNuGet(options.CoreRtVersion).ToToolchain());
+                yield return baseJob.With(Runtime.CoreRT)
+                    .With(CoreRtToolchain.CreateBuilder()
+                        .UseCoreRtNuGet(options.CoreRtVersion)
+                        .AdditionalNuGetFeed("benchmarkdotnet ci", "https://ci.appveyor.com/nuget/benchmarkdotnet")
+                        .ToToolchain());
+            if (!string.IsNullOrEmpty(options.CoreRtPath))
+                yield return baseJob.With(Runtime.CoreRT)
+                    .With(CoreRtToolchain.CreateBuilder()
+                        .UseCoreRtLocal(options.CoreRtPath)
+                        .AdditionalNuGetFeed("benchmarkdotnet ci", "https://ci.appveyor.com/nuget/benchmarkdotnet")
+                        .ToToolchain());
 
             if (options.RunCore)
                 yield return baseJob.With(Runtime.Core).With(CsProjCoreToolchain.Current.Value);
@@ -101,6 +109,8 @@ private static IEnumerable<Job> GetJobs(Options options, Job baseJob)
                 if (!string.IsNullOrEmpty(options.CliPath))
                     builder.DotNetCli(options.CliPath);
 
+                builder.AdditionalNuGetFeed("benchmarkdotnet ci", "https://ci.appveyor.com/nuget/benchmarkdotnet");
+
                 yield return baseJob.With(Runtime.Core).With(builder.ToToolchain());
             }
         }
@@ -138,6 +148,9 @@ public class Options
         [Option("coreRtVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with CoreRT. Example: \"1.0.0-alpha-26414-01\"")]
         public string CoreRtVersion { get; set; }
 
+        [Option("ilcPath", Required = false, HelpText = "Optional IlcPath which should be used to run with private CoreRT build. Example: \"1.0.0-alpha-26414-01\"")]
+        public string CoreRtPath { get; set; }
+
         [Option("core", Required = false, Default = false, HelpText = "Run benchmarks for .NET Core")]
         public bool RunCore { get; set; }
 

src/benchmarks/README.md

@@ -0,0 +1,204 @@
+# Benchmarks
+
+This repo contains various .NET benchmarks. It uses BenchmarkDotNet as the benchmarking engine to run benchmarks for .NET, .NET Core, CoreRT and Mono. Including private runtime builds.
+
+## BenchmarkDotNet
+
+Benchmarking is really hard (especially microbenchmarking), you can easily make a mistake during performance measurements.
+BenchmarkDotNet will protect you from the common pitfalls (even for experienced developers) because it does all the dirty work for you:
+
+* it generates an isolated project per runtime
+* it builds the project in `Release`
+* it runs every benchmark in a stand-alone process (to achieve process isolation and avoid side effects)
+* it estimates the perfect invocation count per iteration
+* it warms-up the code
+* it evaluates the overhead
+* it runs multiple iterations of the method until the requested level of precision is met.
+
+A few useful links for you:
+
+* If you want to know more about BenchmarkDotNet features, check out the [Overview Page](http://benchmarkdotnet.org/Overview.htm).
+* If you want to use BenchmarkDotNet for the first time, the [Getting Started](http://benchmarkdotnet.org/GettingStarted.htm) will help you.
+* If you want to ask a quick question or discuss performance topics, use the [gitter](https://gitter.im/dotnet/BenchmarkDotNet) channel.
+
+## Your first benchmark
+
+It's really easy to design a performance experiment with BenchmarkDotNet. Just mark your method with the `[Benchmark]` attribute and the benchmark is ready.
+
+```cs
+public class Simple
+{
+    [Benchmark]
+    public byte[] CreateByteArray() => new byte[8];
+}
+```
+
+Any public, non-generic type with public `[Benchmark]` method in this assembly will be auto-detected and added to the benchmarks list.
+
+## Running
+
+To run the benchmarks you have to execute `dotnet run -c Release -f net46|netcoreapp2.0|netcoreapp2.1` (choose one of the supported frameworks).
+
+![Choose Benchmark](./img/chooseBenchmark.png)
+
+And select one of the benchmarks from the list by either entering it's number or name. To **run all** the benchmarks simply enter `*` to the console.
+
+BenchmarkDotNet will build the executables, run the benchmarks, print the results to console and **export the results** to `.\BenchmarkDotNet.Artifacts\results`. 
+
+![Exported results](./img/exportedResults.png)
+
+BenchmarkDotNet by default exports the results to GitHub markdown, so you can just find the right `.md` file in `results` folder and copy-paste the markdown to GitHub.
+
+## All Statistics
+
+By default BenchmarkDotNet displays only `Mean`, `Error` and `StdDev` in the results. If you want to see more statistics, please pass `--allStats` as an extra argument to the app: `dotnet run -c Release -f netcoreapp2.1 -- --allStats`. If you build your own config, please use `config.With(StatisticColumn.AllStatistics)`.
+
+|   Method |     Mean |     Error |    StdDev |    StdErr |      Min |       Q1 |   Median |       Q3 |      Max |        Op/s |  Gen 0 | Allocated |
+|--------- |---------:|----------:|----------:|----------:|---------:|---------:|---------:|---------:|---------:|------------:|-------:|----------:|
+|      Jil | 458.2 ns |  38.63 ns |  2.183 ns |  1.260 ns | 455.8 ns | 455.8 ns | 458.9 ns | 460.0 ns | 460.0 ns | 2,182,387.2 | 0.1163 |     736 B |
+| JSON.NET | 869.8 ns |  47.37 ns |  2.677 ns |  1.545 ns | 867.7 ns | 867.7 ns | 868.8 ns | 872.8 ns | 872.8 ns | 1,149,736.0 | 0.2394 |    1512 B |
+| Utf8Json | 272.6 ns | 341.64 ns | 19.303 ns | 11.145 ns | 256.7 ns | 256.7 ns | 266.9 ns | 294.1 ns | 294.1 ns | 3,668,854.8 | 0.0300 |     192 B |
+
+## How to read the Memory Statistics
+
+The project is configured to include managed memory statistics by using [Memory Diagnoser](http://adamsitnik.com/the-new-Memory-Diagnoser/)
+
+|     Method |  Gen 0 | Allocated |
+|----------- |------- |---------- |
+|          A |      - |       0 B |
+|          B |      1 |     496 B |
+
+* Allocated contains the size of allocated **managed** memory. **Stackalloc/native heap allocations are not included.** It's per single invocation, **inclusive**.
+* The `Gen X` column contains the number of `Gen X` collections per ***1 000*** Operations. If the value is equal 1, then it means that GC collects memory once per one thousand of benchmark invocations in generation `X`. BenchmarkDotNet is using some heuristic when running benchmarks, so the number of invocations can be different for different runs. Scaling makes the results comparable.
+* `-` in the Gen column means that no garbage collection was performed.
+* If `Gen X` column is not present, then it means that no garbage collection was performed for generation `X`. If none of your benchmarks induces the GC, the Gen columns are not present.
+
+## How to get the Disassembly
+
+If you want to disassemble the benchmarked code, you need to use the [Disassembly Diagnoser](http://adamsitnik.com/Disassembly-Diagnoser/). It allows to disassemble `asm/C#/IL` in recursive way on Windows for .NET and .NET Core (all Jits) and `asm` for Mono on any OS.
+
+You can do that by passing `--disassm` to the app or by using `[DisassemblyDiagnoser(printAsm: true, printSource: true)]` attribute or by adding it to your config with `config.With(DisassemblyDiagnoser.Create(new DisassemblyDiagnoserConfig(printAsm: true, recursiveDepth: 1))`. 
+
+![Sample Disassm](./img/sampleDisassm.png)
+
+## How to run In Process
+
+If you want to run the benchmarks in process, without creating a dedicated executable and process-level isolation, please pass `--inProcess` as an extra argument to the app: `dotnet run -c Release -f netcoreapp2.1 -- --inProcess`. If you build your own config, please use `config.With(Job.Default.With(InProcessToolchain.Instance))`. Please use this option only when you are sure that the benchmarks you want to run have no side effects.
+
+## How to compare different Runtimes
+
+BenchmarkDotNet allows you to run benchmarks for multiple runtimes. By using this feature you can compare .NET vs .NET Core vs CoreRT vs Mono or .NET Core 2.0 vs .NET Core 2.1. BDN will compile and run the right stuff for you.
+
+* for .NET pass `--clr` to the app or use `Job.Default.With(Runtime.Clr)` in the code.
+* for .NET Core 2.0 pass `--core20` to the app or use `Job.Default.With(Runtime.Core).With(CsProjCoreToolchain.NetCoreApp20)` in the code.
+* for .NET Core 2.1 pass `--core21` to the app or use `Job.Default.With(Runtime.Core).With(CsProjCoreToolchain.NetCoreApp20)` in the code.
+* for the latest CoreRT pass `--coreRt` to the app or use `Job.Default.With(Runtime.CoreRT).With(CoreRtToolchain.LatestMyGetBuild)` in the code. **Be warned!** Downloading latest CoreRT with all the dependencies takes a lot of time. It is recommended to choose one version and use it for comparisions, more info [here](https://github.com/dotnet/BenchmarkDotNet/blob/600e5fa81bd8e7a1d32a60b2bea830e1f46106eb/docs/guide/Configs/Toolchains.md#corert). To use explicit CoreRT version please use `coreRtVersion` argument. Example: `dotnet run -c Release -f netcoreapp2.1 --coreRtVersion 1.0.0-alpha-26414-0`
+* for Mono pass `--mono` to the app or use `Job.Default.With(Runtime.Mono)` in the code.
+
+An example command for comparing 4 runtimes: `dotnet run -c Release -f netcoreapp2.1 -- --core20 --core21 --mono --clr --coreRt`
+
+``` ini
+BenchmarkDotNet=v0.10.14.516-nightly, OS=Windows 10.0.16299.309 (1709/FallCreatorsUpdate/Redstone3)
+Intel Xeon CPU E5-1650 v4 3.60GHz, 1 CPU, 12 logical and 6 physical cores
+Frequency=3507504 Hz, Resolution=285.1030 ns, Timer=TSC
+.NET Core SDK=2.1.300-preview1-008174
+  [Host]     : .NET Core 2.1.0-preview1-26216-03 (CoreCLR 4.6.26216.04, CoreFX 4.6.26216.02), 64bit RyuJIT
+  Job-GALXOG : .NET Framework 4.7.1 (CLR 4.0.30319.42000), 64bit RyuJIT-v4.7.2633.0
+  Job-DRRTOZ : .NET Core 2.0.6 (CoreCLR 4.6.26212.01, CoreFX 4.6.26212.01), 64bit RyuJIT
+  Job-QQFGIW : .NET Core 2.1.0-preview1-26216-03 (CoreCLR 4.6.26216.04, CoreFX 4.6.26216.02), 64bit RyuJIT
+  Job-GKRDGF : .NET CoreRT 1.0.26412.02, 64bit AOT
+  Job-HNFRHF : Mono 5.10.0 (Visual Studio), 64bit 
+
+LaunchCount=1  TargetCount=3  WarmupCount=3  
+```
+
+|   Method | Runtime |                    Toolchain |      Mean |      Error |    StdDev | Allocated |
+|--------- |-------- |----------------------------- |----------:|-----------:|----------:|----------:|
+| ParseInt |     Clr |                      Default |  95.95 ns |   5.354 ns | 0.3025 ns |       0 B |
+| ParseInt |    Core |                .NET Core 2.0 | 104.71 ns | 121.620 ns | 6.8718 ns |       0 B |
+| ParseInt |    Core |                .NET Core 2.1 |  93.16 ns |   6.383 ns | 0.3606 ns |       0 B |
+| ParseInt |  CoreRT | Core RT 1.0.0-alpha-26412-02 | 110.02 ns |  71.947 ns | 4.0651 ns |       0 B |
+| ParseInt |    Mono |                      Default | 133.19 ns | 133.928 ns | 7.5672 ns |       N/A |
+
+## .NET Core 2.0 vs .NET Core 2.1
+
+If you want to compare .NET Core 2.0 vs .NET Core 2.1 you can just pass `-- --core20 --core21`. You can also build a custom config and mark selected runtime as baseline, then all the results will be scaled to the baseline.
+
+```cs
+Add(Job.Default.With(Runtime.Core).With(CsProjCoreToolchain.From(NetCoreAppSettings.NetCoreApp20)).WithId("Core 2.0").AsBaseline());
+Add(Job.Default.With(Runtime.Core).With(CsProjCoreToolchain.From(NetCoreAppSettings.NetCoreApp21)).WithId("Core 2.1"));
+```
+
+|                Method |      Job |     Toolchain | IsBaseline |     Mean |     Error |    StdDev | Scaled |
+|---------------------- |--------- |-------------- |----------- |---------:|----------:|----------:|-------:|
+| CompactLoopBodyLayout | Core 2.0 | .NET Core 2.0 |       True | 36.72 ns | 0.1583 ns | 0.1481 ns |   1.00 |
+| CompactLoopBodyLayout | Core 2.1 | .NET Core 2.1 |    Default | 30.47 ns | 0.1731 ns | 0.1619 ns |   0.83 |
+
+## Benchmarking private CLR build
+
+It's possible to benchmark a private build of .NET Runtime. You just need to pass the value of `COMPLUS_Version` to BenchmarkDotNet. You can do that by either using `--clrVersion $theVersion` as an arugment or `Job.ShortRun.With(new ClrRuntime(version: "$theVersiong"))` in the code.
+
+So if you made a change in CLR and want to measure the difference, you can run the benchmarks with `dotnet run -c Release -f net46 --clr --clrVersion $theVersion`. More info can be found [here](https://github.com/dotnet/BenchmarkDotNet/issues/706).
+
+## Any CoreCLR and CoreFX
+
+BenchmarkDotNet allows the users to run their benchmarks against ANY CoreCLR and CoreFX builds. You can compare your local build vs MyGet feed or Debug vs Release or one version vs another.
+
+To avoid problems described [here](https://github.com/dotnet/coreclr/blob/master/Documentation/workflow/UsingDotNetCli.md#update-coreclr-using-runtime-nuget-package) a temporary folder is used when restoring packages for local builds. This is why it takes 20-30s in total to build the benchmarks. 
+
+Entire feature with many examples is described [here](https://github.com/dotnet/BenchmarkDotNet/blob/600e5fa81bd8e7a1d32a60b2bea830e1f46106eb/docs/guide/Configs/Toolchains.md#custom-coreclr-and-corefx).
+
+### Benchmarking private CoreFX build
+
+To run benchmarks with private CoreFX build you need to provide the version of `Microsoft.Private.CoreFx.NETCoreApp` and the path to folder with CoreFX NuGet packages.
+
+Sample arguments: `dotnet run -c Release -f netcoreapp2.1 -- --coreFxBin C:\Projects\forks\corefx\bin\packages\Release --coreFxVersion 4.5.0-preview2-26307-0`
+
+Sample config: 
+
+```cs
+Job.ShortRun.With(
+    CustomCoreClrToolchain.CreateBuilder()
+        .UseCoreFxLocalBuild("4.5.0-preview2-26313-0", @"C:\Projects\forks\corefx\bin\packages\Release")
+        .UseCoreClrDefault()
+        .AdditionalNuGetFeed("benchmarkdotnet ci", "https://ci.appveyor.com/nuget/benchmarkdotnet");
+        .DisplayName("local corefx")
+        .ToToolchain());
+```
+
+### Benchmarking private CoreCLR build
+
+To run benchmarks with private CoreCLR build you need to provide the version of `Microsoft.NETCore.Runtime`, path to folder with CoreCLR NuGet packages and path to `coreclr\packages` folder.
+
+Sample arguments: `dotnet run -c Release -f netcoreapp2.1 -- --coreClrBin C:\coreclr\bin\Product\Windows_NT.x64.Release\.nuget\pkg --coreClrPackages C:\Projects\coreclr\packages --coreClrVersion 2.1.0-preview2-26305-0`
+
+Sample config: 
+
+```cs
+Job.ShortRun.With(
+    CustomCoreClrToolchain.CreateBuilder()
+        .UseCoreClrLocalBuild("2.1.0-preview2-26313-0", @"C:\Projects\forks\coreclr\bin\Product\Windows_NT.x64.Release\.nuget\pkg", @"C:\Projects\coreclr\packages")
+        .UseCoreFxDefault()
+        .AdditionalNuGetFeed("benchmarkdotnet ci", "https://ci.appveyor.com/nuget/benchmarkdotnet");
+        .DisplayName("local builds")
+        .ToToolchain());
+```
+
+## Benchmarking private CoreRT build
+
+To run benchmarks with private CoreRT build you need to provide the `IlcPath`.
+
+Sample arguments: `dotnet run -c Release -f netcoreapp2.1 -- --ilcPath C:\Projects\corert\bin\Windows_NT.x64.Release`
+
+Sample config: 
+
+```cs
+var config = DefaultConfig.Instance
+    .With(Job.ShortRun
+        .With(Runtime.CoreRT)
+        .With(CoreRtToolchain.CreateBuilder()
+            .UseCoreRtLocal(@"C:\Projects\corert\bin\Windows_NT.x64.Release") // IlcPath
+            .DisplayName("Core RT RyuJit")
+            .ToToolchain()));
+```
+

src/benchmarks/Serializers/README.md

@@ -0,0 +1,34 @@
+# Serializers Benchmarks
+
+This folder contains benchmarks of the most popular serializers.
+
+## Serializers used (latest stable versions)
+
+* XML
+  * System.Xml.XmlSerializer `4.3.0`
+* JSON
+    * System.Runtime.Serialization.Json `4.3.0`
+    * [Jil](https://github.com/kevin-montrose/Jil) `2.15.4` 
+    * [JSON.NET](https://github.com/JamesNK/Newtonsoft.Json) `11.0.1` 
+    * [Utf8Json](https://github.com/neuecc/Utf8Json) `1.3.7` 
+* Binary
+    * BinaryFormatter
+    * [MessagePack](https://github.com/neuecc/MessagePack-CSharp) `1.7.3.4` 
+    * [protobuff-net](https://github.com/mgravell/protobuf-net) `2.3.7`
+    * [ZeroFormatter](https://github.com/neuecc/ZeroFormatter) `1.6.4` 
+
+Missing: ProtoBuff from Google and BOND from MS
+
+## Data Contracts
+
+Data Contracts were copied from a real Web App � [allReady](https://github.com/HTBox/allReady/) to mimic real world scenarios.
+
+* [LoginViewModel](DataGenerator.cs#L120) � class, 3 properties
+* [Location](DataGenerator.cs#L133) � class, 9 properties
+* [IndexViewModel](DataGenerator.cs#L202) � class, nested class + list of 20 Events (8 properties each)
+* [MyEventsListerViewModel](DataGenerator.cs#L224) - class, 3 lists of complex types, each type contains another list of complex types
+
+## Design Decisions
+
+1. We want to compare "apples to apples", so the benchmarks are divided into few groups: `ToStream`, `FromStream`, `ToString`, `FromString`.
+2. Stream benchmarks write to pre-allocated MemoryStream, so the allocated bytes columns include only the cost of serialization.