Prefer XxHash128 in dotnet8

saguiitay · Sep 16, 2023 · 9485cf9 · 9485cf9
1 parent fdc49fd
commit 9485cf9
Show file tree

Hide file tree

Showing 7 changed files with 112 additions and 7 deletions.
diff --git a/...on.Benchmark/BenchmarkDotNet.Artifacts/results/DifferentHashes-report-github.md b/...on.Benchmark/BenchmarkDotNet.Artifacts/results/DifferentHashes-report-github.md
@@ -0,0 +1,29 @@
+```
+
+BenchmarkDotNet v0.13.8, Windows 11 (10.0.22621.2283/22H2/2022Update/SunValley2)
+Intel Core i7-8700 CPU 3.20GHz (Coffee Lake), 1 CPU, 12 logical and 6 physical cores
+.NET SDK 8.0.100-rc.1.23463.5
+  [Host] : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2
+  Core70 : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2
+  Core80 : .NET 8.0.0 (8.0.23.41904), X64 RyuJIT AVX2
+
+
+```
+| Method    | Job    | Runtime  | Bits | Mean       | Error    | StdDev   | Gen0       | Allocated  |
+|---------- |------- |--------- |----- |-----------:|---------:|---------:|-----------:|-----------:|
+| **Murmur3**   | **Core70** | **.NET 7.0** | **4**    | **2,145.8 ms** | **28.16 ms** | **24.96 ms** | **34000.0000** | **2441.41 MB** |
+| Fnv1A     | Core70 | .NET 7.0 | 4    | 1,062.2 ms | 10.78 ms |  9.55 ms | 10000.0000 |  762.95 MB |
+| XxHash64  | Core70 | .NET 7.0 | 4    |   861.5 ms |  7.19 ms |  6.38 ms | 13000.0000 | 1068.12 MB |
+| XxHash128 | Core70 | .NET 7.0 | 4    |   782.3 ms | 11.13 ms |  9.29 ms | 15000.0000 | 1144.42 MB |
+| Murmur3   | Core80 | .NET 8.0 | 4    | 2,007.7 ms | 38.95 ms | 52.00 ms | 27000.0000 | 2441.41 MB |
+| Fnv1A     | Core80 | .NET 8.0 | 4    | 1,013.4 ms |  6.26 ms |  5.85 ms |  8000.0000 |  762.95 MB |
+| XxHash64  | Core80 | .NET 8.0 | 4    |   869.7 ms |  6.63 ms |  5.88 ms | 12000.0000 | 1068.12 MB |
+| XxHash128 | Core80 | .NET 8.0 | 4    |   779.6 ms |  7.87 ms |  7.36 ms | 13000.0000 | 1144.42 MB |
+| **Murmur3**   | **Core70** | **.NET 7.0** | **16**   | **2,177.4 ms** | **42.68 ms** | **70.12 ms** | **34000.0000** | **2441.72 MB** |
+| Fnv1A     | Core70 | .NET 7.0 | 16   | 1,121.2 ms | 10.51 ms |  9.32 ms | 10000.0000 |  763.26 MB |
+| XxHash64  | Core70 | .NET 7.0 | 16   |   909.6 ms | 17.82 ms | 28.27 ms | 14000.0000 | 1068.43 MB |
+| XxHash128 | Core70 | .NET 7.0 | 16   |   825.1 ms | 16.01 ms | 22.96 ms | 15000.0000 | 1144.73 MB |
+| Murmur3   | Core80 | .NET 8.0 | 16   | 2,094.1 ms | 31.54 ms | 27.96 ms | 27000.0000 | 2441.72 MB |
+| Fnv1A     | Core80 | .NET 8.0 | 16   | 1,098.3 ms | 21.94 ms | 23.47 ms |  8000.0000 |  763.26 MB |
+| XxHash64  | Core80 | .NET 8.0 | 16   |   903.3 ms | 16.75 ms | 17.20 ms | 12000.0000 | 1068.43 MB |
+| XxHash128 | Core80 | .NET 8.0 | 16   |   835.4 ms | 15.32 ms | 12.79 ms | 13000.0000 | 1144.73 MB |
diff --git a/CardinalityEstimation.Benchmark/BenchmarkDotNet.Artifacts/results/DifferentHashes-report.csv b/CardinalityEstimation.Benchmark/BenchmarkDotNet.Artifacts/results/DifferentHashes-report.csv
@@ -0,0 +1,17 @@
+Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Bits,Mean,Error,StdDev,Gen0,Allocated
+Murmur3,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,"2,145.8 ms",28.16 ms,24.96 ms,34000.0000,2441.41 MB
+Fnv1A,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,"1,062.2 ms",10.78 ms,9.55 ms,10000.0000,762.95 MB
+XxHash64,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,861.5 ms,7.19 ms,6.38 ms,13000.0000,1068.12 MB
+XxHash128,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,782.3 ms,11.13 ms,9.29 ms,15000.0000,1144.42 MB
+Murmur3,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,"2,007.7 ms",38.95 ms,52.00 ms,27000.0000,2441.41 MB
+Fnv1A,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,"1,013.4 ms",6.26 ms,5.85 ms,8000.0000,762.95 MB
+XxHash64,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,869.7 ms,6.63 ms,5.88 ms,12000.0000,1068.12 MB
+XxHash128,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,4,779.6 ms,7.87 ms,7.36 ms,13000.0000,1144.42 MB
+Murmur3,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,"2,177.4 ms",42.68 ms,70.12 ms,34000.0000,2441.72 MB
+Fnv1A,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,"1,121.2 ms",10.51 ms,9.32 ms,10000.0000,763.26 MB
+XxHash64,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,909.6 ms,17.82 ms,28.27 ms,14000.0000,1068.43 MB
+XxHash128,Core70,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 7.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,825.1 ms,16.01 ms,22.96 ms,15000.0000,1144.73 MB
+Murmur3,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,"2,094.1 ms",31.54 ms,27.96 ms,27000.0000,2441.72 MB
+Fnv1A,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,"1,098.3 ms",21.94 ms,23.47 ms,8000.0000,763.26 MB
+XxHash64,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,903.3 ms,16.75 ms,17.20 ms,12000.0000,1068.43 MB
+XxHash128,Core80,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,True,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,16,835.4 ms,15.32 ms,12.79 ms,13000.0000,1144.73 MB
diff --git a/...inalityEstimation.Benchmark/BenchmarkDotNet.Artifacts/results/DifferentHashes-report.html b/...inalityEstimation.Benchmark/BenchmarkDotNet.Artifacts/results/DifferentHashes-report.html
@@ -0,0 +1,46 @@
+<!DOCTYPE html>
+<html lang='en'>
+<head>
+<meta charset='utf-8' />
+<title>DifferentHashes-20230916-162923</title>
+
+<style type="text/css">
+	table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
+	td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
+	tr { background-color: #fff; border-top: 1px solid #ccc; }
+	tr:nth-child(even) { background: #f8f8f8; }
+</style>
+</head>
+<body>
+<pre><code>
+BenchmarkDotNet v0.13.8, Windows 11 (10.0.22621.2283/22H2/2022Update/SunValley2)
+Intel Core i7-8700 CPU 3.20GHz (Coffee Lake), 1 CPU, 12 logical and 6 physical cores
+.NET SDK 8.0.100-rc.1.23463.5
+  [Host] : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2
+  Core70 : .NET 7.0.11 (7.0.1123.42427), X64 RyuJIT AVX2
+  Core80 : .NET 8.0.0 (8.0.23.41904), X64 RyuJIT AVX2
+</code></pre>
+<pre><code></code></pre>
+
+<table>
+<thead><tr><th>Method</th><th>Job</th><th>Runtime</th><th>Bits</th><th>Mean</th><th>Error</th><th>StdDev</th><th>Gen0</th><th>Allocated</th>
+</tr>
+</thead><tbody><tr><td>Murmur3</td><td>Core70</td><td>.NET 7.0</td><td>4</td><td>2,145.8 ms</td><td>28.16 ms</td><td>24.96 ms</td><td>34000.0000</td><td>2441.41 MB</td>
+</tr><tr><td>Fnv1A</td><td>Core70</td><td>.NET 7.0</td><td>4</td><td>1,062.2 ms</td><td>10.78 ms</td><td>9.55 ms</td><td>10000.0000</td><td>762.95 MB</td>
+</tr><tr><td>XxHash64</td><td>Core70</td><td>.NET 7.0</td><td>4</td><td>861.5 ms</td><td>7.19 ms</td><td>6.38 ms</td><td>13000.0000</td><td>1068.12 MB</td>
+</tr><tr><td>XxHash128</td><td>Core70</td><td>.NET 7.0</td><td>4</td><td>782.3 ms</td><td>11.13 ms</td><td>9.29 ms</td><td>15000.0000</td><td>1144.42 MB</td>
+</tr><tr><td>Murmur3</td><td>Core80</td><td>.NET 8.0</td><td>4</td><td>2,007.7 ms</td><td>38.95 ms</td><td>52.00 ms</td><td>27000.0000</td><td>2441.41 MB</td>
+</tr><tr><td>Fnv1A</td><td>Core80</td><td>.NET 8.0</td><td>4</td><td>1,013.4 ms</td><td>6.26 ms</td><td>5.85 ms</td><td>8000.0000</td><td>762.95 MB</td>
+</tr><tr><td>XxHash64</td><td>Core80</td><td>.NET 8.0</td><td>4</td><td>869.7 ms</td><td>6.63 ms</td><td>5.88 ms</td><td>12000.0000</td><td>1068.12 MB</td>
+</tr><tr><td>XxHash128</td><td>Core80</td><td>.NET 8.0</td><td>4</td><td>779.6 ms</td><td>7.87 ms</td><td>7.36 ms</td><td>13000.0000</td><td>1144.42 MB</td>
+</tr><tr><td>Murmur3</td><td>Core70</td><td>.NET 7.0</td><td>16</td><td>2,177.4 ms</td><td>42.68 ms</td><td>70.12 ms</td><td>34000.0000</td><td>2441.72 MB</td>
+</tr><tr><td>Fnv1A</td><td>Core70</td><td>.NET 7.0</td><td>16</td><td>1,121.2 ms</td><td>10.51 ms</td><td>9.32 ms</td><td>10000.0000</td><td>763.26 MB</td>
+</tr><tr><td>XxHash64</td><td>Core70</td><td>.NET 7.0</td><td>16</td><td>909.6 ms</td><td>17.82 ms</td><td>28.27 ms</td><td>14000.0000</td><td>1068.43 MB</td>
+</tr><tr><td>XxHash128</td><td>Core70</td><td>.NET 7.0</td><td>16</td><td>825.1 ms</td><td>16.01 ms</td><td>22.96 ms</td><td>15000.0000</td><td>1144.73 MB</td>
+</tr><tr><td>Murmur3</td><td>Core80</td><td>.NET 8.0</td><td>16</td><td>2,094.1 ms</td><td>31.54 ms</td><td>27.96 ms</td><td>27000.0000</td><td>2441.72 MB</td>
+</tr><tr><td>Fnv1A</td><td>Core80</td><td>.NET 8.0</td><td>16</td><td>1,098.3 ms</td><td>21.94 ms</td><td>23.47 ms</td><td>8000.0000</td><td>763.26 MB</td>
+</tr><tr><td>XxHash64</td><td>Core80</td><td>.NET 8.0</td><td>16</td><td>903.3 ms</td><td>16.75 ms</td><td>17.20 ms</td><td>12000.0000</td><td>1068.43 MB</td>
+</tr><tr><td>XxHash128</td><td>Core80</td><td>.NET 8.0</td><td>16</td><td>835.4 ms</td><td>15.32 ms</td><td>12.79 ms</td><td>13000.0000</td><td>1144.73 MB</td>
+</tr></tbody></table>
+</body>
+</html>
diff --git a/CardinalityEstimation.Benchmark/CardinalityEstimation.Benchmark.csproj b/CardinalityEstimation.Benchmark/CardinalityEstimation.Benchmark.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>net8.0;net7.0</TargetFrameworks>
+    <TargetFrameworks>net8.0;net7.0;net6.0</TargetFrameworks>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <ServerGarbageCollection>true</ServerGarbageCollection>
@@ -11,7 +11,7 @@
 
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.13.8" />
-    <PackageReference Include="System.IO.Hashing" Version="7.0.0" />
+    <PackageReference Include="System.IO.Hashing" Version="8.0.0-rc.1.23419.4" />
   </ItemGroup>
 
   <ItemGroup>

diff --git a/CardinalityEstimation.Benchmark/Program.cs b/CardinalityEstimation.Benchmark/Program.cs
@@ -16,7 +16,7 @@ public class DifferentHashes
 {
     public static readonly Random Rand = new Random();
 
-    private const int N = 1000;
+    private const int N = 10000000;
 
     private string[] dataStrings = Enumerable.Range(0, N).Select(_ => Rand.Next().ToString() + Guid.NewGuid().ToString() + Rand.Next().ToString()).ToArray();
 
@@ -28,7 +28,7 @@ public class DifferentHashes
     [Benchmark]
     public void Fnv1A() => Run(Bits, CardinalityEstimation.Hash.Fnv1A.GetHashCode);
     [Benchmark]
-    public void XxHash64() => Run(Bits, (x) => BitConverter.ToUInt64(System.IO.Hashing.XxHash64.Hash(x)));
+    public void XxHash128() => Run(Bits, (x) => BitConverter.ToUInt64(System.IO.Hashing.XxHash128.Hash(x)));
 
 
     private void Run(int bits, GetHashCodeDelegate hashFunction)
@@ -46,7 +46,7 @@ public class GetBytesTests
 {
     public static readonly Random Rand = new Random();
 
-    private const int N = 1000;
+    private const int N = 10000000;
 
     private int[] dataInts = Enumerable.Range(0, N).Select(_ => Rand.Next()).ToArray();
 

diff --git a/CardinalityEstimation/CardinalityEstimation.csproj b/CardinalityEstimation/CardinalityEstimation.csproj
@@ -31,6 +31,11 @@
         <SignAssembly>true</SignAssembly>
     </PropertyGroup>
 
+    <ItemGroup >
+        <PackageReference Include="System.IO.Hashing" Version="7.0.0" Condition="'$(TargetFramework)' == 'net7.0'"/>
+        <PackageReference Include="System.IO.Hashing" Version="8.0.0-rc.1.23419.4" Condition="'$(TargetFramework)' == 'net8.0'"/>
+    </ItemGroup>
+
     <ItemGroup Condition=" '$(Configuration)' == 'Release-Signed' ">
         <PackageReference Include="murmurhash-signed" Version="1.0.3" />
     </ItemGroup>

diff --git a/CardinalityEstimation/CardinalityEstimator.cs b/CardinalityEstimation/CardinalityEstimator.cs
@@ -179,7 +179,15 @@ internal CardinalityEstimator(GetHashCodeDelegate hashFunction, CardinalityEstim
             subAlgorithmSelectionThreshold = GetSubAlgorithmSelectionThreshold(bitsPerIndex);
 
             // Init the hash function
-            this.hashFunction = hashFunction ?? Murmur3.GetHashCode;
+            this.hashFunction = hashFunction;
+            if (this.hashFunction == null)
+            {
+#if NET8_0_OR_GREATER
+                hashFunction = (x) => BitConverter.ToUInt64(System.IO.Hashing.XxHash128.Hash(x));
+#else
+                hashFunction = Murmur3.GetHashCode;
+#endif
+            }
 
             // Init the direct count
             directCount = state.DirectCount != null ? new HashSet<ulong>(state.DirectCount) : null;
@@ -214,7 +222,7 @@ internal CardinalityEstimator(GetHashCodeDelegate hashFunction, CardinalityEstim
                 directCount = null;
             }
         }
-        #endregion
+#endregion
 
         #region Public properties
         public ulong CountAdditions { get; private set; }