Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more benchmarks for V8 #1106

Merged
merged 1 commit into from
Apr 11, 2023
Merged

Add more benchmarks for V8 #1106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/Polly.Core.Benchmarks/Benchmarks/RateLimiterBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Polly.Core.Benchmarks;

namespace Polly.Benchmarks;

public class RateLimiterBenchmark
{
private object? _strategyV7;
private object? _strategyV8;

[GlobalSetup]
public void Setup()
{
_strategyV7 = Helper.CreateRateLimiter(PollyVersion.V7);
_strategyV8 = Helper.CreateRateLimiter(PollyVersion.V8);
}

[Benchmark(Baseline = true)]
public ValueTask ExecuteRateLimiter_V7() => _strategyV7!.ExecuteAsync(PollyVersion.V7);

[Benchmark]
public ValueTask ExecuteRateLimiter_V8() => _strategyV8!.ExecuteAsync(PollyVersion.V8);
}
24 changes: 24 additions & 0 deletions src/Polly.Core.Benchmarks/Benchmarks/RetryBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Polly.Core.Benchmarks;

namespace Polly.Benchmarks;

public class RetryBenchmark
{
private object? _strategyV7;
private object? _strategyV8;

[GlobalSetup]
public void Setup()
{
_strategyV7 = Helper.CreateRetry(PollyVersion.V7);
_strategyV8 = Helper.CreateRetry(PollyVersion.V8);
}

[Benchmark(Baseline = true)]
public ValueTask ExecuteRetry_V7() => _strategyV7!.ExecuteAsync(PollyVersion.V7);

[Benchmark]
public ValueTask ExecuteRetry_V8() => _strategyV8!.ExecuteAsync(PollyVersion.V8);
}
23 changes: 23 additions & 0 deletions src/Polly.Core.Benchmarks/Benchmarks/StrategyPipelineBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
using BenchmarkDotNet.Attributes;
using Polly;

namespace Polly.Core.Benchmarks;

public class StrategyPipelineBenchmark
{
private object? _strategyV7;
private object? _strategyV8;

[GlobalSetup]
public void Setup()
{
_strategyV7 = Helper.CreateStrategyPipeline(PollyVersion.V7);
_strategyV8 = Helper.CreateStrategyPipeline(PollyVersion.V8);
}

[Benchmark(Baseline = true)]
public ValueTask ExecuteStrategyPipeline_V7() => _strategyV7!.ExecuteAsync(PollyVersion.V7);

[Benchmark]
public ValueTask ExecuteStrategyPipeline_V8() => _strategyV8!.ExecuteAsync(PollyVersion.V8);
}
24 changes: 24 additions & 0 deletions src/Polly.Core.Benchmarks/Benchmarks/TimeoutBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
using System.Threading.Tasks;
using BenchmarkDotNet.Attributes;
using Polly.Core.Benchmarks;

namespace Polly.Benchmarks;

public class TimeoutBenchmark
{
private object? _strategyV7;
private object? _strategyV8;

[GlobalSetup]
public void Setup()
{
_strategyV7 = Helper.CreateTimeout(PollyVersion.V7);
_strategyV8 = Helper.CreateTimeout(PollyVersion.V8);
}

[Benchmark(Baseline = true)]
public ValueTask ExecuteTimeout_V7() => _strategyV7!.ExecuteAsync(PollyVersion.V7);

[Benchmark]
public ValueTask ExecuteTimeout_V8() => _strategyV8!.ExecuteAsync(PollyVersion.V8);
}
27 changes: 20 additions & 7 deletions src/Polly.Core.Benchmarks/Internals/Helper.Pipeline.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,32 @@
using System.Threading.RateLimiting;
using Polly;

namespace Polly.Core.Benchmarks;

internal static partial class Helper
{
public static object CreatePipeline(PollyVersion technology, int count) => technology switch
public static object CreateStrategyPipeline(PollyVersion technology) => technology switch
{
PollyVersion.V7 => count == 1 ? Policy.NoOpAsync<int>() : Policy.WrapAsync(Enumerable.Repeat(0, count).Select(_ => Policy.NoOpAsync<int>()).ToArray()),

PollyVersion.V7 => Policy.WrapAsync(
Policy.TimeoutAsync<int>(TimeSpan.FromSeconds(1)),
Policy.Handle<InvalidOperationException>().OrResult<int>(10).WaitAndRetryAsync(3, attempt => TimeSpan.FromSeconds(1)),
Policy.TimeoutAsync<int>(TimeSpan.FromSeconds(10)),
Policy.BulkheadAsync<int>(10, 10)),
PollyVersion.V8 => CreateStrategy(builder =>
{
for (var i = 0; i < count; i++)
{
builder.AddStrategy(new EmptyResilienceStrategy());
}
builder
.AddTimeout(TimeSpan.FromSeconds(1))
.AddRetry(
predicate => predicate.Exception<InvalidOperationException>().Result(10),
RetryBackoffType.Constant,
3,
TimeSpan.FromSeconds(1))
.AddTimeout(TimeSpan.FromSeconds(10))
.AddConcurrencyLimiter(new ConcurrencyLimiterOptions
{
QueueLimit = 10,
PermitLimit = 10
});
}),
_ => throw new NotSupportedException()
};
Expand Down
25 changes: 25 additions & 0 deletions src/Polly.Core.Benchmarks/Internals/Helper.RateLimiting.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#pragma warning disable S4225 // Extension methods should not extend "object"

using System;
using System.Threading.RateLimiting;

namespace Polly.Core.Benchmarks;

internal static partial class Helper
{
public static object CreateRateLimiter(PollyVersion technology)
{
var timeout = TimeSpan.FromSeconds(10);

return technology switch
{
PollyVersion.V7 => Policy.BulkheadAsync<int>(10, 10),
PollyVersion.V8 => CreateStrategy(builder => builder.AddConcurrencyLimiter(new ConcurrencyLimiterOptions
{
PermitLimit = 10,
QueueLimit = 10
})),
_ => throw new NotSupportedException()
};
}
}
38 changes: 38 additions & 0 deletions src/Polly.Core.Benchmarks/Internals/Helper.Retry.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#pragma warning disable S4225 // Extension methods should not extend "object"

using System;

namespace Polly.Core.Benchmarks;

internal static partial class Helper
{
public static object CreateRetry(PollyVersion technology)
{
var delay = TimeSpan.FromSeconds(10);

return technology switch
{
PollyVersion.V7 =>
Policy
.HandleResult(10)
.Or<InvalidOperationException>()
.WaitAndRetryAsync(3, attempt => delay, (_, _) => Task.CompletedTask),

PollyVersion.V8 => CreateStrategy(builder =>
{
var options = new RetryStrategyOptions
{
RetryCount = 3,
BackoffType = RetryBackoffType.Constant,
BaseDelay = delay
};

options.ShouldRetry.Outcome<int>((outcome, _) => outcome.Result == 10 || outcome.Exception is InvalidOperationException);
options.OnRetry.Add<int>(() => { });

builder.AddRetry(options);
}),
_ => throw new NotSupportedException()
};
}
}
20 changes: 20 additions & 0 deletions src/Polly.Core.Benchmarks/Internals/Helper.StrategyPipeline.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
using Polly;

namespace Polly.Core.Benchmarks;

internal static partial class Helper
{
public static object CreatePipeline(PollyVersion technology, int count) => technology switch
{
PollyVersion.V7 => count == 1 ? Policy.NoOpAsync<int>() : Policy.WrapAsync(Enumerable.Repeat(0, count).Select(_ => Policy.NoOpAsync<int>()).ToArray()),

PollyVersion.V8 => CreateStrategy(builder =>
{
for (var i = 0; i < count; i++)
{
builder.AddStrategy(new EmptyResilienceStrategy());
}
}),
_ => throw new NotSupportedException()
};
}
20 changes: 20 additions & 0 deletions src/Polly.Core.Benchmarks/Internals/Helper.Timeout.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#pragma warning disable S4225 // Extension methods should not extend "object"

using System;

namespace Polly.Core.Benchmarks;

internal static partial class Helper
{
public static object CreateTimeout(PollyVersion technology)
{
var timeout = TimeSpan.FromSeconds(10);

return technology switch
{
PollyVersion.V7 => Policy.TimeoutAsync<int>(timeout),
PollyVersion.V8 => CreateStrategy(builder => builder.AddTimeout(timeout)),
_ => throw new NotSupportedException()
};
}
}
1 change: 1 addition & 0 deletions src/Polly.Core.Benchmarks/Polly.Core.Benchmarks.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

<ItemGroup>
<ProjectReference Include="..\Polly.Core\Polly.Core.csproj" />
<ProjectReference Include="..\Polly.RateLimiting\Polly.RateLimiting.csproj" />
<ProjectReference Include="..\Polly\Polly.csproj" />
</ItemGroup>

Expand Down
44 changes: 36 additions & 8 deletions src/Polly.Core.Benchmarks/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,42 @@ LaunchCount=2 WarmupCount=10

| Method | Components | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio |
|------------------- |----------- |------------:|----------:|----------:|------:|--------:|-------:|----------:|------------:|
| ExecutePipeline_V7 | 1 | 81.25 ns | 1.304 ns | 1.870 ns | 1.00 | 0.00 | 0.0362 | 304 B | 1.00 |
| ExecutePipeline_V8 | 1 | 82.25 ns | 1.414 ns | 2.073 ns | 1.01 | 0.04 | 0.0048 | 40 B | 0.13 |
| ExecutePipeline_V7 | 1 | 71.29 ns | 1.309 ns | 1.959 ns | 1.00 | 0.00 | 0.0362 | 304 B | 1.00 |
| ExecutePipeline_V8 | 1 | 92.93 ns | 1.398 ns | 2.049 ns | 1.30 | 0.05 | 0.0181 | 152 B | 0.50 |
| | | | | | | | | | |
| ExecutePipeline_V7 | 2 | 166.04 ns | 2.875 ns | 4.215 ns | 1.00 | 0.00 | 0.0658 | 552 B | 1.00 |
| ExecutePipeline_V8 | 2 | 108.29 ns | 1.504 ns | 2.251 ns | 0.65 | 0.02 | 0.0048 | 40 B | 0.07 |
| ExecutePipeline_V7 | 2 | 164.39 ns | 5.294 ns | 7.592 ns | 1.00 | 0.00 | 0.0658 | 552 B | 1.00 |
| ExecutePipeline_V8 | 2 | 126.74 ns | 1.198 ns | 1.755 ns | 0.77 | 0.04 | 0.0181 | 152 B | 0.28 |
| | | | | | | | | | |
| ExecutePipeline_V7 | 5 | 531.04 ns | 4.728 ns | 6.930 ns | 1.00 | 0.00 | 0.1545 | 1296 B | 1.00 |
| ExecutePipeline_V8 | 5 | 245.50 ns | 2.344 ns | 3.509 ns | 0.46 | 0.01 | 0.0048 | 40 B | 0.03 |
| ExecutePipeline_V7 | 5 | 540.07 ns | 16.941 ns | 24.832 ns | 1.00 | 0.00 | 0.1545 | 1296 B | 1.00 |
| ExecutePipeline_V8 | 5 | 257.13 ns | 2.748 ns | 4.114 ns | 0.48 | 0.03 | 0.0181 | 152 B | 0.12 |
| | | | | | | | | | |
| ExecutePipeline_V7 | 10 | 1,128.82 ns | 10.838 ns | 15.886 ns | 1.00 | 0.00 | 0.3014 | 2536 B | 1.00 |
| ExecutePipeline_V8 | 10 | 449.31 ns | 2.926 ns | 4.379 ns | 0.40 | 0.01 | 0.0048 | 40 B | 0.02 |
| ExecutePipeline_V7 | 10 | 1,111.72 ns | 16.405 ns | 23.527 ns | 1.00 | 0.00 | 0.3014 | 2536 B | 1.00 |
| ExecutePipeline_V8 | 10 | 467.93 ns | 6.546 ns | 9.388 ns | 0.42 | 0.01 | 0.0181 | 152 B | 0.06 |

## TIMEOUT

| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio |
|------------------ |---------:|--------:|---------:|------:|--------:|-------:|----------:|------------:|
| ExecuteTimeout_V7 | 287.1 ns | 9.20 ns | 12.59 ns | 1.00 | 0.00 | 0.0868 | 728 B | 1.00 |
| ExecuteTimeout_V8 | 272.9 ns | 3.16 ns | 4.54 ns | 0.95 | 0.04 | 0.0439 | 368 B | 0.51 |

## RETRY

| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio |
|---------------- |---------:|--------:|--------:|------:|--------:|-------:|----------:|------------:|
| ExecuteRetry_V7 | 162.5 ns | 2.33 ns | 3.34 ns | 1.00 | 0.00 | 0.0687 | 576 B | 1.00 |
| ExecuteRetry_V8 | 152.3 ns | 1.31 ns | 1.93 ns | 0.94 | 0.02 | 0.0181 | 152 B | 0.26 |

## RATE LIMITER

| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio |
|---------------------- |---------:|--------:|--------:|------:|--------:|-------:|----------:|------------:|
| ExecuteRateLimiter_V7 | 173.8 ns | 2.33 ns | 3.48 ns | 1.00 | 0.00 | 0.0448 | 376 B | 1.00 |
| ExecuteRateLimiter_V8 | 207.9 ns | 2.06 ns | 2.89 ns | 1.19 | 0.03 | 0.0229 | 192 B | 0.51 |
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting that this one is slower.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup surprised me too. It also allocates on every call (around 40 bytes, the remaining 152 bytes will be addressed by pooling the ResilienceContext).

Might be something that could be addressed with .NET folks.

cc @juraj-blazek @geeknoid


## STRATEGY PIPELINE (TIMEOUT + RETRY + TIMEOUT + RATE LIMITER)

| Method | Mean | Error | StdDev | Median | Ratio | RatioSD | Gen0 | Allocated | Alloc Ratio |
|--------------------------- |---------:|----------:|----------:|---------:|------:|--------:|-------:|----------:|------------:|
| ExecuteStrategyPipeline_V7 | 1.207 us | 0.0201 us | 0.0295 us | 1.202 us | 1.00 | 0.00 | 0.2861 | 2400 B | 1.00 |
| ExecuteStrategyPipeline_V8 | 1.117 us | 0.0297 us | 0.0407 us | 1.118 us | 0.93 | 0.05 | 0.0935 | 792 B | 0.33 |