From 4c82d5e05ae1411674c934959569da06a2b1d972 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 16:27:19 +0100 Subject: [PATCH 1/7] [Docs] Apply chaos selectively --- docs/chaos/index.md | 107 +++++++++++++++++++++++++++++ src/Snippets/Docs/Chaos.Index.cs | 111 +++++++++++++++++++++++++++++++ 2 files changed, 218 insertions(+) diff --git a/docs/chaos/index.md b/docs/chaos/index.md index 05448f50969..f5ca4edb330 100644 --- a/docs/chaos/index.md +++ b/docs/chaos/index.md @@ -90,3 +90,110 @@ All the strategies' options implement the [`ChaosStrategyOptions`](xref:Polly.Si > If both `Enabled` and `EnabledGenerator` are specified then `Enabled` will be ignored. [simmy]: https://github.com/Polly-Contrib/Simmy + +## Patterns + +### Inject chaos selectively + +You aim to dynamically adjust the frequency and timing of chaos injection. For instance, in pre-production and test environments, it's sensible to consistently inject chaos. This proactive approach helps in preparing for potential failures. In production environments, however, you may prefer to limit chaos to certain users and tenants, ensuring that regular users remain unaffected. The Simmy API offers the flexibility needed to manage these varying scenarios. + +Additionally, you have the option to dynamically alter the injection rate and simulate extreme scenarios by setting the injection rate to *1.0 (100%)*. Exercise caution when applying this high rate, restricting it to a subset of tenants and users to avoid rendering the system unusable for regular users. + +The following example illustrates how to configure chaos strategies accordingly: + + +```cs +services.AddResiliencePipeline("chaos-pipeline", (builder, context) => +{ + var environment = context.ServiceProvider.GetRequiredService(); + + builder.AddChaosFault(new ChaosFaultStrategyOptions + { + EnabledGenerator = args => + { + // Enable chaos in development and staging environments. + if (environment.IsDevelopment() || environment.IsStaging()) + { + return ValueTask.FromResult(true); + } + + // Enable chaos for specific users or tenants, even in production environments. + if (ShouldEnableChaos(args.Context)) + { + return ValueTask.FromResult(true); + } + + return ValueTask.FromResult(false); + }, + InjectionRateGenerator = args => + { + if (environment.IsStaging()) + { + // 1% chance of failure on staging environments. + return ValueTask.FromResult(0.01); + } + + if (environment.IsDevelopment()) + { + // 5% chance of failure on development environments. + return ValueTask.FromResult(0.05); + } + + // The context can carry information to help determine the injection rate. + // For instance, in production environments, you might have certain test users or tenants + // for whom you wish to inject chaos. + if (ResolveInjectionRate(args.Context, out double injectionRate)) + { + return ValueTask.FromResult(injectionRate); + } + + // No chaos on production environments. + return ValueTask.FromResult(0.0); + }, + FaultGenerator = new FaultGenerator() + .AddException() + .AddException() + }); +}); +``` + + +We suggest encapsulating the chaos decisions and injection rate in a shared class, such as `IChaosManager`: + + +```cs +public interface IChaosManager +{ + bool IsChaosEnabled(ResilienceContext context); + + double GetInjectionRate(ResilienceContext context); +} +``` + + +This approach allows you to consistently apply and manage chaos-related settings across various chaos strategies by reusing `IChaosManager`. By centralizing the logic for enabling chaos and determining injection rates, you can ensure uniformity and ease of maintenance across your application and reuse it across multiple chaos strategies: + + +```cs +services.AddResiliencePipeline("chaos-pipeline", (builder, context) => +{ + var chaosManager = context.ServiceProvider.GetRequiredService(); + + builder + .AddChaosFault(new ChaosFaultStrategyOptions + { + EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)), + InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)), + FaultGenerator = new FaultGenerator() + .AddException() + .AddException() + }) + .AddChaosLatency(new ChaosLatencyStrategyOptions + { + EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)), + InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)), + Latency = TimeSpan.FromSeconds(60) + }); +}); +``` + diff --git a/src/Snippets/Docs/Chaos.Index.cs b/src/Snippets/Docs/Chaos.Index.cs index 98e40ecb91b..1d26c10d588 100644 --- a/src/Snippets/Docs/Chaos.Index.cs +++ b/src/Snippets/Docs/Chaos.Index.cs @@ -1,7 +1,11 @@ using System.Net.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; using Polly.CircuitBreaker; using Polly.Retry; using Polly.Simmy; +using Polly.Simmy.Fault; +using Polly.Simmy.Latency; namespace Snippets.Docs; @@ -35,5 +39,112 @@ public static void Usage() #endregion } + public static void ApplyChaosSelectively(IServiceCollection services) + { + #region chaos-selective + + services.AddResiliencePipeline("chaos-pipeline", (builder, context) => + { + var environment = context.ServiceProvider.GetRequiredService(); + + builder.AddChaosFault(new ChaosFaultStrategyOptions + { + EnabledGenerator = args => + { + // Enable chaos in development and staging environments. + if (environment.IsDevelopment() || environment.IsStaging()) + { + return ValueTask.FromResult(true); + } + + // Enable chaos for specific users or tenants, even in production environments. + if (ShouldEnableChaos(args.Context)) + { + return ValueTask.FromResult(true); + } + + return ValueTask.FromResult(false); + }, + InjectionRateGenerator = args => + { + if (environment.IsStaging()) + { + // 1% chance of failure on staging environments. + return ValueTask.FromResult(0.01); + } + + if (environment.IsDevelopment()) + { + // 5% chance of failure on development environments. + return ValueTask.FromResult(0.05); + } + + // The context can carry information to help determine the injection rate. + // For instance, in production environments, you might have certain test users or tenants + // for whom you wish to inject chaos. + if (ResolveInjectionRate(args.Context, out double injectionRate)) + { + return ValueTask.FromResult(injectionRate); + } + + // No chaos on production environments. + return ValueTask.FromResult(0.0); + }, + FaultGenerator = new FaultGenerator() + .AddException() + .AddException() + }); + }); + + #endregion + } + + public static void ApplyChaosSelectivelyWithChaosManager(IServiceCollection services) + { + #region chaos-selective-manager + + services.AddResiliencePipeline("chaos-pipeline", (builder, context) => + { + var chaosManager = context.ServiceProvider.GetRequiredService(); + + builder + .AddChaosFault(new ChaosFaultStrategyOptions + { + EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)), + InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)), + FaultGenerator = new FaultGenerator() + .AddException() + .AddException() + }) + .AddChaosLatency(new ChaosLatencyStrategyOptions + { + EnabledGenerator = args => ValueTask.FromResult(chaosManager.IsChaosEnabled(args.Context)), + InjectionRateGenerator = args => ValueTask.FromResult(chaosManager.GetInjectionRate(args.Context)), + Latency = TimeSpan.FromSeconds(60) + }); + }); + + #endregion + } + + private static bool ResolveInjectionRate(ResilienceContext context, out double injectionRate) + { + injectionRate = 0.0; + return false; + } + + private static bool ShouldEnableChaos(ResilienceContext context) => true; + private static ValueTask RestartRedisAsync(CancellationToken cancellationToken) => ValueTask.CompletedTask; + + #region chaos-manager + + public interface IChaosManager + { + bool IsChaosEnabled(ResilienceContext context); + + double GetInjectionRate(ResilienceContext context); + } + + #endregion } From 0bbd8492c49a76d63f08e60e9a18610ed5d0d3b6 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 16:40:49 +0100 Subject: [PATCH 2/7] Add note --- docs/chaos/index.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/chaos/index.md b/docs/chaos/index.md index f5ca4edb330..8c447891d72 100644 --- a/docs/chaos/index.md +++ b/docs/chaos/index.md @@ -197,3 +197,6 @@ services.AddResiliencePipeline("chaos-pipeline", (builder, context) => }); ``` + +> [!NOTE] +> An alternative method involves using [`Microsoft.Extensions.AsyncState`](https://www.nuget.org/packages/Microsoft.Extensions.AsyncState) for storing information relevant to chaos injection decisions. This can be particularly useful in frameworks like ASP.NET Core. For instance, you could implement a middleware that retrieves user information from `HttpContext`, assesses the user type, and then stores this data in `IAsyncLocal`. Subsequently, `IChaosManager` can access `IAsyncLocal` to retrieve this information. This approach eliminates the need to manually insert such data into `ResilienceContext` for each call within the resilience pipeline, thereby streamlining the process. From 70118cb7221828bb0da4e3423e2d1a4601604df3 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 16:43:47 +0100 Subject: [PATCH 3/7] typos --- docs/chaos/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/chaos/index.md b/docs/chaos/index.md index 8c447891d72..6dbb189b0be 100644 --- a/docs/chaos/index.md +++ b/docs/chaos/index.md @@ -199,4 +199,4 @@ services.AddResiliencePipeline("chaos-pipeline", (builder, context) => > [!NOTE] -> An alternative method involves using [`Microsoft.Extensions.AsyncState`](https://www.nuget.org/packages/Microsoft.Extensions.AsyncState) for storing information relevant to chaos injection decisions. This can be particularly useful in frameworks like ASP.NET Core. For instance, you could implement a middleware that retrieves user information from `HttpContext`, assesses the user type, and then stores this data in `IAsyncLocal`. Subsequently, `IChaosManager` can access `IAsyncLocal` to retrieve this information. This approach eliminates the need to manually insert such data into `ResilienceContext` for each call within the resilience pipeline, thereby streamlining the process. +> An alternative method involves using [`Microsoft.Extensions.AsyncState`](https://www.nuget.org/packages/Microsoft.Extensions.AsyncState) for storing information relevant to chaos injection decisions. This can be particularly useful in frameworks like ASP.NET Core. For instance, you could implement a middleware that retrieves user information from `HttpContext`, assesses the user type, and then stores this data in `IAsyncContext`. Subsequently, `IChaosManager` can access `IAsyncContext` to retrieve this information. This approach eliminates the need to manually insert such data into `ResilienceContext` for each call within the resilience pipeline, thereby streamlining the process. From abdeefec1bef0f65a82b675d3a08477fc5af58cf Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 16:48:57 +0100 Subject: [PATCH 4/7] Spell check failure --- .github/wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/wordlist.txt b/.github/wordlist.txt index 6968808c5ec..ca1df1f6423 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -68,3 +68,4 @@ waitandretry wpf xunit enricher +ASP.NET From 200d6b63dd9718c2cc85ae6a3ff848a7bab2c34a Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 16:52:51 +0100 Subject: [PATCH 5/7] Cleanup --- docs/chaos/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/chaos/index.md b/docs/chaos/index.md index 6dbb189b0be..4cec02836f9 100644 --- a/docs/chaos/index.md +++ b/docs/chaos/index.md @@ -95,7 +95,7 @@ All the strategies' options implement the [`ChaosStrategyOptions`](xref:Polly.Si ### Inject chaos selectively -You aim to dynamically adjust the frequency and timing of chaos injection. For instance, in pre-production and test environments, it's sensible to consistently inject chaos. This proactive approach helps in preparing for potential failures. In production environments, however, you may prefer to limit chaos to certain users and tenants, ensuring that regular users remain unaffected. The Simmy API offers the flexibility needed to manage these varying scenarios. +You aim to dynamically adjust the frequency and timing of chaos injection. For instance, in pre-production and test environments, it's sensible to consistently inject chaos. This proactive approach helps in preparing for potential failures. In production environments, however, you may prefer to limit chaos to certain users and tenants, ensuring that regular users remain unaffected. The chaos API offers the flexibility needed to manage these varying scenarios. Additionally, you have the option to dynamically alter the injection rate and simulate extreme scenarios by setting the injection rate to *1.0 (100%)*. Exercise caution when applying this high rate, restricting it to a subset of tenants and users to avoid rendering the system unusable for regular users. From 39c95e96c017094fb57a878a519c49da4551b7f5 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 17:20:02 +0100 Subject: [PATCH 6/7] fix build --- .github/wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/wordlist.txt b/.github/wordlist.txt index ca1df1f6423..d2a5d50f47e 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -69,3 +69,4 @@ wpf xunit enricher ASP.NET +middleware From ce3c3e52fcfe7ee12a9f299a432cc337f81ece86 Mon Sep 17 00:00:00 2001 From: Martin Tomka Date: Thu, 25 Jan 2024 17:56:29 +0100 Subject: [PATCH 7/7] sort the lines --- .github/wordlist.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/wordlist.txt b/.github/wordlist.txt index d2a5d50f47e..0ddf7ccf1ce 100644 --- a/.github/wordlist.txt +++ b/.github/wordlist.txt @@ -1,5 +1,6 @@ alloc apis +ASP.NET async azurefunctions bcl @@ -13,6 +14,7 @@ deserialization dotnet dotnetrocks durations +enricher eshoponcontainers extensibility flurl @@ -23,6 +25,7 @@ jittered json loggingpolicy markdownsnippets +middleware minver moq namespace @@ -30,9 +33,9 @@ natively ndc nuget oss -pcl -parallelize paas +parallelize +pcl pluralsight pollydocs pre @@ -67,6 +70,3 @@ uwp waitandretry wpf xunit -enricher -ASP.NET -middleware