diff --git a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs index 3a28290..846a737 100644 --- a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs +++ b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs @@ -34,6 +34,9 @@ public class ExponentialBackoffWithJitterRetryHandler : DelegatingHandler private readonly IRetrySettings retrySettings; private readonly ISystemClock clock; + private long callsCount; + private long retryCallsCount; + /// public ExponentialBackoffWithJitterRetryHandler( IRetrySettings retrySettings, @@ -75,18 +78,58 @@ protected override async Task SendAsync( context[ContextKey.RetryAttempt] = 0; context[ContextKey.SleepDurations] = sleepDurationsWithJitter; + Interlocked.Increment(ref callsCount); + var tuple = await retryPolicy.ExecuteAsync( action: async (context, ct) => { var retryAttempt = (int)context[ContextKey.RetryAttempt]; request.Properties[RequestProperties.PollyRetryAttempt] = retryAttempt; + + long retryCalls; + if (retryAttempt >= 1) + { + retryCalls = Interlocked.Increment(ref retryCallsCount); + } + else + { + retryCalls = Interlocked.Read(ref retryCallsCount); + } + var calls = Interlocked.Read(ref callsCount); +#if DEBUG + Console.WriteLine($"retryCalls: {retryCallsCount}, callsCount: {callsCount}"); +#endif + double retryCallsPercentage = 0; + if (retryAttempt >= 1) + { + if (calls > 0 + //&& calls > 2 + && (retryCallsPercentage = ((double)retryCalls / (double)calls * 100)) > retrySettings.RetryCallsPercentageThreshold) + { + throw new TokenBucketRetryException( + $"retryCallsPercentage (threshold): {retrySettings.RetryCallsPercentageThreshold}, but was retryCallsPercentage: {retryCallsPercentage}"); + } + } +#if DEBUG + Console.WriteLine($"retryAttempt: {retryAttempt}. retryCallsPercentage (threshold): {retrySettings.RetryCallsPercentageThreshold}, but was retryCallsPercentage: {retryCallsPercentage}"); +#endif + var response = await base.SendAsync(request, ct) .ConfigureAwait(false); + + if (retryAttempt >= 1) + { + Interlocked.Decrement(ref retryCallsCount); + } + return (response, context); }, context: context, cancellationToken: cancellationToken) .ConfigureAwait(false); + + Interlocked.Decrement(ref callsCount); + return tuple.response; } @@ -179,12 +222,14 @@ public interface IRetrySettings { int RetryCount { get; } int RetryDelayInMilliseconds { get; } + double RetryCallsPercentageThreshold { get; } } public record class RetrySettings : IRetrySettings { public int RetryCount { get; init; } public int RetryDelayInMilliseconds { get; init; } + public double RetryCallsPercentageThreshold { get; init; } } internal static class ContextKey diff --git a/tests/rm.DelegatingHandlersTest/TokenBucketRetryHandlerTests.cs b/tests/rm.DelegatingHandlersTest/TokenBucketRetryHandlerTests.cs index 0e3fbca..0d597f9 100644 --- a/tests/rm.DelegatingHandlersTest/TokenBucketRetryHandlerTests.cs +++ b/tests/rm.DelegatingHandlersTest/TokenBucketRetryHandlerTests.cs @@ -27,11 +27,6 @@ public void Throws_TokenBucketRetryException() StatusCode = (HttpStatusCode)500, Content = content, }); - var tokenBucketRetryHandler = new TokenBucketRetryHandler( - new TokenBucketRetryHandlerSettings - { - Percentage = 0.10d, - }); var clockMock = fixture.Freeze>(); clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( @@ -39,11 +34,12 @@ public void Throws_TokenBucketRetryException() { RetryCount = 2, RetryDelayInMilliseconds = 0, + RetryCallsPercentageThreshold = 10.00d, }, clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( - retryHandler, tokenBucketRetryHandler, shortCircuitingResponseHandler); + retryHandler, shortCircuitingResponseHandler); using var requestMessage = fixture.Create(); var ex = Assert.ThrowsAsync(async () => @@ -65,11 +61,6 @@ public async Task Does_Not_Throw_TokenBucketRetryException() StatusCode = (HttpStatusCode)200, Content = content, }); - var tokenBucketRetryHandler = new TokenBucketRetryHandler( - new TokenBucketRetryHandlerSettings - { - Percentage = 0.10d, - }); var clockMock = fixture.Freeze>(); clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( @@ -77,11 +68,12 @@ public async Task Does_Not_Throw_TokenBucketRetryException() { RetryCount = 2, RetryDelayInMilliseconds = 0, + RetryCallsPercentageThreshold = 10.00d, }, clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( - retryHandler, tokenBucketRetryHandler, shortCircuitingResponseHandler); + retryHandler, shortCircuitingResponseHandler); using var requestMessage = fixture.Create(); using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); @@ -100,11 +92,6 @@ public async Task Does_Not_Throw_TokenBucketRetryException_Iterations() StatusCode = (HttpStatusCode)200, Content = content, }); - var tokenBucketRetryHandler = new TokenBucketRetryHandler( - new TokenBucketRetryHandlerSettings - { - Percentage = 0.05d, - }); var clockMock = fixture.Freeze>(); clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( @@ -112,11 +99,12 @@ public async Task Does_Not_Throw_TokenBucketRetryException_Iterations() { RetryCount = 2, RetryDelayInMilliseconds = 0, + RetryCallsPercentageThreshold = 10.00d, }, clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( - retryHandler, tokenBucketRetryHandler, shortCircuitingResponseHandler); + retryHandler, shortCircuitingResponseHandler); const int iterations = 1_000; for (int i = 0; i < iterations; i++) @@ -138,19 +126,21 @@ public async Task Does_Not_Throw_TokenBucketRetryException_Probability_Iteration StatusCode = (HttpStatusCode)200, Content = fixture.Create(), }); + var procrastinatingWithProbabilityHandler = new ProcrastinatingGaussianHandler( + new ProcrastinatingGaussianHandlerSettings + { + Mu = 10, + Sigma = 50, + }, + rng); var shortCircuitingResponseWithProbabilityHandler = new ShortCircuitingResponseWithProbabilityHandler( new ShortCircuitingResponseWithProbabilityHandlerSettings { - ProbabilityPercentage = 0.1d, + ProbabilityPercentage = 40.00d, StatusCode = (HttpStatusCode)500, Content = fixture.Create(), }, rng); - var tokenBucketRetryHandler = new TokenBucketRetryHandler( - new TokenBucketRetryHandlerSettings - { - Percentage = 0.10d, - }); var clockMock = fixture.Freeze>(); clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( @@ -158,11 +148,11 @@ public async Task Does_Not_Throw_TokenBucketRetryException_Probability_Iteration { RetryCount = 2, RetryDelayInMilliseconds = 0, + RetryCallsPercentageThreshold = 10.00d, }, clockMock.Object); - using var invoker = HttpMessageInvokerFactory.Create( - retryHandler, tokenBucketRetryHandler, shortCircuitingResponseWithProbabilityHandler, shortCircuitingResponseHandler); + retryHandler, procrastinatingWithProbabilityHandler, shortCircuitingResponseWithProbabilityHandler, shortCircuitingResponseHandler); const int iterations = 1_000; const int batchSize = 100;