Skip to content

Commit

Permalink
diag
Browse files Browse the repository at this point in the history
  • Loading branch information
@rmandvikar
rmandvikar committed Mar 6, 2024
1 parent 720da57 commit b7858d1
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 26 deletions.
45 changes: 45 additions & 0 deletions src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ public class ExponentialBackoffWithJitterRetryHandler : DelegatingHandler
private readonly IRetrySettings retrySettings;
private readonly ISystemClock clock;

private long callsCount;
private long retryCallsCount;

/// <inheritdoc cref="ExponentialBackoffWithJitterRetryHandler" />
public ExponentialBackoffWithJitterRetryHandler(
IRetrySettings retrySettings,
Expand Down Expand Up @@ -75,18 +78,58 @@ protected override async Task<HttpResponseMessage> SendAsync(
context[ContextKey.RetryAttempt] = 0;
context[ContextKey.SleepDurations] = sleepDurationsWithJitter;

Interlocked.Increment(ref callsCount);

var tuple = await retryPolicy.ExecuteAsync(
action: async (context, ct) =>
{
var retryAttempt = (int)context[ContextKey.RetryAttempt];
request.Properties[RequestProperties.PollyRetryAttempt] = retryAttempt;
long retryCalls;
if (retryAttempt >= 1)
{
retryCalls = Interlocked.Increment(ref retryCallsCount);
}
else
{
retryCalls = Interlocked.Read(ref retryCallsCount);
}
var calls = Interlocked.Read(ref callsCount);
#if DEBUG
Console.WriteLine($"retryCalls: {retryCallsCount}, callsCount: {callsCount}");
#endif
double retryCallsPercentage = 0;
if (retryAttempt >= 1)
{
if (calls > 0
//&& calls > 2
&& (retryCallsPercentage = ((double)retryCalls / (double)calls * 100)) > retrySettings.RetryCallsPercentageThreshold)
{
throw new TokenBucketRetryException(
$"retryCallsPercentage (threshold): {retrySettings.RetryCallsPercentageThreshold}, but was retryCallsPercentage: {retryCallsPercentage}");
}
}
#if DEBUG
Console.WriteLine($"retryAttempt: {retryAttempt}. retryCallsPercentage (threshold): {retrySettings.RetryCallsPercentageThreshold}, but was retryCallsPercentage: {retryCallsPercentage}");
#endif
var response = await base.SendAsync(request, ct)
.ConfigureAwait(false);
if (retryAttempt >= 1)
{
Interlocked.Decrement(ref retryCallsCount);
}
return (response, context);
},
context: context,
cancellationToken: cancellationToken)
.ConfigureAwait(false);

Interlocked.Decrement(ref callsCount);

return tuple.response;
}

Expand Down Expand Up @@ -179,12 +222,14 @@ public interface IRetrySettings
{
int RetryCount { get; }
int RetryDelayInMilliseconds { get; }
double RetryCallsPercentageThreshold { get; }
}

public record class RetrySettings : IRetrySettings
{
public int RetryCount { get; init; }
public int RetryDelayInMilliseconds { get; init; }
public double RetryCallsPercentageThreshold { get; init; }
}

internal static class ContextKey
Expand Down
42 changes: 16 additions & 26 deletions tests/rm.DelegatingHandlersTest/TokenBucketRetryHandlerTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,19 @@ public void Throws_TokenBucketRetryException()
StatusCode = (HttpStatusCode)500,
Content = content,
});
var tokenBucketRetryHandler = new TokenBucketRetryHandler(
new TokenBucketRetryHandlerSettings
{
Percentage = 0.10d,
});
var clockMock = fixture.Freeze<Mock<ISystemClock>>();
clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 2,
RetryDelayInMilliseconds = 0,
RetryCallsPercentageThreshold = 10.00d,
},
clockMock.Object);

using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, tokenBucketRetryHandler, shortCircuitingResponseHandler);
retryHandler, shortCircuitingResponseHandler);

using var requestMessage = fixture.Create<HttpRequestMessage>();
var ex = Assert.ThrowsAsync<TokenBucketRetryException>(async () =>
Expand All @@ -65,23 +61,19 @@ public async Task Does_Not_Throw_TokenBucketRetryException()
StatusCode = (HttpStatusCode)200,
Content = content,
});
var tokenBucketRetryHandler = new TokenBucketRetryHandler(
new TokenBucketRetryHandlerSettings
{
Percentage = 0.10d,
});
var clockMock = fixture.Freeze<Mock<ISystemClock>>();
clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 2,
RetryDelayInMilliseconds = 0,
RetryCallsPercentageThreshold = 10.00d,
},
clockMock.Object);

using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, tokenBucketRetryHandler, shortCircuitingResponseHandler);
retryHandler, shortCircuitingResponseHandler);

using var requestMessage = fixture.Create<HttpRequestMessage>();
using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
Expand All @@ -100,23 +92,19 @@ public async Task Does_Not_Throw_TokenBucketRetryException_Iterations()
StatusCode = (HttpStatusCode)200,
Content = content,
});
var tokenBucketRetryHandler = new TokenBucketRetryHandler(
new TokenBucketRetryHandlerSettings
{
Percentage = 0.05d,
});
var clockMock = fixture.Freeze<Mock<ISystemClock>>();
clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 2,
RetryDelayInMilliseconds = 0,
RetryCallsPercentageThreshold = 10.00d,
},
clockMock.Object);

using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, tokenBucketRetryHandler, shortCircuitingResponseHandler);
retryHandler, shortCircuitingResponseHandler);

const int iterations = 1_000;
for (int i = 0; i < iterations; i++)
Expand All @@ -138,31 +126,33 @@ public async Task Does_Not_Throw_TokenBucketRetryException_Probability_Iteration
StatusCode = (HttpStatusCode)200,
Content = fixture.Create<string>(),
});
var procrastinatingWithProbabilityHandler = new ProcrastinatingGaussianHandler(
new ProcrastinatingGaussianHandlerSettings
{
Mu = 10,
Sigma = 50,
},
rng);
var shortCircuitingResponseWithProbabilityHandler = new ShortCircuitingResponseWithProbabilityHandler(
new ShortCircuitingResponseWithProbabilityHandlerSettings
{
ProbabilityPercentage = 0.1d,
ProbabilityPercentage = 40.00d,
StatusCode = (HttpStatusCode)500,
Content = fixture.Create<string>(),
},
rng);
var tokenBucketRetryHandler = new TokenBucketRetryHandler(
new TokenBucketRetryHandlerSettings
{
Percentage = 0.10d,
});
var clockMock = fixture.Freeze<Mock<ISystemClock>>();
clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 2,
RetryDelayInMilliseconds = 0,
RetryCallsPercentageThreshold = 10.00d,
},
clockMock.Object);

using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, tokenBucketRetryHandler, shortCircuitingResponseWithProbabilityHandler, shortCircuitingResponseHandler);
retryHandler, procrastinatingWithProbabilityHandler, shortCircuitingResponseWithProbabilityHandler, shortCircuitingResponseHandler);

const int iterations = 1_000;
const int batchSize = 100;
Expand Down

0 comments on commit b7858d1

Please sign in to comment.