diff --git a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs
index 67b5c6b..cd8d2dc 100644
--- a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs
+++ b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs
@@ -1,10 +1,13 @@
using System;
+using System.Linq;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;
using Polly;
using Polly.Contrib.WaitAndRetry;
using Polly.Retry;
+using rm.Clock;
+using rm.Extensions;
namespace rm.DelegatingHandlers;
@@ -12,38 +15,48 @@ namespace rm.DelegatingHandlers;
/// Retries on certain conditions with exponential backoff jitter (DecorrelatedJitterBackoffV2).
///
/// Retry conditions:
-/// HttpRequestException, 5xx.
+/// HttpRequestException, 5xx, 429 (see retry-after header below).
+///
+/// retry-after header:
+///
+/// For 503: retry honoring header if present, else retry as usual.
+///
+/// For 429: retry honoring header only if present, else do not retry.
///
///
/// source
///
public class ExponentialBackoffWithJitterRetryHandler : DelegatingHandler
{
- private readonly AsyncRetryPolicy retryPolicy;
+ private readonly AsyncRetryPolicy<(HttpResponseMessage response, Context Context)> retryPolicy;
+ private readonly IRetrySettings retrySettings;
+ private readonly ISystemClock clock;
///
public ExponentialBackoffWithJitterRetryHandler(
- IRetrySettings retrySettings)
+ IRetrySettings retrySettings,
+ ISystemClock clock)
{
- _ = retrySettings
+ this.retrySettings = retrySettings
?? throw new ArgumentNullException(nameof(retrySettings));
-
- var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2(
- medianFirstRetryDelay: TimeSpan.FromMilliseconds(retrySettings.RetryDelayInMilliseconds),
- retryCount: retrySettings.RetryCount);
+ this.clock = clock
+ ?? throw new ArgumentNullException(nameof(clock));
// note: response can't be null
// ref: https://github.com/dotnet/runtime/issues/19925#issuecomment-272664671
retryPolicy = Policy
.Handle()
.Or()
- .OrResult(response => response.Is5xx())
+ .OrResult<(HttpResponseMessage response, Context context)>(
+ tuple => CanRetry(tuple.response, tuple.context))
.WaitAndRetryAsync(
- sleepDurations: sleepDurationsWithJitter,
+ retryCount: retrySettings.RetryCount,
+ sleepDurationProvider: (retryAttempt, responseResult, context) =>
+ ((TimeSpan[])context[ContextKey.SleepDurations])[retryAttempt - 1],
onRetry: (responseResult, delay, retryAttempt, context) =>
{
// note: response can be null in case of handled exception
- responseResult.Result?.Dispose();
+ responseResult.Result.response?.Dispose();
context[ContextKey.RetryAttempt] = retryAttempt;
});
}
@@ -52,19 +65,112 @@ protected override async Task SendAsync(
HttpRequestMessage request,
CancellationToken cancellationToken)
{
- return await retryPolicy.ExecuteAsync(
+ // read the retry delays upfront
+ var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2(
+ medianFirstRetryDelay: TimeSpan.FromMilliseconds(retrySettings.RetryDelayInMilliseconds),
+ retryCount: retrySettings.RetryCount).ToArray();
+ var context = new Context();
+ context[ContextKey.SleepDurations] = sleepDurationsWithJitter;
+
+ var tuple = await retryPolicy.ExecuteAsync(
action: async (context, ct) =>
{
if (context.TryGetValue(ContextKey.RetryAttempt, out var retryAttempt))
{
request.Properties[RequestProperties.PollyRetryAttempt] = retryAttempt;
}
- return await base.SendAsync(request, ct)
+ var response = await base.SendAsync(request, ct)
.ConfigureAwait(false);
+ return (response, context);
},
- context: new Context(),
+ context: context,
cancellationToken: cancellationToken)
.ConfigureAwait(false);
+ return tuple.response;
+ }
+
+ ///
+ /// Returns true if the response can be retried considering things as,
+ /// retry attempt, and retry-after header (if present).
+ ///
+ private bool CanRetry(
+ HttpResponseMessage response,
+ Context context)
+ {
+ var sleepDurationsWithJitter = (TimeSpan[])context[ContextKey.SleepDurations];
+ if (sleepDurationsWithJitter.IsEmpty())
+ {
+ return false;
+ }
+ // retryAttempt is 0-based
+ var retryAttempt = context.TryGetValue(ContextKey.RetryAttempt, out object retryAttemptObj) ? (int)retryAttemptObj : 0;
+ if (retryAttempt == sleepDurationsWithJitter.Count())
+ {
+ return false;
+ }
+ var sleepDurationWithJitter = sleepDurationsWithJitter[retryAttempt];
+
+ var statusCode = (int)response.StatusCode;
+
+ // here be dragons
+ var retry = false;
+ // retry on 5xx, 429 only
+ if (response.Is5xx() || statusCode == 429)
+ {
+ // retry on 503, 429 looking at retry-after value
+ if (statusCode == 503 || statusCode == 429)
+ {
+ // note: look at retry-after value but don't use it to avoid surges at same time;
+ // use it to determine whether to retry or not
+ var isRetryAfterPresent = response.Headers.TryGetValue(ResponseHeaders.RetryAfter, out var retryAfterValue)
+ && retryAfterValue != null;
+
+#if DEBUG
+ Console.WriteLine($"retryAfterValue: {retryAfterValue}");
+#endif
+
+ // retry on 503, 429 only on valid retry-after value
+ if (isRetryAfterPresent)
+ {
+ TimeSpan retryAfter;
+ retry =
+ ((double.TryParse(retryAfterValue, out double retryAfterDelay)
+ // ignore network latency, delay could be 0
+ && Math.Max((retryAfter = TimeSpan.FromSeconds(retryAfterDelay)).TotalSeconds, 0) >= 0)
+ ||
+ (DateTimeOffset.TryParse(retryAfterValue, out DateTimeOffset retryAfterDate)
+ // date could be in the past due to network latency
+ && Math.Max((retryAfter = retryAfterDate - clock.UtcNow).TotalSeconds, 0) >= 0))
+ // only retry if delay is at or above retry-after value
+ && retryAfter <= sleepDurationWithJitter;
+ }
+ else
+ {
+ // retry on 503 if retry-after not present as typical
+ if (statusCode == 503)
+ {
+ retry = true;
+ }
+ // do NOT retry on 429 if retry-after not present as typical
+ else if (statusCode == 429)
+ {
+ retry = false;
+ }
+ }
+ }
+ else
+ {
+ // retry on 5xx (other than 503) as typical
+ retry = true;
+ }
+ }
+
+#if DEBUG
+ Console.WriteLine($"sleepDurationWithJitter: {sleepDurationWithJitter}");
+ Console.WriteLine($"retry: {retry}");
+#endif
+
+ return retry;
}
}
@@ -83,4 +189,5 @@ public record class RetrySettings : IRetrySettings
internal static class ContextKey
{
internal const string RetryAttempt = nameof(RetryAttempt);
+ internal const string SleepDurations = nameof(SleepDurations);
}
diff --git a/src/rm.DelegatingHandlers/misc/AsyncRetryTResultSyntax.cs b/src/rm.DelegatingHandlers/misc/AsyncRetryTResultSyntax.cs
new file mode 100644
index 0000000..423b3b9
--- /dev/null
+++ b/src/rm.DelegatingHandlers/misc/AsyncRetryTResultSyntax.cs
@@ -0,0 +1,41 @@
+using System;
+using Polly.Retry;
+
+namespace Polly;
+
+public static class AsyncRetryTResultSyntax
+{
+ ///
+ /// Builds an that will wait and retry times
+ /// calling on each retry with the handled exception or result, the current sleep duration, retry count, and context data.
+ /// On each retry, the duration to wait is calculated by calling with
+ /// the current retry number (1 for first retry, 2 for second etc), result of previous execution, and execution context.
+ ///
+ /// The policy builder.
+ /// The retry count.
+ /// The function that provides the duration to wait for for a particular retry attempt.
+ /// The action to call on each retry.
+ /// The policy instance.
+ /// retryCount;Value must be greater than or equal to zero.
+ ///
+ /// sleepDurationProvider
+ /// or
+ /// onRetryAsync
+ ///
+ ///
+ /// issue: https://github.com/App-vNext/Polly/issues/908
+ ///
+ public static AsyncRetryPolicy WaitAndRetryAsync(this PolicyBuilder policyBuilder, int retryCount,
+ Func, Context, TimeSpan> sleepDurationProvider, Action, TimeSpan, int, Context> onRetry)
+ {
+ if (onRetry == null) throw new ArgumentNullException(nameof(onRetry));
+
+ return policyBuilder.WaitAndRetryAsync(
+ retryCount,
+ sleepDurationProvider,
+#pragma warning disable 1998 // async method has no awaits, will run synchronously
+ onRetryAsync: async (outcome, timespan, i, ctx) => onRetry(outcome, timespan, i, ctx)
+#pragma warning restore 1998
+ );
+ }
+}
diff --git a/src/rm.DelegatingHandlers/misc/HttpStatusCodeIntExtensions.cs b/src/rm.DelegatingHandlers/misc/HttpStatusCodeIntExtensions.cs
new file mode 100644
index 0000000..2e9b53d
--- /dev/null
+++ b/src/rm.DelegatingHandlers/misc/HttpStatusCodeIntExtensions.cs
@@ -0,0 +1,55 @@
+using System.Net;
+
+namespace rm.DelegatingHandlers;
+
+public static class HttpStatusCodeIntExtensions
+{
+ public static bool Is1xx(this int statusCode)
+ {
+ return ((HttpStatusCode)statusCode).Is1xx();
+ }
+
+ public static bool Is2xx(this int statusCode)
+ {
+ return ((HttpStatusCode)statusCode).Is2xx();
+ }
+
+ public static bool Is3xx(this int statusCode)
+ {
+ return ((HttpStatusCode)statusCode).Is3xx();
+ }
+
+ public static bool Is4xx(this int statusCode)
+ {
+ return ((HttpStatusCode)statusCode).Is4xx();
+ }
+
+ public static bool Is5xx(this int statusCode)
+ {
+ return ((HttpStatusCode)statusCode).Is5xx();
+ }
+
+ ///
+ /// Returns true if status code is a client error status code (4xx).
+ ///
+ public static bool IsClientErrorStatusCode(this int statusCode)
+ {
+ return statusCode.Is4xx();
+ }
+
+ ///
+ /// Returns true if status code is a server error status code (5xx).
+ ///
+ public static bool IsServerErrorStatusCode(this int statusCode)
+ {
+ return statusCode.Is5xx();
+ }
+
+ ///
+ /// Returns true if status code is an error status code (4xx, 5xx).
+ ///
+ public static bool IsErrorStatusCode(this int statusCode)
+ {
+ return statusCode.Is4xx() || statusCode.Is5xx();
+ }
+}
diff --git a/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj b/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj
index 871b968..53f7217 100644
--- a/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj
+++ b/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj
@@ -30,6 +30,7 @@
+
diff --git a/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs b/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs
index 1ca8915..6cd6a99 100644
--- a/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs
+++ b/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs
@@ -1,7 +1,10 @@
using System.Net;
using AutoFixture;
using AutoFixture.AutoMoq;
+using Moq;
using NUnit.Framework;
+using Polly.Contrib.WaitAndRetry;
+using rm.Clock;
using rm.DelegatingHandlers;
namespace rm.DelegatingHandlersTest;
@@ -9,23 +12,28 @@ namespace rm.DelegatingHandlersTest;
[TestFixture]
public class ExponentialBackoffWithJitterRetryHandlerTests
{
- private static Exception[] handledExceptions =
+ private static readonly Exception[] handledExceptions =
{
new HttpRequestException(),
new TimeoutExpiredException(),
};
[Test]
- public async Task Retries_On_5xx()
+ [TestCase(500)]
+ [TestCase(501)]
+ [TestCase(502)]
+ [TestCase(503)]
+ [TestCase(504)]
+ [TestCase(542)]
+ public async Task Retries_On_5xx(int statusCode)
{
var fixture = new Fixture().Customize(new AutoMoqCustomization());
- var statusCode = (HttpStatusCode)542;
var content = fixture.Create();
var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
new ShortCircuitingResponseHandlerSettings
{
- StatusCode = statusCode,
+ StatusCode = (HttpStatusCode)statusCode,
Content = content,
});
var retryAttempt = -1;
@@ -35,12 +43,15 @@ public async Task Retries_On_5xx()
retryAttempt++;
return Task.CompletedTask;
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 1,
RetryDelayInMilliseconds = 0,
- });
+ },
+ clockMock.Object);
using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, delegateHandler, shortCircuitingResponseHandler);
@@ -65,12 +76,15 @@ public void Retries_On_Exceptions(Exception handledException)
retryAttempt++;
return Task.CompletedTask;
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 1,
RetryDelayInMilliseconds = 0,
- });
+ },
+ clockMock.Object);
using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, delegateHandler, throwingHandler);
@@ -85,11 +99,24 @@ public void Retries_On_Exceptions(Exception handledException)
}
[Test]
- public void Does_Not_Retries_On_TaskCanceledException()
+ [TestCase(400)]
+ [TestCase(401)]
+ [TestCase(402)]
+ [TestCase(403)]
+ [TestCase(404)]
+ [TestCase(420)] // calm down
+ [TestCase(442)]
+ public async Task Does_Not_Retry_On_4xx(int statusCode)
{
var fixture = new Fixture().Customize(new AutoMoqCustomization());
- var throwingHandler = new ThrowingHandler(new TaskCanceledException());
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
var retryAttempt = -1;
var delegateHandler = new DelegateHandler(
(request, ct) =>
@@ -97,12 +124,47 @@ public void Does_Not_Retries_On_TaskCanceledException()
retryAttempt++;
return Task.CompletedTask;
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 1,
RetryDelayInMilliseconds = 0,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(0, retryAttempt);
+ }
+
+ [Test]
+ public void Does_Not_Retry_On_TaskCanceledException()
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var throwingHandler = new ThrowingHandler(new TaskCanceledException());
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 0,
+ },
+ clockMock.Object);
using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, delegateHandler, throwingHandler);
@@ -129,12 +191,14 @@ public async Task When_0_Retries_PollyRetryAttempt_Property_Is_Not_Present()
StatusCode = statusCode,
Content = content,
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 0,
RetryDelayInMilliseconds = 0,
- });
+ }, clockMock.Object);
using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, shortCircuitingResponseHandler);
@@ -162,12 +226,15 @@ public async Task When_N_Retries_PollyRetryAttempt_Property_Is_Present(int retry
StatusCode = statusCode,
Content = content,
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = retryCount,
RetryDelayInMilliseconds = 0,
- });
+ },
+ clockMock.Object);
using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, shortCircuitingResponseHandler);
@@ -179,4 +246,325 @@ public async Task When_N_Retries_PollyRetryAttempt_Property_Is_Present(int retry
Assert.AreEqual(retryCount, requestMessage.Properties[RequestProperties.PollyRetryAttempt]);
#pragma warning restore CS0618 // Type or member is obsolete
}
+
+ [Test]
+ [TestCase(503)]
+ [TestCase(429)]
+ public async Task Retries_When_RetryAfter_Date_Header(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var date = DateTimeOffsetValues.Chernobyl.AddSeconds(0);
+ var retryAfterDateHandler = new RetryAfterDateHandler(date);
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 0,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, retryAfterDateHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(1, retryAttempt);
+ }
+
+ [Test]
+ [TestCase(503)]
+ [TestCase(429)]
+ public async Task Retries_When_RetryAfter_Delay_Header(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var delayInSeconds = 0;
+ var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds);
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 0,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(1, retryAttempt);
+ }
+
+ [Test]
+ [TestCase(503)]
+ [TestCase(429)]
+ public async Task Does_Not_Retry_Retries_When_RetryAfter_Date_Header_High(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var date = DateTimeOffsetValues.Chernobyl.AddSeconds(5);
+ var retryAfterDateHandler = new RetryAfterDateHandler(date);
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 0,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, retryAfterDateHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(0, retryAttempt);
+ }
+
+ [Test]
+ [TestCase(503)]
+ [TestCase(429)]
+ public async Task Does_Not_Retry_Retries_When_RetryAfter_Delay_Header_High(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var delayInSeconds = 5;
+ var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds);
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 0,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(0, retryAttempt);
+ }
+
+ [Explicit]
+ [Test]
+ public void Print_SleepDurations()
+ {
+ var retryDelayInMilliseconds = 500;
+ var retryCount = 5;
+ var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2(
+ medianFirstRetryDelay: TimeSpan.FromMilliseconds(retryDelayInMilliseconds),
+ retryCount: retryCount);
+
+ Console.WriteLine($"retryCount: {retryCount}");
+ foreach (var sleepDurationWithJitter in sleepDurationsWithJitter)
+ {
+ Console.WriteLine(sleepDurationWithJitter);
+ }
+ }
+
+ [Explicit]
+ [Test]
+ [TestCase(503)]
+ [TestCase(429)]
+ [TestCase(500)] // DNC for retry-after
+ public async Task Showcase_Retries_With_RetryAfter_Delay_Header(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var delayInSeconds = 1;
+ var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds);
+ var retryAttempt = -1;
+ var tsPrevious = default(TimeSpan);
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ var tsCurrent = DateTime.Now.TimeOfDay;
+ var delta = (tsCurrent - (tsPrevious != default ? tsPrevious : tsCurrent)).TotalMilliseconds;
+ tsPrevious = tsCurrent;
+ retryAttempt++;
+ Console.WriteLine($"[{tsCurrent.ToString(@"hh\:mm\:ss\.fff")}] making attempt: {retryAttempt}, delta: {delta,7:F0}");
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 3,
+ RetryDelayInMilliseconds = 500,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler);
+
+ Console.WriteLine($"retry-after: {delayInSeconds}");
+ using var requestMessage = fixture.Create();
+ Console.WriteLine($"[{DateTime.Now.TimeOfDay.ToString(@"hh\:mm\:ss\.fff")}] starting");
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+ Console.WriteLine($"[{DateTime.Now.TimeOfDay.ToString(@"hh\:mm\:ss\.fff")}] ending");
+ Console.WriteLine($"retry-attempt: {retryAttempt}");
+ }
+
+ [Explicit]
+ [Repeat(1_000)]
+ [Test]
+ [TestCase(503)]
+ [TestCase(429)]
+ public async Task Perf_Retries_When_RetryAfter_Delay_Header(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var delayInSeconds = 0;
+ var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds);
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 10,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(1, retryAttempt);
+ }
+
+ [Explicit]
+ [Repeat(1_000)]
+ [Test]
+ [TestCase(503)]
+ public async Task Perf_Retries_When_No_RetryAfter_Delay_Header(int statusCode)
+ {
+ var fixture = new Fixture().Customize(new AutoMoqCustomization());
+
+ var content = fixture.Create();
+ var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler(
+ new ShortCircuitingResponseHandlerSettings
+ {
+ StatusCode = (HttpStatusCode)statusCode,
+ Content = content,
+ });
+ var retryAttempt = -1;
+ var delegateHandler = new DelegateHandler(
+ (request, ct) =>
+ {
+ retryAttempt++;
+ return Task.CompletedTask;
+ });
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
+ var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
+ new RetrySettings
+ {
+ RetryCount = 1,
+ RetryDelayInMilliseconds = 10,
+ },
+ clockMock.Object);
+
+ using var invoker = HttpMessageInvokerFactory.Create(
+ retryHandler, delegateHandler, shortCircuitingResponseHandler);
+
+ using var requestMessage = fixture.Create();
+ using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None);
+
+ Assert.AreEqual(1, retryAttempt);
+ }
}
diff --git a/tests/rm.DelegatingHandlersTest/ScenarioTests.cs b/tests/rm.DelegatingHandlersTest/ScenarioTests.cs
index 2630ce7..3ec98dc 100644
--- a/tests/rm.DelegatingHandlersTest/ScenarioTests.cs
+++ b/tests/rm.DelegatingHandlersTest/ScenarioTests.cs
@@ -1,6 +1,8 @@
using AutoFixture;
using AutoFixture.AutoMoq;
+using Moq;
using NUnit.Framework;
+using rm.Clock;
using rm.DelegatingHandlers;
namespace rm.DelegatingHandlersTest;
@@ -30,12 +32,15 @@ public void Retry_To_Fix_Infrequent_TaskCanceledException_Using_HttpMessageInvok
{
TimeoutInMilliseconds = 10,
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 1,
RetryDelayInMilliseconds = 0,
- });
+ },
+ clockMock.Object);
using var invoker = HttpMessageInvokerFactory.Create(
retryHandler, timeoutHandler, delegateHandler, procrastinatingHandler);
@@ -75,12 +80,15 @@ public void Retry_To_Fix_Infrequent_TaskCanceledException()
{
TimeoutInMilliseconds = 10,
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 1,
RetryDelayInMilliseconds = 0,
- });
+ },
+ clockMock.Object);
using var httpClient = HttpClientFactory.Create(
retryHandler, timeoutHandler, delegateHandler, procrastinatingHandler);
@@ -126,12 +134,15 @@ public void Retry_With_Higher_Timeout_Does_Not_Throw_TimeoutExpiredException()
{
TimeoutInMilliseconds = 10_000,
});
+ var clockMock = fixture.Freeze>();
+ clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl);
var retryHandler = new ExponentialBackoffWithJitterRetryHandler(
new RetrySettings
{
RetryCount = 1,
RetryDelayInMilliseconds = 0,
- });
+ },
+ clockMock.Object);
using var httpClient = HttpClientFactory.Create(
retryHandler, timeoutHandler, delegateHandler, procrastinatingHandler);
diff --git a/tests/rm.DelegatingHandlersTest/misc/DateTimeOffsetValues.cs b/tests/rm.DelegatingHandlersTest/misc/DateTimeOffsetValues.cs
new file mode 100644
index 0000000..e78bf28
--- /dev/null
+++ b/tests/rm.DelegatingHandlersTest/misc/DateTimeOffsetValues.cs
@@ -0,0 +1,6 @@
+namespace rm.DelegatingHandlersTest;
+
+public class DateTimeOffsetValues
+{
+ public static readonly DateTimeOffset Chernobyl = new DateTimeOffset(1986, 4, 26, 01, 23, 04, TimeSpan.FromHours(4));
+}