diff --git a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs index 67b5c6b..5560c7c 100644 --- a/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs +++ b/src/rm.DelegatingHandlers/ExponentialBackoffWithJitterRetryHandler.cs @@ -1,10 +1,13 @@ using System; +using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Polly; using Polly.Contrib.WaitAndRetry; using Polly.Retry; +using rm.Clock; +using rm.Extensions; namespace rm.DelegatingHandlers; @@ -12,38 +15,50 @@ namespace rm.DelegatingHandlers; /// Retries on certain conditions with exponential backoff jitter (DecorrelatedJitterBackoffV2). /// /// Retry conditions: -/// HttpRequestException, 5xx. +/// HttpRequestException, 5xx, 429 (see retry-after header below). +///
+/// retry-after header: +///
+/// For 503: retry honoring header if present, else retry as usual. +///
+/// For 429: retry honoring header only if present, else do not retry. /// /// -/// source +/// retry with jitter +///
+/// retry-after /// public class ExponentialBackoffWithJitterRetryHandler : DelegatingHandler { - private readonly AsyncRetryPolicy retryPolicy; + private readonly AsyncRetryPolicy<(HttpResponseMessage response, Context Context)> retryPolicy; + private readonly IRetrySettings retrySettings; + private readonly ISystemClock clock; /// public ExponentialBackoffWithJitterRetryHandler( - IRetrySettings retrySettings) + IRetrySettings retrySettings, + ISystemClock clock) { - _ = retrySettings + this.retrySettings = retrySettings ?? throw new ArgumentNullException(nameof(retrySettings)); - - var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2( - medianFirstRetryDelay: TimeSpan.FromMilliseconds(retrySettings.RetryDelayInMilliseconds), - retryCount: retrySettings.RetryCount); + this.clock = clock + ?? throw new ArgumentNullException(nameof(clock)); // note: response can't be null // ref: https://github.com/dotnet/runtime/issues/19925#issuecomment-272664671 retryPolicy = Policy .Handle() .Or() - .OrResult(response => response.Is5xx()) + .OrResult<(HttpResponseMessage response, Context context)>( + tuple => CanRetry(tuple.response, tuple.context)) .WaitAndRetryAsync( - sleepDurations: sleepDurationsWithJitter, + retryCount: retrySettings.RetryCount, + sleepDurationProvider: (retryAttempt, responseResult, context) => + ((TimeSpan[])context[ContextKey.SleepDurations])[retryAttempt - 1], onRetry: (responseResult, delay, retryAttempt, context) => { // note: response can be null in case of handled exception - responseResult.Result?.Dispose(); + responseResult.Result.response?.Dispose(); context[ContextKey.RetryAttempt] = retryAttempt; }); } @@ -52,19 +67,112 @@ protected override async Task SendAsync( HttpRequestMessage request, CancellationToken cancellationToken) { - return await retryPolicy.ExecuteAsync( + // read the retry delays upfront + var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2( + medianFirstRetryDelay: TimeSpan.FromMilliseconds(retrySettings.RetryDelayInMilliseconds), + retryCount: retrySettings.RetryCount).ToArray(); + var context = new Context(); + context[ContextKey.SleepDurations] = sleepDurationsWithJitter; + + var tuple = await retryPolicy.ExecuteAsync( action: async (context, ct) => { if (context.TryGetValue(ContextKey.RetryAttempt, out var retryAttempt)) { request.Properties[RequestProperties.PollyRetryAttempt] = retryAttempt; } - return await base.SendAsync(request, ct) + var response = await base.SendAsync(request, ct) .ConfigureAwait(false); + return (response, context); }, - context: new Context(), + context: context, cancellationToken: cancellationToken) .ConfigureAwait(false); + return tuple.response; + } + + /// + /// Returns true if the response can be retried considering things as, + /// retry attempt, and retry-after header (if present). + /// + private bool CanRetry( + HttpResponseMessage response, + Context context) + { + var sleepDurationsWithJitter = (TimeSpan[])context[ContextKey.SleepDurations]; + if (sleepDurationsWithJitter.IsEmpty()) + { + return false; + } + // retryAttempt is 0-based + var retryAttempt = context.TryGetValue(ContextKey.RetryAttempt, out object retryAttemptObj) ? (int)retryAttemptObj : 0; + if (retryAttempt == sleepDurationsWithJitter.Count()) + { + return false; + } + var sleepDurationWithJitter = sleepDurationsWithJitter[retryAttempt]; + + var statusCode = (int)response.StatusCode; + + // here be dragons + var retry = false; + // retry on 5xx, 429 only + if (response.Is5xx() || statusCode == 429) + { + // retry on 503, 429 looking at retry-after value + if (statusCode == 503 || statusCode == 429) + { + // note: look at retry-after value but don't use it to avoid surges at same time; + // use it to determine whether to retry or not + var isRetryAfterPresent = response.Headers.TryGetValue(ResponseHeaders.RetryAfter, out var retryAfterValue) + && retryAfterValue != null; + +#if DEBUG + Console.WriteLine($"retryAfterValue: {retryAfterValue}"); +#endif + + // retry on 503, 429 only on valid retry-after value + if (isRetryAfterPresent) + { + TimeSpan retryAfter; + retry = + ((double.TryParse(retryAfterValue, out double retryAfterDelay) + // ignore network latency, delay could be 0 + && Math.Max((retryAfter = TimeSpan.FromSeconds(retryAfterDelay)).TotalSeconds, 0) >= 0) + || + (DateTimeOffset.TryParse(retryAfterValue, out DateTimeOffset retryAfterDate) + // date could be in the past due to network latency + && Math.Max((retryAfter = retryAfterDate - clock.UtcNow).TotalSeconds, 0) >= 0)) + // only retry if delay is at or above retry-after value + && retryAfter <= sleepDurationWithJitter; + } + else + { + // retry on 503 if retry-after not present as typical + if (statusCode == 503) + { + retry = true; + } + // do NOT retry on 429 if retry-after not present as typical + else if (statusCode == 429) + { + retry = false; + } + } + } + else + { + // retry on 5xx (other than 503) as typical + retry = true; + } + } + +#if DEBUG + Console.WriteLine($"sleepDurationWithJitter: {sleepDurationWithJitter}"); + Console.WriteLine($"retry: {retry}"); +#endif + + return retry; } } @@ -83,4 +191,5 @@ public record class RetrySettings : IRetrySettings internal static class ContextKey { internal const string RetryAttempt = nameof(RetryAttempt); + internal const string SleepDurations = nameof(SleepDurations); } diff --git a/src/rm.DelegatingHandlers/misc/AsyncRetryTResultSyntax.cs b/src/rm.DelegatingHandlers/misc/AsyncRetryTResultSyntax.cs new file mode 100644 index 0000000..423b3b9 --- /dev/null +++ b/src/rm.DelegatingHandlers/misc/AsyncRetryTResultSyntax.cs @@ -0,0 +1,41 @@ +using System; +using Polly.Retry; + +namespace Polly; + +public static class AsyncRetryTResultSyntax +{ + /// + /// Builds an that will wait and retry times + /// calling on each retry with the handled exception or result, the current sleep duration, retry count, and context data. + /// On each retry, the duration to wait is calculated by calling with + /// the current retry number (1 for first retry, 2 for second etc), result of previous execution, and execution context. + /// + /// The policy builder. + /// The retry count. + /// The function that provides the duration to wait for for a particular retry attempt. + /// The action to call on each retry. + /// The policy instance. + /// retryCount;Value must be greater than or equal to zero. + /// + /// sleepDurationProvider + /// or + /// onRetryAsync + /// + /// + /// issue: https://github.com/App-vNext/Polly/issues/908 + /// + public static AsyncRetryPolicy WaitAndRetryAsync(this PolicyBuilder policyBuilder, int retryCount, + Func, Context, TimeSpan> sleepDurationProvider, Action, TimeSpan, int, Context> onRetry) + { + if (onRetry == null) throw new ArgumentNullException(nameof(onRetry)); + + return policyBuilder.WaitAndRetryAsync( + retryCount, + sleepDurationProvider, +#pragma warning disable 1998 // async method has no awaits, will run synchronously + onRetryAsync: async (outcome, timespan, i, ctx) => onRetry(outcome, timespan, i, ctx) +#pragma warning restore 1998 + ); + } +} diff --git a/src/rm.DelegatingHandlers/misc/HttpStatusCodeIntExtensions.cs b/src/rm.DelegatingHandlers/misc/HttpStatusCodeIntExtensions.cs new file mode 100644 index 0000000..2e9b53d --- /dev/null +++ b/src/rm.DelegatingHandlers/misc/HttpStatusCodeIntExtensions.cs @@ -0,0 +1,55 @@ +using System.Net; + +namespace rm.DelegatingHandlers; + +public static class HttpStatusCodeIntExtensions +{ + public static bool Is1xx(this int statusCode) + { + return ((HttpStatusCode)statusCode).Is1xx(); + } + + public static bool Is2xx(this int statusCode) + { + return ((HttpStatusCode)statusCode).Is2xx(); + } + + public static bool Is3xx(this int statusCode) + { + return ((HttpStatusCode)statusCode).Is3xx(); + } + + public static bool Is4xx(this int statusCode) + { + return ((HttpStatusCode)statusCode).Is4xx(); + } + + public static bool Is5xx(this int statusCode) + { + return ((HttpStatusCode)statusCode).Is5xx(); + } + + /// + /// Returns true if status code is a client error status code (4xx). + /// + public static bool IsClientErrorStatusCode(this int statusCode) + { + return statusCode.Is4xx(); + } + + /// + /// Returns true if status code is a server error status code (5xx). + /// + public static bool IsServerErrorStatusCode(this int statusCode) + { + return statusCode.Is5xx(); + } + + /// + /// Returns true if status code is an error status code (4xx, 5xx). + /// + public static bool IsErrorStatusCode(this int statusCode) + { + return statusCode.Is4xx() || statusCode.Is5xx(); + } +} diff --git a/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj b/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj index 871b968..53f7217 100644 --- a/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj +++ b/src/rm.DelegatingHandlers/rm.DelegatingHandlers.csproj @@ -30,6 +30,7 @@ + diff --git a/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs b/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs index 1ca8915..6cd6a99 100644 --- a/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs +++ b/tests/rm.DelegatingHandlersTest/ExponentialBackoffWithJitterRetryHandlerTests.cs @@ -1,7 +1,10 @@ using System.Net; using AutoFixture; using AutoFixture.AutoMoq; +using Moq; using NUnit.Framework; +using Polly.Contrib.WaitAndRetry; +using rm.Clock; using rm.DelegatingHandlers; namespace rm.DelegatingHandlersTest; @@ -9,23 +12,28 @@ namespace rm.DelegatingHandlersTest; [TestFixture] public class ExponentialBackoffWithJitterRetryHandlerTests { - private static Exception[] handledExceptions = + private static readonly Exception[] handledExceptions = { new HttpRequestException(), new TimeoutExpiredException(), }; [Test] - public async Task Retries_On_5xx() + [TestCase(500)] + [TestCase(501)] + [TestCase(502)] + [TestCase(503)] + [TestCase(504)] + [TestCase(542)] + public async Task Retries_On_5xx(int statusCode) { var fixture = new Fixture().Customize(new AutoMoqCustomization()); - var statusCode = (HttpStatusCode)542; var content = fixture.Create(); var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( new ShortCircuitingResponseHandlerSettings { - StatusCode = statusCode, + StatusCode = (HttpStatusCode)statusCode, Content = content, }); var retryAttempt = -1; @@ -35,12 +43,15 @@ public async Task Retries_On_5xx() retryAttempt++; return Task.CompletedTask; }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 1, RetryDelayInMilliseconds = 0, - }); + }, + clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( retryHandler, delegateHandler, shortCircuitingResponseHandler); @@ -65,12 +76,15 @@ public void Retries_On_Exceptions(Exception handledException) retryAttempt++; return Task.CompletedTask; }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 1, RetryDelayInMilliseconds = 0, - }); + }, + clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( retryHandler, delegateHandler, throwingHandler); @@ -85,11 +99,24 @@ public void Retries_On_Exceptions(Exception handledException) } [Test] - public void Does_Not_Retries_On_TaskCanceledException() + [TestCase(400)] + [TestCase(401)] + [TestCase(402)] + [TestCase(403)] + [TestCase(404)] + [TestCase(420)] // calm down + [TestCase(442)] + public async Task Does_Not_Retry_On_4xx(int statusCode) { var fixture = new Fixture().Customize(new AutoMoqCustomization()); - var throwingHandler = new ThrowingHandler(new TaskCanceledException()); + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); var retryAttempt = -1; var delegateHandler = new DelegateHandler( (request, ct) => @@ -97,12 +124,47 @@ public void Does_Not_Retries_On_TaskCanceledException() retryAttempt++; return Task.CompletedTask; }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 1, RetryDelayInMilliseconds = 0, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(0, retryAttempt); + } + + [Test] + public void Does_Not_Retry_On_TaskCanceledException() + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var throwingHandler = new ThrowingHandler(new TaskCanceledException()); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 0, + }, + clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( retryHandler, delegateHandler, throwingHandler); @@ -129,12 +191,14 @@ public async Task When_0_Retries_PollyRetryAttempt_Property_Is_Not_Present() StatusCode = statusCode, Content = content, }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 0, RetryDelayInMilliseconds = 0, - }); + }, clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( retryHandler, shortCircuitingResponseHandler); @@ -162,12 +226,15 @@ public async Task When_N_Retries_PollyRetryAttempt_Property_Is_Present(int retry StatusCode = statusCode, Content = content, }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = retryCount, RetryDelayInMilliseconds = 0, - }); + }, + clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( retryHandler, shortCircuitingResponseHandler); @@ -179,4 +246,325 @@ public async Task When_N_Retries_PollyRetryAttempt_Property_Is_Present(int retry Assert.AreEqual(retryCount, requestMessage.Properties[RequestProperties.PollyRetryAttempt]); #pragma warning restore CS0618 // Type or member is obsolete } + + [Test] + [TestCase(503)] + [TestCase(429)] + public async Task Retries_When_RetryAfter_Date_Header(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var date = DateTimeOffsetValues.Chernobyl.AddSeconds(0); + var retryAfterDateHandler = new RetryAfterDateHandler(date); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 0, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retryAfterDateHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(1, retryAttempt); + } + + [Test] + [TestCase(503)] + [TestCase(429)] + public async Task Retries_When_RetryAfter_Delay_Header(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var delayInSeconds = 0; + var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 0, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(1, retryAttempt); + } + + [Test] + [TestCase(503)] + [TestCase(429)] + public async Task Does_Not_Retry_Retries_When_RetryAfter_Date_Header_High(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var date = DateTimeOffsetValues.Chernobyl.AddSeconds(5); + var retryAfterDateHandler = new RetryAfterDateHandler(date); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 0, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retryAfterDateHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(0, retryAttempt); + } + + [Test] + [TestCase(503)] + [TestCase(429)] + public async Task Does_Not_Retry_Retries_When_RetryAfter_Delay_Header_High(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var delayInSeconds = 5; + var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 0, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(0, retryAttempt); + } + + [Explicit] + [Test] + public void Print_SleepDurations() + { + var retryDelayInMilliseconds = 500; + var retryCount = 5; + var sleepDurationsWithJitter = Backoff.DecorrelatedJitterBackoffV2( + medianFirstRetryDelay: TimeSpan.FromMilliseconds(retryDelayInMilliseconds), + retryCount: retryCount); + + Console.WriteLine($"retryCount: {retryCount}"); + foreach (var sleepDurationWithJitter in sleepDurationsWithJitter) + { + Console.WriteLine(sleepDurationWithJitter); + } + } + + [Explicit] + [Test] + [TestCase(503)] + [TestCase(429)] + [TestCase(500)] // DNC for retry-after + public async Task Showcase_Retries_With_RetryAfter_Delay_Header(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var delayInSeconds = 1; + var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds); + var retryAttempt = -1; + var tsPrevious = default(TimeSpan); + var delegateHandler = new DelegateHandler( + (request, ct) => + { + var tsCurrent = DateTime.Now.TimeOfDay; + var delta = (tsCurrent - (tsPrevious != default ? tsPrevious : tsCurrent)).TotalMilliseconds; + tsPrevious = tsCurrent; + retryAttempt++; + Console.WriteLine($"[{tsCurrent.ToString(@"hh\:mm\:ss\.fff")}] making attempt: {retryAttempt}, delta: {delta,7:F0}"); + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 3, + RetryDelayInMilliseconds = 500, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler); + + Console.WriteLine($"retry-after: {delayInSeconds}"); + using var requestMessage = fixture.Create(); + Console.WriteLine($"[{DateTime.Now.TimeOfDay.ToString(@"hh\:mm\:ss\.fff")}] starting"); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + Console.WriteLine($"[{DateTime.Now.TimeOfDay.ToString(@"hh\:mm\:ss\.fff")}] ending"); + Console.WriteLine($"retry-attempt: {retryAttempt}"); + } + + [Explicit] + [Repeat(1_000)] + [Test] + [TestCase(503)] + [TestCase(429)] + public async Task Perf_Retries_When_RetryAfter_Delay_Header(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var delayInSeconds = 0; + var retryAfterDelayHandler = new RetryAfterDelayHandler(delayInSeconds); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 10, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, retryAfterDelayHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(1, retryAttempt); + } + + [Explicit] + [Repeat(1_000)] + [Test] + [TestCase(503)] + public async Task Perf_Retries_When_No_RetryAfter_Delay_Header(int statusCode) + { + var fixture = new Fixture().Customize(new AutoMoqCustomization()); + + var content = fixture.Create(); + var shortCircuitingResponseHandler = new ShortCircuitingResponseHandler( + new ShortCircuitingResponseHandlerSettings + { + StatusCode = (HttpStatusCode)statusCode, + Content = content, + }); + var retryAttempt = -1; + var delegateHandler = new DelegateHandler( + (request, ct) => + { + retryAttempt++; + return Task.CompletedTask; + }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); + var retryHandler = new ExponentialBackoffWithJitterRetryHandler( + new RetrySettings + { + RetryCount = 1, + RetryDelayInMilliseconds = 10, + }, + clockMock.Object); + + using var invoker = HttpMessageInvokerFactory.Create( + retryHandler, delegateHandler, shortCircuitingResponseHandler); + + using var requestMessage = fixture.Create(); + using var _ = await invoker.SendAsync(requestMessage, CancellationToken.None); + + Assert.AreEqual(1, retryAttempt); + } } diff --git a/tests/rm.DelegatingHandlersTest/ScenarioTests.cs b/tests/rm.DelegatingHandlersTest/ScenarioTests.cs index 2630ce7..3ec98dc 100644 --- a/tests/rm.DelegatingHandlersTest/ScenarioTests.cs +++ b/tests/rm.DelegatingHandlersTest/ScenarioTests.cs @@ -1,6 +1,8 @@ using AutoFixture; using AutoFixture.AutoMoq; +using Moq; using NUnit.Framework; +using rm.Clock; using rm.DelegatingHandlers; namespace rm.DelegatingHandlersTest; @@ -30,12 +32,15 @@ public void Retry_To_Fix_Infrequent_TaskCanceledException_Using_HttpMessageInvok { TimeoutInMilliseconds = 10, }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 1, RetryDelayInMilliseconds = 0, - }); + }, + clockMock.Object); using var invoker = HttpMessageInvokerFactory.Create( retryHandler, timeoutHandler, delegateHandler, procrastinatingHandler); @@ -75,12 +80,15 @@ public void Retry_To_Fix_Infrequent_TaskCanceledException() { TimeoutInMilliseconds = 10, }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 1, RetryDelayInMilliseconds = 0, - }); + }, + clockMock.Object); using var httpClient = HttpClientFactory.Create( retryHandler, timeoutHandler, delegateHandler, procrastinatingHandler); @@ -126,12 +134,15 @@ public void Retry_With_Higher_Timeout_Does_Not_Throw_TimeoutExpiredException() { TimeoutInMilliseconds = 10_000, }); + var clockMock = fixture.Freeze>(); + clockMock.Setup(x => x.UtcNow).Returns(DateTimeOffsetValues.Chernobyl); var retryHandler = new ExponentialBackoffWithJitterRetryHandler( new RetrySettings { RetryCount = 1, RetryDelayInMilliseconds = 0, - }); + }, + clockMock.Object); using var httpClient = HttpClientFactory.Create( retryHandler, timeoutHandler, delegateHandler, procrastinatingHandler); diff --git a/tests/rm.DelegatingHandlersTest/misc/DateTimeOffsetValues.cs b/tests/rm.DelegatingHandlersTest/misc/DateTimeOffsetValues.cs new file mode 100644 index 0000000..e78bf28 --- /dev/null +++ b/tests/rm.DelegatingHandlersTest/misc/DateTimeOffsetValues.cs @@ -0,0 +1,6 @@ +namespace rm.DelegatingHandlersTest; + +public class DateTimeOffsetValues +{ + public static readonly DateTimeOffset Chernobyl = new DateTimeOffset(1986, 4, 26, 01, 23, 04, TimeSpan.FromHours(4)); +}