using System.Net; using ClawdDotNet.Core.Api; using ClawdDotNet.Core.Api.Models; using ClawdDotNet.Core.Tests.Infrastructure; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; namespace ClawdDotNet.Core.Tests.Api; /// /// B12 from the inventory: the client threw immediately on every non-2xx. A /// single HTTP 429 thus ended a complete scheduled run, although /// rate limits and brief 5xx are normal operation at OpenRouter. /// public sealed class RetryPolicyTests { [Theory] [InlineData(HttpStatusCode.TooManyRequests, true)] [InlineData(HttpStatusCode.RequestTimeout, true)] [InlineData(HttpStatusCode.InternalServerError, true)] [InlineData(HttpStatusCode.BadGateway, true)] [InlineData(HttpStatusCode.ServiceUnavailable, true)] [InlineData(HttpStatusCode.GatewayTimeout, true)] [InlineData(HttpStatusCode.BadRequest, false)] [InlineData(HttpStatusCode.Unauthorized, false)] [InlineData(HttpStatusCode.Forbidden, false)] [InlineData(HttpStatusCode.NotFound, false)] [InlineData(HttpStatusCode.Conflict, false)] public void Only_temporary_disruptions_are_retried(HttpStatusCode status, bool expected) { // A 400 or 401 does not repeat itself by itself — waiting for it // only delays the error message. RetryPolicy.Default.ShouldRetry(status).ShouldBe(expected); } [Fact] public void The_wait_time_grows_exponentially() { var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(1), JitterFactor = 0 }; policy.GetDelay(1).ShouldBe(TimeSpan.FromSeconds(1)); policy.GetDelay(2).ShouldBe(TimeSpan.FromSeconds(2)); policy.GetDelay(3).ShouldBe(TimeSpan.FromSeconds(4)); policy.GetDelay(4).ShouldBe(TimeSpan.FromSeconds(8)); } [Fact] public void The_wait_time_is_capped() { var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(1), MaxDelay = TimeSpan.FromSeconds(10), JitterFactor = 0 }; policy.GetDelay(20).ShouldBe(TimeSpan.FromSeconds(10)); } [Fact] public void A_server_RetryAfter_takes_precedence() { var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(1), JitterFactor = 0 }; policy.GetDelay(1, TimeSpan.FromSeconds(7)).ShouldBe(TimeSpan.FromSeconds(7)); } [Fact] public void A_RetryAfter_is_capped_too() { // A server could demand hours — no run may block that long. var policy = new RetryPolicy { MaxDelay = TimeSpan.FromSeconds(30) }; policy.GetDelay(1, TimeSpan.FromHours(1)).ShouldBe(TimeSpan.FromSeconds(30)); } [Theory] [InlineData(0.0)] [InlineData(0.5)] [InlineData(1.0)] public void The_spread_stays_within_the_expected_range(double sample) { var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(4), JitterFactor = 0.25 }; var delay = policy.GetDelay(1, null, sample); delay.TotalSeconds.ShouldBeInRange(3.0, 5.0); } } public sealed class OpenRouterClientRetryTests { private static ChatRequest SimpleRequest() => new() { Model = "test/model", Messages = [ChatMessage.User("Hallo")] }; private const string SuccessBody = """{"id":"x","choices":[{"index":0,"message":{"role":"assistant","content":"Hi"}}]}"""; /// Does not really wait — otherwise the suite would take seconds. private static Task NoWait(TimeSpan _, CancellationToken __) => Task.CompletedTask; private static OpenRouterClient CreateClient(FakeHttpMessageHandler handler, RetryPolicy? policy = null) { var http = new HttpClient(handler) { BaseAddress = new Uri("https://openrouter.ai/api/v1/") }; return new OpenRouterClient("sk-test", NullLogger.Instance, http, policy ?? RetryPolicy.Default, NoWait); } [Fact] public async Task A_rate_limit_no_longer_ends_the_run() { var handler = new FakeHttpMessageHandler() .Responds(HttpStatusCode.TooManyRequests, """{"error":"rate limit"}""") .RespondsWithSuccess(SuccessBody); using var client = CreateClient(handler); var response = await client.CompleteAsync(SimpleRequest(), default); response.Choices[0].Message.Content.ShouldBe("Hi"); handler.RequestCount.ShouldBe(2); } [Fact] public async Task Several_disruptions_in_a_row_are_bridged() { var handler = new FakeHttpMessageHandler() .Responds(HttpStatusCode.ServiceUnavailable) .Responds(HttpStatusCode.BadGateway) .RespondsWithSuccess(SuccessBody); using var client = CreateClient(handler); await client.CompleteAsync(SimpleRequest(), default); handler.RequestCount.ShouldBe(3); } [Fact] public async Task After_the_last_attempt_the_error_is_reported() { var policy = new RetryPolicy { MaxAttempts = 3 }; var handler = new FakeHttpMessageHandler() .Responds(HttpStatusCode.TooManyRequests) .Responds(HttpStatusCode.TooManyRequests) .Responds(HttpStatusCode.TooManyRequests); using var client = CreateClient(handler, policy); var ex = await Should.ThrowAsync( () => client.CompleteAsync(SimpleRequest(), default)); ex.StatusCode.ShouldBe(429); handler.RequestCount.ShouldBe(3); } [Fact] public async Task A_permanent_error_is_not_retried() { // A 401 does not get better by waiting — report immediately. var handler = new FakeHttpMessageHandler() .Responds(HttpStatusCode.Unauthorized, """{"error":"invalid key"}"""); using var client = CreateClient(handler); await Should.ThrowAsync( () => client.CompleteAsync(SimpleRequest(), default)); handler.RequestCount.ShouldBe(1); } [Fact] public async Task Network_errors_are_retried() { var handler = new FakeHttpMessageHandler() .Throws(new HttpRequestException("Verbindung abgebrochen")) .RespondsWithSuccess(SuccessBody); using var client = CreateClient(handler); await client.CompleteAsync(SimpleRequest(), default); handler.RequestCount.ShouldBe(2); } [Fact] public async Task An_abort_by_the_user_is_not_retried() { var handler = new FakeHttpMessageHandler().Responds(HttpStatusCode.TooManyRequests); using var client = CreateClient(handler); using var cts = new CancellationTokenSource(); await cts.CancelAsync(); await Should.ThrowAsync( () => client.CompleteAsync(SimpleRequest(), cts.Token)); handler.RequestCount.ShouldBe(0); } [Fact] public async Task A_successful_first_attempt_triggers_no_retry() { var handler = new FakeHttpMessageHandler().RespondsWithSuccess(SuccessBody); using var client = CreateClient(handler); await client.CompleteAsync(SimpleRequest(), default); handler.RequestCount.ShouldBe(1); } }