ClawdDotNet/tests/ClawdDotNet.Core.Tests/Api/RetryTests.cs

using System.Net;
using ClawdDotNet.Core.Api;
using ClawdDotNet.Core.Api.Models;
using ClawdDotNet.Core.Tests.Infrastructure;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Api;

/// <summary>
/// B12 from the inventory: the client threw immediately on every non-2xx. A
/// single HTTP 429 thus ended a complete scheduled run, although
/// rate limits and brief 5xx are normal operation at OpenRouter.
/// </summary>
public sealed class RetryPolicyTests
{
    [Theory]
    [InlineData(HttpStatusCode.TooManyRequests, true)]
    [InlineData(HttpStatusCode.RequestTimeout, true)]
    [InlineData(HttpStatusCode.InternalServerError, true)]
    [InlineData(HttpStatusCode.BadGateway, true)]
    [InlineData(HttpStatusCode.ServiceUnavailable, true)]
    [InlineData(HttpStatusCode.GatewayTimeout, true)]
    [InlineData(HttpStatusCode.BadRequest, false)]
    [InlineData(HttpStatusCode.Unauthorized, false)]
    [InlineData(HttpStatusCode.Forbidden, false)]
    [InlineData(HttpStatusCode.NotFound, false)]
    [InlineData(HttpStatusCode.Conflict, false)]
    public void Only_temporary_disruptions_are_retried(HttpStatusCode status, bool expected)
    {
        // A 400 or 401 does not repeat itself by itself — waiting for it
        // only delays the error message.
        RetryPolicy.Default.ShouldRetry(status).ShouldBe(expected);
    }

    [Fact]
    public void The_wait_time_grows_exponentially()
    {
        var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(1), JitterFactor = 0 };

        policy.GetDelay(1).ShouldBe(TimeSpan.FromSeconds(1));
        policy.GetDelay(2).ShouldBe(TimeSpan.FromSeconds(2));
        policy.GetDelay(3).ShouldBe(TimeSpan.FromSeconds(4));
        policy.GetDelay(4).ShouldBe(TimeSpan.FromSeconds(8));
    }

    [Fact]
    public void The_wait_time_is_capped()
    {
        var policy = new RetryPolicy
        {
            BaseDelay = TimeSpan.FromSeconds(1),
            MaxDelay = TimeSpan.FromSeconds(10),
            JitterFactor = 0
        };

        policy.GetDelay(20).ShouldBe(TimeSpan.FromSeconds(10));
    }

    [Fact]
    public void A_server_RetryAfter_takes_precedence()
    {
        var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(1), JitterFactor = 0 };

        policy.GetDelay(1, TimeSpan.FromSeconds(7)).ShouldBe(TimeSpan.FromSeconds(7));
    }

    [Fact]
    public void A_RetryAfter_is_capped_too()
    {
        // A server could demand hours — no run may block that long.
        var policy = new RetryPolicy { MaxDelay = TimeSpan.FromSeconds(30) };

        policy.GetDelay(1, TimeSpan.FromHours(1)).ShouldBe(TimeSpan.FromSeconds(30));
    }

    [Theory]
    [InlineData(0.0)]
    [InlineData(0.5)]
    [InlineData(1.0)]
    public void The_spread_stays_within_the_expected_range(double sample)
    {
        var policy = new RetryPolicy { BaseDelay = TimeSpan.FromSeconds(4), JitterFactor = 0.25 };

        var delay = policy.GetDelay(1, null, sample);

        delay.TotalSeconds.ShouldBeInRange(3.0, 5.0);
    }
}

public sealed class OpenRouterClientRetryTests
{
    private static ChatRequest SimpleRequest() => new()
    {
        Model = "test/model",
        Messages = [ChatMessage.User("Hallo")]
    };

    private const string SuccessBody =
        """{"id":"x","choices":[{"index":0,"message":{"role":"assistant","content":"Hi"}}]}""";

    /// <summary>Does not really wait — otherwise the suite would take seconds.</summary>
    private static Task NoWait(TimeSpan _, CancellationToken __) => Task.CompletedTask;

    private static OpenRouterClient CreateClient(FakeHttpMessageHandler handler, RetryPolicy? policy = null)
    {
        var http = new HttpClient(handler) { BaseAddress = new Uri("https://openrouter.ai/api/v1/") };
        return new OpenRouterClient("sk-test", NullLogger.Instance, http, policy ?? RetryPolicy.Default, NoWait);
    }

    [Fact]
    public async Task A_rate_limit_no_longer_ends_the_run()
    {
        var handler = new FakeHttpMessageHandler()
            .Responds(HttpStatusCode.TooManyRequests, """{"error":"rate limit"}""")
            .RespondsWithSuccess(SuccessBody);

        using var client = CreateClient(handler);

        var response = await client.CompleteAsync(SimpleRequest(), default);

        response.Choices[0].Message.Content.ShouldBe("Hi");
        handler.RequestCount.ShouldBe(2);
    }

    [Fact]
    public async Task Several_disruptions_in_a_row_are_bridged()
    {
        var handler = new FakeHttpMessageHandler()
            .Responds(HttpStatusCode.ServiceUnavailable)
            .Responds(HttpStatusCode.BadGateway)
            .RespondsWithSuccess(SuccessBody);

        using var client = CreateClient(handler);

        await client.CompleteAsync(SimpleRequest(), default);

        handler.RequestCount.ShouldBe(3);
    }

    [Fact]
    public async Task After_the_last_attempt_the_error_is_reported()
    {
        var policy = new RetryPolicy { MaxAttempts = 3 };
        var handler = new FakeHttpMessageHandler()
            .Responds(HttpStatusCode.TooManyRequests)
            .Responds(HttpStatusCode.TooManyRequests)
            .Responds(HttpStatusCode.TooManyRequests);

        using var client = CreateClient(handler, policy);

        var ex = await Should.ThrowAsync<OpenRouterException>(
            () => client.CompleteAsync(SimpleRequest(), default));

        ex.StatusCode.ShouldBe(429);
        handler.RequestCount.ShouldBe(3);
    }

    [Fact]
    public async Task A_permanent_error_is_not_retried()
    {
        // A 401 does not get better by waiting — report immediately.
        var handler = new FakeHttpMessageHandler()
            .Responds(HttpStatusCode.Unauthorized, """{"error":"invalid key"}""");

        using var client = CreateClient(handler);

        await Should.ThrowAsync<OpenRouterException>(
            () => client.CompleteAsync(SimpleRequest(), default));

        handler.RequestCount.ShouldBe(1);
    }

    [Fact]
    public async Task Network_errors_are_retried()
    {
        var handler = new FakeHttpMessageHandler()
            .Throws(new HttpRequestException("Verbindung abgebrochen"))
            .RespondsWithSuccess(SuccessBody);

        using var client = CreateClient(handler);

        await client.CompleteAsync(SimpleRequest(), default);

        handler.RequestCount.ShouldBe(2);
    }

    [Fact]
    public async Task An_abort_by_the_user_is_not_retried()
    {
        var handler = new FakeHttpMessageHandler().Responds(HttpStatusCode.TooManyRequests);
        using var client = CreateClient(handler);

        using var cts = new CancellationTokenSource();
        await cts.CancelAsync();

        await Should.ThrowAsync<OperationCanceledException>(
            () => client.CompleteAsync(SimpleRequest(), cts.Token));

        handler.RequestCount.ShouldBe(0);
    }

    [Fact]
    public async Task A_successful_first_attempt_triggers_no_retry()
    {
        var handler = new FakeHttpMessageHandler().RespondsWithSuccess(SuccessBody);
        using var client = CreateClient(handler);

        await client.CompleteAsync(SimpleRequest(), default);

        handler.RequestCount.ShouldBe(1);
    }
}