ClawdDotNet/tests/ClawdDotNet.Core.Tests/Engine/ProactiveCompactionTests.cs

using ClawdDotNet.Core.Engine;
using ClawdDotNet.Core.Tests.Infrastructure;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Engine;

/// <summary>
/// #203 — compaction ran after the API call.
///
/// <para>It relied on the <c>promptTokens</c> of the request just answered:
/// by the time the value was available, the overfull prompt had long been billed, and truncation
/// only happened for the next step. Exactly one step too late — and the most expensive one.</para>
///
/// <para>The promise now reads: <b>no request goes beyond the context limit.</b>
/// The check happens before sending, with the estimate; the measured values adjust the
/// estimate afterwards.</para>
/// </summary>
public sealed class ProactiveCompactionTests
{
    private const string AgentModel = "test/model";
    private const string SummaryModel = "test/summary";

    [Fact]
    public async Task No_request_goes_beyond_the_context_limit()
    {
        var fixture = new EngineFixture();
        var agent = fixture.AddAgent("agent-a");

        // Threshold: 1000 estimated tokens.
        agent.LoopGuard.MaxContextTokens = 2_000;
        agent.LoopGuard.CompactionThreshold = 0.5;
        agent.LoopGuard.SummaryModel = SummaryModel;

        fixture.Client.AlwaysRespondsWithText("Verstanden.");

        // Every message brings about 150 estimated tokens; after a few rounds
        // the context would be above the limit without intervention. Small enough that even the
        // six protected tail messages together still fit below the limit -
        // otherwise not even a compaction could get under it any more.
        for (var i = 0; i < 20; i++)
        {
            await fixture.Engine.ChatAsync(
                agent, $"Frage {i}: " + new string('y', 600), "test-instance", default);
        }

        var requestsToAgent = fixture.Client.ReceivedRequests
            .Where(r => r.Model == AgentModel)
            .ToList();

        requestsToAgent.Count.ShouldBe(20);

        // The actual point: not "at some point it was truncated", but "nothing overfull ever
        // went out". Previously the check hung on the promptTokens of the
        // answer — always 100 with the fake —, so it never got over the threshold and never truncated.
        foreach (var request in requestsToAgent)
        {
            ContextCompactor.EstimateTokens(request.Messages)
                .ShouldBeLessThan(agent.LoopGuard.MaxContextTokens);
        }
    }

    [Fact]
    public async Task Shortening_uses_the_cheap_model_before_sending()
    {
        var fixture = new EngineFixture();
        var agent = fixture.AddAgent("agent-a");

        agent.LoopGuard.MaxContextTokens = 2_000;
        agent.LoopGuard.CompactionThreshold = 0.5;
        agent.LoopGuard.SummaryModel = SummaryModel;

        fixture.Client.AlwaysRespondsWithText("Verstanden.");

        for (var i = 0; i < 20; i++)
        {
            await fixture.Engine.ChatAsync(
                agent, $"Frage {i}: " + new string('y', 600), "test-instance", default);
        }

        // Compaction happened at all — otherwise the test says nothing about it.
        fixture.Client.ReceivedRequests.ShouldContain(r => r.Model == SummaryModel);
    }

    // ═══════════════════════════════════════════════════════════
    //  Calibration
    // ═══════════════════════════════════════════════════════════

    private static ContextCompactor NewCompactor()
        => new(new FakeChatClient(), TestLogging.Factory);

    [Fact]
    public void Without_measurements_the_raw_estimate_applies()
    {
        var messages = Conversation.Start().Repeat(4).Build();

        NewCompactor().EstimateFor("m", messages)
            .ShouldBe(ContextCompactor.EstimateTokens(messages));
    }

    [Fact]
    public void An_underestimated_request_raises_the_estimate()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var raw = ContextCompactor.EstimateTokens(messages);

        var compactor = NewCompactor();

        // The API consistently charges twice as much as estimated.
        for (var i = 0; i < 20; i++)
            compactor.Calibrate("m", 1_000, 2_000);

        compactor.EstimateFor("m", messages).ShouldBeGreaterThan(raw);
    }

    [Fact]
    public void A_single_outlier_does_not_tip_the_threshold()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var raw = ContextCompactor.EstimateTokens(messages);

        var compactor = NewCompactor();
        compactor.Calibrate("m", 1_000, 4_000);   // far off once

        // A measurement only shifts the factor by its weight, not to its value.
        compactor.EstimateFor("m", messages).ShouldBeLessThan(raw * 2);
    }

    [Fact]
    public void An_absurd_measurement_is_capped()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var raw = ContextCompactor.EstimateTokens(messages);

        var compactor = NewCompactor();
        for (var i = 0; i < 50; i++)
            compactor.Calibrate("m", 1, 1_000_000);

        // At most four times — anything above that is no longer a calibration case.
        compactor.EstimateFor("m", messages).ShouldBeLessThanOrEqualTo(raw * 4);
    }

    [Fact]
    public void Every_model_is_calibrated_on_its_own()
    {
        // Different tokenizers split the same text differently — a common
        // factor would be the average of things that have nothing to do with each other.
        var messages = Conversation.Start().Repeat(4).Build();
        var raw = ContextCompactor.EstimateTokens(messages);

        var compactor = NewCompactor();
        for (var i = 0; i < 20; i++)
            compactor.Calibrate("gross", 1_000, 3_000);

        compactor.EstimateFor("gross", messages).ShouldBeGreaterThan(raw);
        compactor.EstimateFor("unberuehrt", messages).ShouldBe(raw);
    }

    [Fact]
    public void Unusable_measurements_are_discarded()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var raw = ContextCompactor.EstimateTokens(messages);

        var compactor = NewCompactor();
        compactor.Calibrate("m", 0, 5_000);    // without an estimate no ratio
        compactor.Calibrate("m", 1_000, 0);    // without a measurement neither

        compactor.EstimateFor("m", messages).ShouldBe(raw);
    }
}