ClawdDotNet/tests/ClawdDotNet.Core.Tests/Engine/ProactiveCompactionTests.cs

using ClawdDotNet.Core.Engine;
using ClawdDotNet.Core.Tests.Infrastructure;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Engine;

/// <summary>
/// #203 — compaction ran after the API call.
///
/// <para>It relied on the <c>promptTokens</c> of the request just answered:
/// by the time the value was available, the overfull prompt had long been billed, and truncation
/// only happened for the next step. Exactly one step too late — and the most expensive one.</para>
///
/// <para>The promise now reads: <b>no request goes beyond the context limit.</b>
/// The check happens before sending, with the estimate; the measured values adjust the
/// estimate afterwards.</para>
/// </summary>
public sealed class ProactiveCompactionTests
{
    private const string AgentModel = "test/model";
    private const string SummaryModel = "test/summary";

    [Fact]
    public async Task Keine_Anfrage_geht_ueber_die_Kontextgrenze_hinaus()
    {
        var fixture = new EngineFixture();
        var agent = fixture.AddAgent("agent-a");

        // Threshold: 1000 estimated tokens.
        agent.LoopGuard.MaxContextTokens = 2_000;
        agent.LoopGuard.CompactionThreshold = 0.5;
        agent.LoopGuard.SummaryModel = SummaryModel;

        fixture.Client.AlwaysRespondsWithText("Verstanden.");

        // Every message brings about 150 estimated tokens; after a few rounds
        // the context would be above the limit without intervention. Small enough that even the
        // six protected tail messages together still fit below the limit -
        // otherwise not even a compaction could get under it any more.
        for (var i = 0; i < 20; i++)
        {
            await fixture.Engine.ChatAsync(
                agent, $"Frage {i}: " + new string('y', 600), "test-instance", default);
        }

        var anfragenAnDenAgenten = fixture.Client.ReceivedRequests
            .Where(r => r.Model == AgentModel)
            .ToList();

        anfragenAnDenAgenten.Count.ShouldBe(20);

        // The actual point: not "at some point it was truncated", but "nothing overfull ever
        // went out". Previously the check hung on the promptTokens of the
        // answer — always 100 with the fake —, so it never got over the threshold and never truncated.
        foreach (var anfrage in anfragenAnDenAgenten)
        {
            ContextCompactor.EstimateTokens(anfrage.Messages)
                .ShouldBeLessThan(agent.LoopGuard.MaxContextTokens);
        }
    }

    [Fact]
    public async Task Gekuerzt_wird_mit_dem_guenstigen_Modell_bevor_gesendet_wird()
    {
        var fixture = new EngineFixture();
        var agent = fixture.AddAgent("agent-a");

        agent.LoopGuard.MaxContextTokens = 2_000;
        agent.LoopGuard.CompactionThreshold = 0.5;
        agent.LoopGuard.SummaryModel = SummaryModel;

        fixture.Client.AlwaysRespondsWithText("Verstanden.");

        for (var i = 0; i < 20; i++)
        {
            await fixture.Engine.ChatAsync(
                agent, $"Frage {i}: " + new string('y', 600), "test-instance", default);
        }

        // Compaction happened at all — otherwise the test says nothing about it.
        fixture.Client.ReceivedRequests.ShouldContain(r => r.Model == SummaryModel);
    }

    // ═══════════════════════════════════════════════════════════
    //  Calibration
    // ═══════════════════════════════════════════════════════════

    private static ContextCompactor NeuerKompaktierer()
        => new(new FakeChatClient(), TestLogging.Factory);

    [Fact]
    public void Ohne_Messwerte_gilt_die_rohe_Schaetzung()
    {
        var messages = Conversation.Start().Repeat(4).Build();

        NeuerKompaktierer().EstimateFor("m", messages)
            .ShouldBe(ContextCompactor.EstimateTokens(messages));
    }

    [Fact]
    public void Eine_unterschaetzte_Anfrage_hebt_die_Schaetzung_an()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var roh = ContextCompactor.EstimateTokens(messages);

        var compactor = NeuerKompaktierer();

        // The API consistently charges twice as much as estimated.
        for (var i = 0; i < 20; i++)
            compactor.Calibrate("m", 1_000, 2_000);

        compactor.EstimateFor("m", messages).ShouldBeGreaterThan(roh);
    }

    [Fact]
    public void Ein_einzelner_Ausreisser_kippt_die_Schwelle_nicht()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var roh = ContextCompactor.EstimateTokens(messages);

        var compactor = NeuerKompaktierer();
        compactor.Calibrate("m", 1_000, 4_000);   // far off once

        // A measurement only shifts the factor by its weight, not to its value.
        compactor.EstimateFor("m", messages).ShouldBeLessThan(roh * 2);
    }

    [Fact]
    public void Ein_absurder_Messwert_wird_gedeckelt()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var roh = ContextCompactor.EstimateTokens(messages);

        var compactor = NeuerKompaktierer();
        for (var i = 0; i < 50; i++)
            compactor.Calibrate("m", 1, 1_000_000);

        // At most four times — anything above that is no longer a calibration case.
        compactor.EstimateFor("m", messages).ShouldBeLessThanOrEqualTo(roh * 4);
    }

    [Fact]
    public void Jedes_Modell_wird_fuer_sich_kalibriert()
    {
        // Different tokenizers split the same text differently — a common
        // factor would be the average of things that have nothing to do with each other.
        var messages = Conversation.Start().Repeat(4).Build();
        var roh = ContextCompactor.EstimateTokens(messages);

        var compactor = NeuerKompaktierer();
        for (var i = 0; i < 20; i++)
            compactor.Calibrate("gross", 1_000, 3_000);

        compactor.EstimateFor("gross", messages).ShouldBeGreaterThan(roh);
        compactor.EstimateFor("unberuehrt", messages).ShouldBe(roh);
    }

    [Fact]
    public void Unbrauchbare_Messwerte_werden_verworfen()
    {
        var messages = Conversation.Start().Repeat(4).Build();
        var roh = ContextCompactor.EstimateTokens(messages);

        var compactor = NeuerKompaktierer();
        compactor.Calibrate("m", 0, 5_000);    // without an estimate no ratio
        compactor.Calibrate("m", 1_000, 0);    // without a measurement neither

        compactor.EstimateFor("m", messages).ShouldBe(roh);
    }
}