using ClawdDotNet.Core.Engine; using ClawdDotNet.Core.Tests.Infrastructure; using Shouldly; namespace ClawdDotNet.Core.Tests.Engine; /// /// #203 — compaction ran after the API call. /// /// It relied on the promptTokens of the request just answered: /// by the time the value was available, the overfull prompt had long been billed, and truncation /// only happened for the next step. Exactly one step too late — and the most expensive one. /// /// The promise now reads: no request goes beyond the context limit. /// The check happens before sending, with the estimate; the measured values adjust the /// estimate afterwards. /// public sealed class ProactiveCompactionTests { private const string AgentModel = "test/model"; private const string SummaryModel = "test/summary"; [Fact] public async Task Keine_Anfrage_geht_ueber_die_Kontextgrenze_hinaus() { var fixture = new EngineFixture(); var agent = fixture.AddAgent("agent-a"); // Threshold: 1000 estimated tokens. agent.LoopGuard.MaxContextTokens = 2_000; agent.LoopGuard.CompactionThreshold = 0.5; agent.LoopGuard.SummaryModel = SummaryModel; fixture.Client.AlwaysRespondsWithText("Verstanden."); // Every message brings about 150 estimated tokens; after a few rounds // the context would be above the limit without intervention. Small enough that even the // six protected tail messages together still fit below the limit - // otherwise not even a compaction could get under it any more. for (var i = 0; i < 20; i++) { await fixture.Engine.ChatAsync( agent, $"Frage {i}: " + new string('y', 600), "test-instance", default); } var anfragenAnDenAgenten = fixture.Client.ReceivedRequests .Where(r => r.Model == AgentModel) .ToList(); anfragenAnDenAgenten.Count.ShouldBe(20); // The actual point: not "at some point it was truncated", but "nothing overfull ever // went out". Previously the check hung on the promptTokens of the // answer — always 100 with the fake —, so it never got over the threshold and never truncated. foreach (var anfrage in anfragenAnDenAgenten) { ContextCompactor.EstimateTokens(anfrage.Messages) .ShouldBeLessThan(agent.LoopGuard.MaxContextTokens); } } [Fact] public async Task Gekuerzt_wird_mit_dem_guenstigen_Modell_bevor_gesendet_wird() { var fixture = new EngineFixture(); var agent = fixture.AddAgent("agent-a"); agent.LoopGuard.MaxContextTokens = 2_000; agent.LoopGuard.CompactionThreshold = 0.5; agent.LoopGuard.SummaryModel = SummaryModel; fixture.Client.AlwaysRespondsWithText("Verstanden."); for (var i = 0; i < 20; i++) { await fixture.Engine.ChatAsync( agent, $"Frage {i}: " + new string('y', 600), "test-instance", default); } // Compaction happened at all — otherwise the test says nothing about it. fixture.Client.ReceivedRequests.ShouldContain(r => r.Model == SummaryModel); } // ═══════════════════════════════════════════════════════════ // Calibration // ═══════════════════════════════════════════════════════════ private static ContextCompactor NeuerKompaktierer() => new(new FakeChatClient(), TestLogging.Factory); [Fact] public void Ohne_Messwerte_gilt_die_rohe_Schaetzung() { var messages = Conversation.Start().Repeat(4).Build(); NeuerKompaktierer().EstimateFor("m", messages) .ShouldBe(ContextCompactor.EstimateTokens(messages)); } [Fact] public void Eine_unterschaetzte_Anfrage_hebt_die_Schaetzung_an() { var messages = Conversation.Start().Repeat(4).Build(); var roh = ContextCompactor.EstimateTokens(messages); var compactor = NeuerKompaktierer(); // The API consistently charges twice as much as estimated. for (var i = 0; i < 20; i++) compactor.Calibrate("m", 1_000, 2_000); compactor.EstimateFor("m", messages).ShouldBeGreaterThan(roh); } [Fact] public void Ein_einzelner_Ausreisser_kippt_die_Schwelle_nicht() { var messages = Conversation.Start().Repeat(4).Build(); var roh = ContextCompactor.EstimateTokens(messages); var compactor = NeuerKompaktierer(); compactor.Calibrate("m", 1_000, 4_000); // far off once // A measurement only shifts the factor by its weight, not to its value. compactor.EstimateFor("m", messages).ShouldBeLessThan(roh * 2); } [Fact] public void Ein_absurder_Messwert_wird_gedeckelt() { var messages = Conversation.Start().Repeat(4).Build(); var roh = ContextCompactor.EstimateTokens(messages); var compactor = NeuerKompaktierer(); for (var i = 0; i < 50; i++) compactor.Calibrate("m", 1, 1_000_000); // At most four times — anything above that is no longer a calibration case. compactor.EstimateFor("m", messages).ShouldBeLessThanOrEqualTo(roh * 4); } [Fact] public void Jedes_Modell_wird_fuer_sich_kalibriert() { // Different tokenizers split the same text differently — a common // factor would be the average of things that have nothing to do with each other. var messages = Conversation.Start().Repeat(4).Build(); var roh = ContextCompactor.EstimateTokens(messages); var compactor = NeuerKompaktierer(); for (var i = 0; i < 20; i++) compactor.Calibrate("gross", 1_000, 3_000); compactor.EstimateFor("gross", messages).ShouldBeGreaterThan(roh); compactor.EstimateFor("unberuehrt", messages).ShouldBe(roh); } [Fact] public void Unbrauchbare_Messwerte_werden_verworfen() { var messages = Conversation.Start().Repeat(4).Build(); var roh = ContextCompactor.EstimateTokens(messages); var compactor = NeuerKompaktierer(); compactor.Calibrate("m", 0, 5_000); // without an estimate no ratio compactor.Calibrate("m", 1_000, 0); // without a measurement neither compactor.EstimateFor("m", messages).ShouldBe(roh); } }