using ClawdDotNet.Core.Engine; using ClawdDotNet.Core.Tests.Infrastructure; using Shouldly; namespace ClawdDotNet.Core.Tests.Engine; /// /// #203 — compaction ran after the API call. /// /// It relied on the promptTokens of the request just answered: /// by the time the value was available, the overfull prompt had long been billed, and truncation /// only happened for the next step. Exactly one step too late — and the most expensive one. /// /// The promise now reads: no request goes beyond the context limit. /// The check happens before sending, with the estimate; the measured values adjust the /// estimate afterwards. /// public sealed class ProactiveCompactionTests { private const string AgentModel = "test/model"; private const string SummaryModel = "test/summary"; [Fact] public async Task No_request_goes_beyond_the_context_limit() { var fixture = new EngineFixture(); var agent = fixture.AddAgent("agent-a"); // Threshold: 1000 estimated tokens. agent.LoopGuard.MaxContextTokens = 2_000; agent.LoopGuard.CompactionThreshold = 0.5; agent.LoopGuard.SummaryModel = SummaryModel; fixture.Client.AlwaysRespondsWithText("Verstanden."); // Every message brings about 150 estimated tokens; after a few rounds // the context would be above the limit without intervention. Small enough that even the // six protected tail messages together still fit below the limit - // otherwise not even a compaction could get under it any more. for (var i = 0; i < 20; i++) { await fixture.Engine.ChatAsync( agent, $"Frage {i}: " + new string('y', 600), "test-instance", default); } var requestsToAgent = fixture.Client.ReceivedRequests .Where(r => r.Model == AgentModel) .ToList(); requestsToAgent.Count.ShouldBe(20); // The actual point: not "at some point it was truncated", but "nothing overfull ever // went out". Previously the check hung on the promptTokens of the // answer — always 100 with the fake —, so it never got over the threshold and never truncated. foreach (var request in requestsToAgent) { ContextCompactor.EstimateTokens(request.Messages) .ShouldBeLessThan(agent.LoopGuard.MaxContextTokens); } } [Fact] public async Task Shortening_uses_the_cheap_model_before_sending() { var fixture = new EngineFixture(); var agent = fixture.AddAgent("agent-a"); agent.LoopGuard.MaxContextTokens = 2_000; agent.LoopGuard.CompactionThreshold = 0.5; agent.LoopGuard.SummaryModel = SummaryModel; fixture.Client.AlwaysRespondsWithText("Verstanden."); for (var i = 0; i < 20; i++) { await fixture.Engine.ChatAsync( agent, $"Frage {i}: " + new string('y', 600), "test-instance", default); } // Compaction happened at all — otherwise the test says nothing about it. fixture.Client.ReceivedRequests.ShouldContain(r => r.Model == SummaryModel); } // ═══════════════════════════════════════════════════════════ // Calibration // ═══════════════════════════════════════════════════════════ private static ContextCompactor NewCompactor() => new(new FakeChatClient(), TestLogging.Factory); [Fact] public void Without_measurements_the_raw_estimate_applies() { var messages = Conversation.Start().Repeat(4).Build(); NewCompactor().EstimateFor("m", messages) .ShouldBe(ContextCompactor.EstimateTokens(messages)); } [Fact] public void An_underestimated_request_raises_the_estimate() { var messages = Conversation.Start().Repeat(4).Build(); var raw = ContextCompactor.EstimateTokens(messages); var compactor = NewCompactor(); // The API consistently charges twice as much as estimated. for (var i = 0; i < 20; i++) compactor.Calibrate("m", 1_000, 2_000); compactor.EstimateFor("m", messages).ShouldBeGreaterThan(raw); } [Fact] public void A_single_outlier_does_not_tip_the_threshold() { var messages = Conversation.Start().Repeat(4).Build(); var raw = ContextCompactor.EstimateTokens(messages); var compactor = NewCompactor(); compactor.Calibrate("m", 1_000, 4_000); // far off once // A measurement only shifts the factor by its weight, not to its value. compactor.EstimateFor("m", messages).ShouldBeLessThan(raw * 2); } [Fact] public void An_absurd_measurement_is_capped() { var messages = Conversation.Start().Repeat(4).Build(); var raw = ContextCompactor.EstimateTokens(messages); var compactor = NewCompactor(); for (var i = 0; i < 50; i++) compactor.Calibrate("m", 1, 1_000_000); // At most four times — anything above that is no longer a calibration case. compactor.EstimateFor("m", messages).ShouldBeLessThanOrEqualTo(raw * 4); } [Fact] public void Every_model_is_calibrated_on_its_own() { // Different tokenizers split the same text differently — a common // factor would be the average of things that have nothing to do with each other. var messages = Conversation.Start().Repeat(4).Build(); var raw = ContextCompactor.EstimateTokens(messages); var compactor = NewCompactor(); for (var i = 0; i < 20; i++) compactor.Calibrate("gross", 1_000, 3_000); compactor.EstimateFor("gross", messages).ShouldBeGreaterThan(raw); compactor.EstimateFor("unberuehrt", messages).ShouldBe(raw); } [Fact] public void Unusable_measurements_are_discarded() { var messages = Conversation.Start().Repeat(4).Build(); var raw = ContextCompactor.EstimateTokens(messages); var compactor = NewCompactor(); compactor.Calibrate("m", 0, 5_000); // without an estimate no ratio compactor.Calibrate("m", 1_000, 0); // without a measurement neither compactor.EstimateFor("m", messages).ShouldBe(raw); } }