using ClawdDotNet.Core.Api.Models; using ClawdDotNet.Core.Storage; using ClawdDotNet.Core.Tests.Infrastructure; using ClawdDotNet.Core.Transcripts; using Shouldly; namespace ClawdDotNet.Core.Tests.Transcripts; /// /// Prompt history (#660): prompt and response of every run are recorded, bounded in size, /// scrubbed of recognisable secrets, searchable, and deleted once the retention of the /// agent - or the instance default - says so. /// public sealed class PromptHistoryTests : IDisposable { private readonly string _directory; private readonly SqliteStorage _storage; private readonly SqliteRunTranscriptStore _store; public PromptHistoryTests() { _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N")); _storage = new SqliteStorage(Path.Combine(_directory, "state.db")); _store = new SqliteRunTranscriptStore(_storage); } public void Dispose() { Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); try { Directory.Delete(_directory, recursive: true); } catch { /* cleanup is best effort */ } } private static ToolCall Call(string id, string name, string args) => new() { Id = id, Function = new ToolCallFunction { Name = name, Arguments = args } }; // ─── Retention settings ─── [Theory] [InlineData(null, 14, 14)] [InlineData(0, 14, 0)] [InlineData(-1, 14, -1)] [InlineData(365, 14, 365)] [InlineData(null, 0, 0)] [InlineData(-7, 14, -1)] public void The_agent_setting_overrides_the_instance_default(int? agent, int instance, int expected) => PromptHistorySettings.Resolve(agent, instance).ShouldBe(expected); // ─── Recorder ─── [Fact] public void The_recorder_keeps_task_steps_tool_results_and_usage() { var recorder = new RunTranscriptRecorder(); recorder.Begin("You are a test.", "Fetch the weather."); recorder.ModelStep(new ChatMessage { Role = "assistant", Content = "Let me look.", ToolCalls = [Call("c1", "WebFetch", "{\"url\":\"x\"}")] }, new Usage { PromptTokens = 120, CompletionTokens = 30, Cost = 0.0004m }, "tool_calls"); recorder.ToolResults([Call("c1", "WebFetch", "{}")], ["18 °C"]); recorder.ModelStep(new ChatMessage { Role = "assistant", Content = "ERGEBNIS: ok - 18 °C" }, null, "stop"); var t = recorder.Build("run-1", "wetter", "m", "task", "t-1", "Completed"); t.Entries.Select(e => e.Kind).ShouldBe(["user", "assistant", "tool_result", "assistant"]); t.Entries[1].ToolCalls!.Single().Name.ShouldBe("WebFetch"); t.Entries[1].PromptTokens.ShouldBe(120); t.Entries[1].CostUsd.ShouldBe(0.0004m); t.Entries[2].ToolName.ShouldBe("WebFetch"); t.SystemPromptHash.ShouldNotBeNull(); recorder.SystemPrompt.ShouldBe("You are a test."); } [Fact] public void Long_tool_results_keep_start_and_end() { var recorder = new RunTranscriptRecorder(); var text = "START" + new string('x', 20_000) + "END"; recorder.ToolResults([Call("c", "FileRW", "{}")], [text]); var entry = recorder.Build("r", "a", "m", "task", null, "Completed").Entries.Single(); entry.Clipped.ShouldBeTrue(); entry.Text!.Length.ShouldBeLessThan(RunTranscriptRecorder.MaxToolResultChars + 100); entry.Text.ShouldStartWith("START"); entry.Text.ShouldEndWith("END"); } [Fact] public void A_runaway_run_stops_storing_text_but_keeps_the_heads() { var recorder = new RunTranscriptRecorder(); for (var i = 0; i < 100; i++) recorder.ToolResults([Call($"c{i}", "WebFetch", "{}")], [new string('y', 4_000)]); var t = recorder.Build("r", "a", "m", "task", null, "LoopLimitExceeded"); t.Truncated.ShouldBeTrue(); t.SizeChars.ShouldBeLessThanOrEqualTo(RunTranscriptRecorder.MaxTotalChars); t.Entries.Count.ShouldBe(100, "every call is still listed"); t.Entries.Last().Text.ShouldBeNull(); } [Theory] [InlineData("key sk-or-v1-0123456789abcdef0123456789abcdef here")] [InlineData("Server=db;Password=geheim123;Database=x")] [InlineData("Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.abcdefghijk")] [InlineData("Lizenz ABCDE-12345-FGHIJ-67890-KLMNO")] public void Recognisable_secrets_are_scrubbed(string text) { var recorder = new RunTranscriptRecorder(); recorder.Begin(null, text); var stored = recorder.Build("r", "a", "m", "task", null, "Completed").Entries.Single().Text!; stored.ShouldContain(SecretScrubber.Mask); stored.ShouldNotContain("geheim123"); stored.ShouldNotContain("0123456789abcdef0123456789abcdef"); } [Fact] public void A_disabled_recorder_keeps_nothing() { var recorder = RunTranscriptRecorder.Disabled; recorder.Begin("s", "u"); recorder.Enabled.ShouldBeFalse(); } // ─── Store ─── private static RunTranscript Transcript(string runId, string agent, DateTime started, string text = "Hallo Grüße") => new() { RunId = runId, AgentId = agent, Model = "m", Source = "task", StartedAt = started, CompletedAt = started.AddSeconds(5), Status = "Completed", Entries = [ new TranscriptEntry { Kind = TranscriptEntryKind.User, At = started, Text = text }, new TranscriptEntry { Kind = TranscriptEntryKind.Assistant, At = started, Text = "ERGEBNIS: ok" } ], SizeChars = text.Length + 12 }; [Fact] public async Task A_transcript_is_stored_and_read_back_with_its_system_prompt() { var t = Transcript("run-1", "wetter", DateTime.UtcNow) with { SystemPromptHash = RunTranscriptRecorder.Hash("sys") }; await _store.SaveAsync(t, "sys", default); var back = (await _store.GetAsync("run-1", default)).ShouldNotBeNull(); back.Entries.Count.ShouldBe(2); back.Entries[0].Text.ShouldBe("Hallo Grüße"); (await _store.GetSystemPromptAsync(t.SystemPromptHash!, default)).ShouldBe("sys"); } [Fact] public async Task The_search_finds_text_with_umlauts_and_filters_by_agent() { await _store.SaveAsync(Transcript("a", "wetter", DateTime.UtcNow, "Unwetterwarnung für Berlin"), null, default); await _store.SaveAsync(Transcript("b", "kurse", DateTime.UtcNow, "Kurs von NVDA"), null, default); var (items, total) = await _store.ListAsync(new TranscriptQuery { Search = "für Berlin" }, default); total.ShouldBe(1); items.Single().RunId.ShouldBe("a"); items.Single().Prompt.ShouldBe("Unwetterwarnung für Berlin"); items.Single().Response.ShouldBe("ERGEBNIS: ok"); (await _store.ListAsync(new TranscriptQuery { AgentId = "kurse" }, default)).Total.ShouldBe(1); } [Fact] public async Task Purge_follows_the_retention_of_each_agent() { var now = DateTime.UtcNow; await _store.SaveAsync(Transcript("old-default", "a", now.AddDays(-20)), null, default); await _store.SaveAsync(Transcript("new-default", "a", now.AddDays(-2)), null, default); await _store.SaveAsync(Transcript("old-forever", "debug", now.AddDays(-400)), null, default); await _store.SaveAsync(Transcript("fresh-never", "secret", now), null, default); var deleted = await _store.PurgeAsync(agent => agent switch { "debug" => PromptHistorySettings.Forever, "secret" => PromptHistorySettings.Never, _ => 14 }, now, default); deleted.ShouldBe(2); (await _store.GetAsync("old-default", default)).ShouldBeNull(); (await _store.GetAsync("new-default", default)).ShouldNotBeNull(); (await _store.GetAsync("old-forever", default)).ShouldNotBeNull(); (await _store.GetAsync("fresh-never", default)).ShouldBeNull("never also removes what was recorded before"); } // ─── Engine ─── [Fact] public async Task Every_run_leaves_its_transcript() { var fixture = new EngineFixture(transcripts: _store).WithTool(FakeTool.Returning("18 °C", "Wetter")); var agent = fixture.AddAgent("wetterdienst", "Wetter"); fixture.Client.RespondsWithToolCall("Wetter", "{\"ort\":\"Berlin\"}").RespondsWithText("Es sind 18 °C."); await fixture.Engine.RunAsync(agent, "Wie ist das Wetter?", "test-instance", default); var (items, _) = await _store.ListAsync(new TranscriptQuery(), default); var t = (await _store.GetAsync(items.Single().RunId, default)).ShouldNotBeNull(); t.AgentId.ShouldBe("wetterdienst"); t.Entries.Select(e => e.Kind).ShouldBe(["user", "assistant", "tool_result", "assistant"]); t.Entries[0].Text.ShouldContain("Wie ist das Wetter?"); items.Single().Prompt.ShouldBe("Wie ist das Wetter?", "the list shows the message, not the time line the engine adds"); t.Entries[2].Text.ShouldNotBeNull().ShouldContain("18 °C", Case.Sensitive, "stored as the model saw it, frame included"); t.Entries[3].Text.ShouldBe("Es sind 18 °C."); (await _store.GetSystemPromptAsync(t.SystemPromptHash!, default)).ShouldNotBeNull().ShouldContain("wetterdienst"); } [Fact] public async Task An_agent_set_to_never_is_not_recorded() { var fixture = new EngineFixture(transcripts: _store); var agent = fixture.AddAgent("tresor"); agent.PromptHistory.RetentionDays = PromptHistorySettings.Never; fixture.Client.RespondsWithText("geheim"); await fixture.Engine.RunAsync(agent, "Passwort?", "test-instance", default); (await _store.ListAsync(new TranscriptQuery(), default)).Total.ShouldBe(0); } [Fact] public async Task Chats_are_recorded_per_run_not_as_the_whole_history() { var fixture = new EngineFixture(transcripts: _store); var agent = fixture.AddAgent("assistent"); fixture.Client.RespondsWithText("Erste Antwort").RespondsWithText("Zweite Antwort"); await fixture.Engine.ChatAsync(agent, "Erste Frage", "test-instance", default); await fixture.Engine.ChatAsync(agent, "Zweite Frage", "test-instance", default); var (items, _) = await _store.ListAsync(new TranscriptQuery(), default); items.Count.ShouldBe(2); var second = (await _store.GetAsync(items.First().RunId, default)).ShouldNotBeNull(); second.Entries.Count(e => e.Kind == TranscriptEntryKind.User).ShouldBe(1); second.Entries[0].Text.ShouldContain("Zweite Frage"); } }