ClawdDotNet/tests/ClawdDotNet.Core.Tests/Transcripts/PromptHistoryTests.cs

using ClawdDotNet.Core.Api.Models;
using ClawdDotNet.Core.Storage;
using ClawdDotNet.Core.Tests.Infrastructure;
using ClawdDotNet.Core.Transcripts;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Transcripts;

/// <summary>
/// Prompt history (#660): prompt and response of every run are recorded, bounded in size,
/// scrubbed of recognisable secrets, searchable, and deleted once the retention of the
/// agent - or the instance default - says so.
/// </summary>
public sealed class PromptHistoryTests : IDisposable
{
    private readonly string _directory;
    private readonly SqliteStorage _storage;
    private readonly SqliteRunTranscriptStore _store;

    public PromptHistoryTests()
    {
        _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"));
        _storage = new SqliteStorage(Path.Combine(_directory, "state.db"));
        _store = new SqliteRunTranscriptStore(_storage);
    }

    public void Dispose()
    {
        Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
        try { Directory.Delete(_directory, recursive: true); }
        catch { /* cleanup is best effort */ }
    }

    private static ToolCall Call(string id, string name, string args)
        => new() { Id = id, Function = new ToolCallFunction { Name = name, Arguments = args } };

    // ─── Retention settings ───

    [Theory]
    [InlineData(null, 14, 14)]
    [InlineData(0, 14, 0)]
    [InlineData(-1, 14, -1)]
    [InlineData(365, 14, 365)]
    [InlineData(null, 0, 0)]
    [InlineData(-7, 14, -1)]
    public void The_agent_setting_overrides_the_instance_default(int? agent, int instance, int expected)
        => PromptHistorySettings.Resolve(agent, instance).ShouldBe(expected);

    // ─── Recorder ───

    [Fact]
    public void The_recorder_keeps_task_steps_tool_results_and_usage()
    {
        var recorder = new RunTranscriptRecorder();
        recorder.Begin("You are a test.", "Fetch the weather.");
        recorder.ModelStep(new ChatMessage { Role = "assistant", Content = "Let me look.", ToolCalls = [Call("c1", "WebFetch", "{\"url\":\"x\"}")] },
            new Usage { PromptTokens = 120, CompletionTokens = 30, Cost = 0.0004m }, "tool_calls");
        recorder.ToolResults([Call("c1", "WebFetch", "{}")], ["18 °C"]);
        recorder.ModelStep(new ChatMessage { Role = "assistant", Content = "ERGEBNIS: ok - 18 °C" }, null, "stop");

        var t = recorder.Build("run-1", "wetter", "m", "task", "t-1", "Completed");

        t.Entries.Select(e => e.Kind).ShouldBe(["user", "assistant", "tool_result", "assistant"]);
        t.Entries[1].ToolCalls!.Single().Name.ShouldBe("WebFetch");
        t.Entries[1].PromptTokens.ShouldBe(120);
        t.Entries[1].CostUsd.ShouldBe(0.0004m);
        t.Entries[2].ToolName.ShouldBe("WebFetch");
        t.SystemPromptHash.ShouldNotBeNull();
        recorder.SystemPrompt.ShouldBe("You are a test.");
    }

    [Fact]
    public void Long_tool_results_keep_start_and_end()
    {
        var recorder = new RunTranscriptRecorder();
        var text = "START" + new string('x', 20_000) + "END";
        recorder.ToolResults([Call("c", "FileRW", "{}")], [text]);

        var entry = recorder.Build("r", "a", "m", "task", null, "Completed").Entries.Single();
        entry.Clipped.ShouldBeTrue();
        entry.Text!.Length.ShouldBeLessThan(RunTranscriptRecorder.MaxToolResultChars + 100);
        entry.Text.ShouldStartWith("START");
        entry.Text.ShouldEndWith("END");
    }

    [Fact]
    public void A_runaway_run_stops_storing_text_but_keeps_the_heads()
    {
        var recorder = new RunTranscriptRecorder();
        for (var i = 0; i < 100; i++)
            recorder.ToolResults([Call($"c{i}", "WebFetch", "{}")], [new string('y', 4_000)]);

        var t = recorder.Build("r", "a", "m", "task", null, "LoopLimitExceeded");
        t.Truncated.ShouldBeTrue();
        t.SizeChars.ShouldBeLessThanOrEqualTo(RunTranscriptRecorder.MaxTotalChars);
        t.Entries.Count.ShouldBe(100, "every call is still listed");
        t.Entries.Last().Text.ShouldBeNull();
    }

    [Theory]
    [InlineData("key sk-or-v1-0123456789abcdef0123456789abcdef here")]
    [InlineData("Server=db;Password=geheim123;Database=x")]
    [InlineData("Authorization: Bearer eyJhbGciOiJIUzI1NiJ9.abcdefghijk")]
    [InlineData("Lizenz ABCDE-12345-FGHIJ-67890-KLMNO")]
    public void Recognisable_secrets_are_scrubbed(string text)
    {
        var recorder = new RunTranscriptRecorder();
        recorder.Begin(null, text);

        var stored = recorder.Build("r", "a", "m", "task", null, "Completed").Entries.Single().Text!;
        stored.ShouldContain(SecretScrubber.Mask);
        stored.ShouldNotContain("geheim123");
        stored.ShouldNotContain("0123456789abcdef0123456789abcdef");
    }

    [Fact]
    public void A_disabled_recorder_keeps_nothing()
    {
        var recorder = RunTranscriptRecorder.Disabled;
        recorder.Begin("s", "u");
        recorder.Enabled.ShouldBeFalse();
    }

    // ─── Store ───

    private static RunTranscript Transcript(string runId, string agent, DateTime started, string text = "Hallo Grüße")
        => new()
        {
            RunId = runId,
            AgentId = agent,
            Model = "m",
            Source = "task",
            StartedAt = started,
            CompletedAt = started.AddSeconds(5),
            Status = "Completed",
            Entries =
            [
                new TranscriptEntry { Kind = TranscriptEntryKind.User, At = started, Text = text },
                new TranscriptEntry { Kind = TranscriptEntryKind.Assistant, At = started, Text = "ERGEBNIS: ok" }
            ],
            SizeChars = text.Length + 12
        };

    [Fact]
    public async Task A_transcript_is_stored_and_read_back_with_its_system_prompt()
    {
        var t = Transcript("run-1", "wetter", DateTime.UtcNow) with { SystemPromptHash = RunTranscriptRecorder.Hash("sys") };
        await _store.SaveAsync(t, "sys", default);

        var back = (await _store.GetAsync("run-1", default)).ShouldNotBeNull();
        back.Entries.Count.ShouldBe(2);
        back.Entries[0].Text.ShouldBe("Hallo Grüße");
        (await _store.GetSystemPromptAsync(t.SystemPromptHash!, default)).ShouldBe("sys");
    }

    [Fact]
    public async Task The_search_finds_text_with_umlauts_and_filters_by_agent()
    {
        await _store.SaveAsync(Transcript("a", "wetter", DateTime.UtcNow, "Unwetterwarnung für Berlin"), null, default);
        await _store.SaveAsync(Transcript("b", "kurse", DateTime.UtcNow, "Kurs von NVDA"), null, default);

        var (items, total) = await _store.ListAsync(new TranscriptQuery { Search = "für Berlin" }, default);
        total.ShouldBe(1);
        items.Single().RunId.ShouldBe("a");
        items.Single().Prompt.ShouldBe("Unwetterwarnung für Berlin");
        items.Single().Response.ShouldBe("ERGEBNIS: ok");

        (await _store.ListAsync(new TranscriptQuery { AgentId = "kurse" }, default)).Total.ShouldBe(1);
    }

    [Fact]
    public async Task Purge_follows_the_retention_of_each_agent()
    {
        var now = DateTime.UtcNow;
        await _store.SaveAsync(Transcript("old-default", "a", now.AddDays(-20)), null, default);
        await _store.SaveAsync(Transcript("new-default", "a", now.AddDays(-2)), null, default);
        await _store.SaveAsync(Transcript("old-forever", "debug", now.AddDays(-400)), null, default);
        await _store.SaveAsync(Transcript("fresh-never", "secret", now), null, default);

        var deleted = await _store.PurgeAsync(agent => agent switch
        {
            "debug" => PromptHistorySettings.Forever,
            "secret" => PromptHistorySettings.Never,
            _ => 14
        }, now, default);

        deleted.ShouldBe(2);
        (await _store.GetAsync("old-default", default)).ShouldBeNull();
        (await _store.GetAsync("new-default", default)).ShouldNotBeNull();
        (await _store.GetAsync("old-forever", default)).ShouldNotBeNull();
        (await _store.GetAsync("fresh-never", default)).ShouldBeNull("never also removes what was recorded before");
    }

    // ─── Engine ───

    [Fact]
    public async Task Every_run_leaves_its_transcript()
    {
        var fixture = new EngineFixture(transcripts: _store).WithTool(FakeTool.Returning("18 °C", "Wetter"));
        var agent = fixture.AddAgent("wetterdienst", "Wetter");
        fixture.Client.RespondsWithToolCall("Wetter", "{\"ort\":\"Berlin\"}").RespondsWithText("Es sind 18 °C.");

        await fixture.Engine.RunAsync(agent, "Wie ist das Wetter?", "test-instance", default);

        var (items, _) = await _store.ListAsync(new TranscriptQuery(), default);
        var t = (await _store.GetAsync(items.Single().RunId, default)).ShouldNotBeNull();
        t.AgentId.ShouldBe("wetterdienst");
        t.Entries.Select(e => e.Kind).ShouldBe(["user", "assistant", "tool_result", "assistant"]);
        t.Entries[0].Text.ShouldContain("Wie ist das Wetter?");
        items.Single().Prompt.ShouldBe("Wie ist das Wetter?", "the list shows the message, not the time line the engine adds");
        t.Entries[2].Text.ShouldNotBeNull().ShouldContain("18 °C", Case.Sensitive, "stored as the model saw it, frame included");
        t.Entries[3].Text.ShouldBe("Es sind 18 °C.");
        (await _store.GetSystemPromptAsync(t.SystemPromptHash!, default)).ShouldNotBeNull().ShouldContain("wetterdienst");
    }

    [Fact]
    public async Task An_agent_set_to_never_is_not_recorded()
    {
        var fixture = new EngineFixture(transcripts: _store);
        var agent = fixture.AddAgent("tresor");
        agent.PromptHistory.RetentionDays = PromptHistorySettings.Never;
        fixture.Client.RespondsWithText("geheim");

        await fixture.Engine.RunAsync(agent, "Passwort?", "test-instance", default);

        (await _store.ListAsync(new TranscriptQuery(), default)).Total.ShouldBe(0);
    }

    [Fact]
    public async Task Chats_are_recorded_per_run_not_as_the_whole_history()
    {
        var fixture = new EngineFixture(transcripts: _store);
        var agent = fixture.AddAgent("assistent");
        fixture.Client.RespondsWithText("Erste Antwort").RespondsWithText("Zweite Antwort");

        await fixture.Engine.ChatAsync(agent, "Erste Frage", "test-instance", default);
        await fixture.Engine.ChatAsync(agent, "Zweite Frage", "test-instance", default);

        var (items, _) = await _store.ListAsync(new TranscriptQuery(), default);
        items.Count.ShouldBe(2);
        var second = (await _store.GetAsync(items.First().RunId, default)).ShouldNotBeNull();
        second.Entries.Count(e => e.Kind == TranscriptEntryKind.User).ShouldBe(1);
        second.Entries[0].Text.ShouldContain("Zweite Frage");
    }
}