using ClawdDotNet.Core.Audit; using ClawdDotNet.Core.Engine; using ClawdDotNet.Core.Tests.Infrastructure; using ClawdDotNet.Core.Tools; namespace ClawdDotNet.Core.Tests.Audit; /// /// Recovery after a hard crash (#554) and the evaluation of the log (#556). /// public class RecoveryTests { // ─────────────── The "started" entry ─────────────── [Fact] public async Task Before_the_tool_there_is_an_entry_in_the_log() { var audit = new InMemoryAuditRepository(); var fixture = new EngineFixture(audit).WithTool(FakeTool.Returning("ok")); var agent = fixture.AddAgent("a1", "TestTool"); fixture.Client .RespondsWithToolCall("TestTool", id: "c1") .RespondsWithText("Fertig."); await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None); var entries = audit.Entries.Where(e => e.CallId == "c1").ToList(); // Exactly two: first the start, then the completion — in this order. Assert.Equal(2, entries.Count); Assert.Equal(AuditStatus.Started, entries[0].Status); Assert.Equal(AuditStatus.Ok, entries[1].Status); } [Fact] public async Task A_completed_call_does_not_count_as_open() { var audit = new InMemoryAuditRepository(); var fixture = new EngineFixture(audit).WithTool(FakeTool.Returning("ok")); var agent = fixture.AddAgent("a1", "TestTool"); fixture.Client .RespondsWithToolCall("TestTool", id: "c1") .RespondsWithText("Fertig."); await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None); Assert.Empty(await audit.ListOpenCallsAsync(CancellationToken.None)); } [Fact] public async Task An_abort_does_not_leave_the_call_open() { // Otherwise every ordinary user cancellation would look like a crash at the next start // — and nobody would look any more. var audit = new InMemoryAuditRepository(); var fixture = new EngineFixture(audit).WithTool( FakeTool.Slow(TimeSpan.FromSeconds(5), name: "LangsamTool")); var agent = fixture.AddAgent("a1", "LangsamTool"); fixture.Client .RespondsWithToolCall("LangsamTool", id: "c1") .AlwaysRespondsWithText("Fertig."); var run = fixture.Engine.ChatAsync(agent, "Los", "inst", CancellationToken.None); await Task.Delay(300); fixture.Engine.AbortChat("a1"); await run; Assert.Empty(await audit.ListOpenCallsAsync(CancellationToken.None)); var last = audit.Entries.Last(e => e.CallId == "c1"); Assert.Equal(AuditStatus.Cancelled, last.Status); } [Fact] public async Task A_call_without_completion_is_found_as_open() { // Simulates the hard crash: the start is in the log, nothing more. var audit = new InMemoryAuditRepository(); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "Mail", Arguments = """{"to":"x@example.com"}""", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var open = await audit.ListOpenCallsAsync(CancellationToken.None); Assert.Single(open); Assert.Equal("Mail", open[0].Tool); } [Fact] public async Task Legacy_entries_without_CallId_do_not_count_as_open() { // Otherwise the first start after the update would present the whole old log // for review. var audit = new InMemoryAuditRepository(); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "", AgentId = "a1", Tool = "Mail", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); Assert.Empty(await audit.ListOpenCallsAsync(CancellationToken.None)); } // ─────────────── The classification ─────────────── [Fact] public async Task A_repeatable_tool_needs_no_decision() { var audit = new InMemoryAuditRepository(); var registry = new ToolRegistry(); registry.Register(new RepeatableTool()); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "Abruf", Arguments = "{}", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var recovery = new CrashRecovery(audit, registry, TestLogging.Factory); var cases = await recovery.ReviewAsync(CancellationToken.None); Assert.Single(cases); Assert.Equal(RecoveryVerdict.Repeatable, cases[0].Verdict); Assert.True(cases[0].Erledigt); } [Fact] public async Task A_non_repeatable_tool_is_submitted() { var audit = new InMemoryAuditRepository(); var registry = new ToolRegistry(); registry.Register(FakeTool.Returning("ok", "Versand")); // default: not repeatable await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "Versand", Arguments = "{}", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var recovery = new CrashRecovery(audit, registry, TestLogging.Factory); var cases = await recovery.ReviewAsync(CancellationToken.None); Assert.Equal(RecoveryVerdict.NeedsReview, cases[0].Verdict); Assert.False(cases[0].Erledigt); } [Fact] public async Task A_vanished_tool_is_reported_as_such() { var audit = new InMemoryAuditRepository(); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "GibtsNichtMehr", Arguments = "{}", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var recovery = new CrashRecovery(audit, new ToolRegistry(), TestLogging.Factory); var cases = await recovery.ReviewAsync(CancellationToken.None); Assert.Equal(RecoveryVerdict.ToolUnknown, cases[0].Verdict); Assert.False(cases[0].Erledigt); } [Fact] public async Task After_an_orderly_end_there_is_nothing_to_review() { var recovery = new CrashRecovery( new InMemoryAuditRepository(), new ToolRegistry(), TestLogging.Factory); Assert.Empty(await recovery.ReviewAsync(CancellationToken.None)); } // ─────────────── The evaluation (#556) ─────────────── [Fact] public async Task The_metrics_count_only_completed_calls() { var audit = new InMemoryAuditRepository(); var now = DateTime.Now; // Two complete calls of 100 ms each, one of them failed, // plus a third that never ended. foreach (var (call, status, duration) in new[] { ("c1", AuditStatus.Ok, 100L), ("c2", AuditStatus.Error, 100L), }) { await audit.AppendAsync(new AuditEntry { CallId = call, AgentId = "a1", Tool = "T", Status = AuditStatus.Started, DurationMs = 0, OccurredAt = now }, CancellationToken.None); await audit.AppendAsync(new AuditEntry { CallId = call, AgentId = "a1", Tool = "T", Status = status, DurationMs = duration, OccurredAt = now }, CancellationToken.None); } await audit.AppendAsync(new AuditEntry { CallId = "c3", AgentId = "a1", Tool = "T", Status = AuditStatus.Started, DurationMs = 0, OccurredAt = now }, CancellationToken.None); var stats = await audit.SummarizeToolUsageAsync("a1", null, CancellationToken.None); var t = stats.Single(s => s.Tool == "T"); Assert.Equal(2, t.Calls); Assert.Equal(1, t.Failed); Assert.Equal(1, t.Unfinished); // 200 ms over two completed calls. If the three Started entries were counted // with their duration 0, this would say 40 — the tool would look twice as // fast as it is. Assert.Equal(100d, t.AverageDurationMs); Assert.Equal(0.5d, t.FailureRate); } [Fact] public async Task The_per_agent_query_separates_the_agents() { var audit = new InMemoryAuditRepository(); foreach (var agent in new[] { "a1", "a1", "a2" }) { await audit.AppendAsync(new AuditEntry { CallId = Guid.NewGuid().ToString("N"), AgentId = agent, Tool = "T", Status = AuditStatus.Ok, OccurredAt = DateTime.Now }, CancellationToken.None); } var forA1 = await audit.ListForAgentAsync("a1", null, 100, CancellationToken.None); Assert.Equal(2, forA1.Count); Assert.All(forA1, e => Assert.Equal("a1", e.AgentId)); } } /// A tool that declares itself repeatable. internal sealed class RepeatableTool : IAgentTool { public string Name => "Abruf"; public string Description => "Liest nur."; public bool IsRepeatable => true; public System.Text.Json.JsonElement InputSchema { get; } = System.Text.Json.JsonDocument.Parse("""{"type":"object"}""").RootElement.Clone(); public Task ExecuteAsync( System.Text.Json.JsonElement input, AgentToolContext context, CancellationToken ct) => Task.FromResult(ToolResult.Ok("ok")); }