using ClawdDotNet.Core.Audit; using ClawdDotNet.Core.Engine; using ClawdDotNet.Core.Tests.Infrastructure; using ClawdDotNet.Core.Tools; namespace ClawdDotNet.Core.Tests.Audit; /// /// Recovery after a hard crash (#554) and the evaluation of the log (#556). /// public class WiederaufnahmeTests { // ─────────────── The "started" entry ─────────────── [Fact] public async Task Vor_dem_Werkzeug_steht_ein_Eintrag_im_Protokoll() { var audit = new InMemoryAuditRepository(); var fixture = new EngineFixture(audit).WithTool(FakeTool.Returning("ok")); var agent = fixture.AddAgent("a1", "TestTool"); fixture.Client .RespondsWithToolCall("TestTool", id: "c1") .RespondsWithText("Fertig."); await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None); var eintraege = audit.Entries.Where(e => e.CallId == "c1").ToList(); // Exactly two: first the start, then the completion — in this order. Assert.Equal(2, eintraege.Count); Assert.Equal(AuditStatus.Started, eintraege[0].Status); Assert.Equal(AuditStatus.Ok, eintraege[1].Status); } [Fact] public async Task Ein_abgeschlossener_Aufruf_gilt_nicht_als_offen() { var audit = new InMemoryAuditRepository(); var fixture = new EngineFixture(audit).WithTool(FakeTool.Returning("ok")); var agent = fixture.AddAgent("a1", "TestTool"); fixture.Client .RespondsWithToolCall("TestTool", id: "c1") .RespondsWithText("Fertig."); await fixture.Engine.RunAsync(agent, "Los", "inst", CancellationToken.None); Assert.Empty(await audit.ListOpenCallsAsync(CancellationToken.None)); } [Fact] public async Task Ein_Abbruch_laesst_den_Aufruf_nicht_offen_stehen() { // Otherwise every ordinary user cancellation would look like a crash at the next start // — and nobody would look any more. var audit = new InMemoryAuditRepository(); var fixture = new EngineFixture(audit).WithTool( FakeTool.Slow(TimeSpan.FromSeconds(5), name: "LangsamTool")); var agent = fixture.AddAgent("a1", "LangsamTool"); fixture.Client .RespondsWithToolCall("LangsamTool", id: "c1") .AlwaysRespondsWithText("Fertig."); var lauf = fixture.Engine.ChatAsync(agent, "Los", "inst", CancellationToken.None); await Task.Delay(300); fixture.Engine.AbortChat("a1"); await lauf; Assert.Empty(await audit.ListOpenCallsAsync(CancellationToken.None)); var letzter = audit.Entries.Last(e => e.CallId == "c1"); Assert.Equal(AuditStatus.Cancelled, letzter.Status); } [Fact] public async Task Ein_Aufruf_ohne_Abschluss_wird_als_offen_gefunden() { // Simulates the hard crash: the start is in the log, nothing more. var audit = new InMemoryAuditRepository(); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "Mail", Arguments = """{"to":"x@example.com"}""", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var offen = await audit.ListOpenCallsAsync(CancellationToken.None); Assert.Single(offen); Assert.Equal("Mail", offen[0].Tool); } [Fact] public async Task Alteintraege_ohne_CallId_gelten_nicht_als_offen() { // Otherwise the first start after the update would present the whole old log // for review. var audit = new InMemoryAuditRepository(); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "", AgentId = "a1", Tool = "Mail", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); Assert.Empty(await audit.ListOpenCallsAsync(CancellationToken.None)); } // ─────────────── The classification ─────────────── [Fact] public async Task Ein_wiederholbares_Werkzeug_braucht_keine_Entscheidung() { var audit = new InMemoryAuditRepository(); var registry = new ToolRegistry(); registry.Register(new WiederholbaresTool()); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "Abruf", Arguments = "{}", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var recovery = new CrashRecovery(audit, registry, TestLogging.Factory); var faelle = await recovery.SichtenAsync(CancellationToken.None); Assert.Single(faelle); Assert.Equal(RecoveryVerdict.Repeatable, faelle[0].Verdict); Assert.True(faelle[0].Erledigt); } [Fact] public async Task Ein_nicht_wiederholbares_Werkzeug_wird_vorgelegt() { var audit = new InMemoryAuditRepository(); var registry = new ToolRegistry(); registry.Register(FakeTool.Returning("ok", "Versand")); // default: not repeatable await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "Versand", Arguments = "{}", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var recovery = new CrashRecovery(audit, registry, TestLogging.Factory); var faelle = await recovery.SichtenAsync(CancellationToken.None); Assert.Equal(RecoveryVerdict.NeedsReview, faelle[0].Verdict); Assert.False(faelle[0].Erledigt); } [Fact] public async Task Ein_verschwundenes_Werkzeug_wird_als_solches_gemeldet() { var audit = new InMemoryAuditRepository(); await audit.AppendAsync(new AuditEntry { RunId = "r1", CallId = "c1", AgentId = "a1", Tool = "GibtsNichtMehr", Arguments = "{}", Status = AuditStatus.Started, OccurredAt = DateTime.Now }, CancellationToken.None); var recovery = new CrashRecovery(audit, new ToolRegistry(), TestLogging.Factory); var faelle = await recovery.SichtenAsync(CancellationToken.None); Assert.Equal(RecoveryVerdict.ToolUnknown, faelle[0].Verdict); Assert.False(faelle[0].Erledigt); } [Fact] public async Task Nach_geordnetem_Ende_ist_nichts_zu_sichten() { var recovery = new CrashRecovery( new InMemoryAuditRepository(), new ToolRegistry(), TestLogging.Factory); Assert.Empty(await recovery.SichtenAsync(CancellationToken.None)); } // ─────────────── The evaluation (#556) ─────────────── [Fact] public async Task Die_Kennzahlen_zaehlen_nur_abgeschlossene_Aufrufe() { var audit = new InMemoryAuditRepository(); var jetzt = DateTime.Now; // Two complete calls of 100 ms each, one of them failed, // plus a third that never ended. foreach (var (call, status, dauer) in new[] { ("c1", AuditStatus.Ok, 100L), ("c2", AuditStatus.Error, 100L), }) { await audit.AppendAsync(new AuditEntry { CallId = call, AgentId = "a1", Tool = "T", Status = AuditStatus.Started, DurationMs = 0, OccurredAt = jetzt }, CancellationToken.None); await audit.AppendAsync(new AuditEntry { CallId = call, AgentId = "a1", Tool = "T", Status = status, DurationMs = dauer, OccurredAt = jetzt }, CancellationToken.None); } await audit.AppendAsync(new AuditEntry { CallId = "c3", AgentId = "a1", Tool = "T", Status = AuditStatus.Started, DurationMs = 0, OccurredAt = jetzt }, CancellationToken.None); var stats = await audit.SummarizeToolUsageAsync("a1", null, CancellationToken.None); var t = stats.Single(s => s.Tool == "T"); Assert.Equal(2, t.Calls); Assert.Equal(1, t.Failed); Assert.Equal(1, t.Unfinished); // 200 ms over two completed calls. If the three Started entries were counted // with their duration 0, this would say 40 — the tool would look twice as // fast as it is. Assert.Equal(100d, t.AverageDurationMs); Assert.Equal(0.5d, t.FailureRate); } [Fact] public async Task Die_Abfrage_je_Agent_trennt_die_Agenten() { var audit = new InMemoryAuditRepository(); foreach (var agent in new[] { "a1", "a1", "a2" }) { await audit.AppendAsync(new AuditEntry { CallId = Guid.NewGuid().ToString("N"), AgentId = agent, Tool = "T", Status = AuditStatus.Ok, OccurredAt = DateTime.Now }, CancellationToken.None); } var fuerA1 = await audit.ListForAgentAsync("a1", null, 100, CancellationToken.None); Assert.Equal(2, fuerA1.Count); Assert.All(fuerA1, e => Assert.Equal("a1", e.AgentId)); } } /// A tool that declares itself repeatable. internal sealed class WiederholbaresTool : IAgentTool { public string Name => "Abruf"; public string Description => "Liest nur."; public bool IsRepeatable => true; public System.Text.Json.JsonElement InputSchema { get; } = System.Text.Json.JsonDocument.Parse("""{"type":"object"}""").RootElement.Clone(); public Task ExecuteAsync( System.Text.Json.JsonElement input, AgentToolContext context, CancellationToken ct) => Task.FromResult(ToolResult.Ok("ok")); }