using ClawdDotNet.Core.Audit; using ClawdDotNet.Core.Config; using ClawdDotNet.Core.Inspection; using ClawdDotNet.Core.Organisation; using ClawdDotNet.Core.Storage; using ClawdDotNet.Core.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; namespace ClawdDotNet.Core.Tests.Organisation; /// /// #693: A lead looks at the runs of the agents it leads — against what the engine stamped, /// never by asking the agent; denser where it matters, capped; and what follows is proposed, /// not done. /// public sealed class SpotCheckTests : IDisposable { private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N")); private readonly SqliteStorage _storage; private readonly SqliteAuditRepository _audit; private readonly SqliteTaskRepository _tasks; private readonly TaskboardService _board; private static readonly IReadOnlyDictionary> Vocabulary = new Dictionary> { ["AgentEditor"] = new HashSet { "read_soul", "update_soul", "update_identity" }, ["AgentComm"] = new HashSet { "send_message", "list_agents" }, ["Memory"] = new HashSet { "remember", "recall" }, ["FileRW"] = new HashSet { "read", "write", "append" }, ["Taskboard"] = new HashSet { "complete", "comment" } }; public SpotCheckTests() { _storage = new SqliteStorage(Path.Combine(_directory, "state.db")); _audit = new SqliteAuditRepository(_storage); _tasks = new SqliteTaskRepository(_storage); _board = new TaskboardService(_tasks, Path.Combine(_directory, "tasks")); } public void Dispose() { SqliteStorage.ReleasePooledConnections(_directory); try { Directory.Delete(_directory, recursive: true); } catch { /* cleanup is secondary */ } } /// purchasing (buy-lead; buyer, clerk) with spot checks; sales (sales-lead; seller) without. private static OrgChart Org(UnitInspection? inspection = null) => new( [ new UnitConfig { Id = "purchasing", Name = "Purchasing", LeaderAgentId = "buy-lead", Inspection = inspection ?? new UnitInspection { Rate = 1, DenseRate = 1, PerDay = 2 } }, new UnitConfig { Id = "sales", Name = "Sales", LeaderAgentId = "sales-lead" } ], [ new AgentConfig { AgentId = "buy-lead", UnitId = "purchasing" }, new AgentConfig { AgentId = "buyer", UnitId = "purchasing" }, new AgentConfig { AgentId = "clerk", UnitId = "purchasing" }, new AgentConfig { AgentId = "sales-lead", UnitId = "sales" }, new AgentConfig { AgentId = "seller", UnitId = "sales" } ]); private static RunReceipt Receipt(string runId, string agent = "buyer", string status = "Completed", string model = "m", DateTime? at = null, string? task = null) => new() { RunId = runId, AgentId = agent, Model = model, Status = status, StepCount = 3, OccurredAt = at ?? DateTime.UtcNow, TaskId = task, ResultRef = "Done." }; private static AuditEntry Call(string tool, string? action, AuditStatus status = AuditStatus.Ok, string callId = "", string summary = "") => new() { RunId = "r", AgentId = "buyer", Tool = tool, Status = status, CallId = callId, Summary = summary, Arguments = action is null ? "{}" : $"{{\"action\":\"{action}\"}}", OccurredAt = DateTime.UtcNow }; private SpotChecks Checks(OrgChart? org = null) { var chart = org ?? Org(); return new SpotChecks(_storage, _audit, new RunInspector(_audit, _tasks, null, () => Vocabulary), () => chart, _board, NullLogger.Instance); } // ─── the mechanical check ─── [Fact] public void Failed_runs_errors_refusals_and_unfinished_calls_are_found() { var findings = RunInspector.Analyse( Receipt("r", status: "LoopLimitExceeded"), [ Call("FileRW", "write", AuditStatus.Error, "c1", "disk full"), Call("FileRW", "write", AuditStatus.Denied, "c2"), Call("FileRW", "read", AuditStatus.Started, "c3") ], "", Vocabulary, null); findings.Select(f => f.Kind).ShouldBe([FindingKind.RunFailed, FindingKind.ToolErrors, FindingKind.Denied, FindingKind.Unfinished], ignoreOrder: true); findings.Single(f => f.Kind == FindingKind.ToolErrors).Text.ShouldContain("FileRW.write (disk full)"); } [Fact] public void Operations_the_agent_names_but_never_called_are_found() { // The 2026-09-14 pattern: a findings file full of "tool answers" from a run that only appended to a file. var said = "AgentEditor read_soul answered 'User confirmation needed'. send_message was intercepted. I will remember this in Memory."; var findings = RunInspector.Analyse(Receipt("r"), [Call("FileRW", "append")], said, Vocabulary, null); var claim = findings.Single(f => f.Kind == FindingKind.UnsupportedClaim).Text; claim.ShouldContain("AgentEditor"); claim.ShouldContain("read_soul"); claim.ShouldContain("send_message"); claim.ShouldNotContain("remember", customMessage: "an ordinary word is no claim"); claim.ShouldNotContain("Memory"); } [Fact] public void A_refused_call_is_a_call() { var findings = RunInspector.Analyse(Receipt("r"), [Call("AgentEditor", "update_soul", AuditStatus.Denied)], "AgentEditor update_soul was refused, as expected.", Vocabulary, null); findings.ShouldNotContain(f => f.Kind == FindingKind.UnsupportedClaim); } [Fact] public void A_task_answered_with_bookkeeping_alone_is_marked() { var task = new TaskItem { Id = "t1", Title = "Order paper" }; RunInspector.Analyse(Receipt("r"), [Call("Taskboard", "complete")], "", Vocabulary, task) .ShouldContain(f => f.Kind == FindingKind.NoWork); RunInspector.Analyse(Receipt("r"), [Call("FileRW", "write"), Call("Taskboard", "complete")], "", Vocabulary, task) .ShouldNotContain(f => f.Kind == FindingKind.NoWork); } [Fact] public void The_operation_is_read_even_from_cut_arguments() { RunInspector.OperationOf("{\"action\":\"write\",\"content\":\"abc").ShouldBe("write"); RunInspector.OperationOf("not json").ShouldBeNull(); } // ─── which runs ─── [Fact] public void New_agents_new_models_and_the_run_after_a_failure_are_checked_densely() { var onlyDense = new UnitInspection { Rate = 0, DenseRate = 1, NewDays = 7 }; var now = DateTime.UtcNow; var settled = Receipt("old", at: now.AddDays(-20)); SpotChecks.Pick(Receipt("a", at: now), [], onlyDense).ShouldBe("new agent"); SpotChecks.Pick(Receipt("b", at: now), [settled], onlyDense).ShouldBeNull("a settled agent with the same model and no failure is an ordinary run"); SpotChecks.Pick(Receipt("c", model: "other", at: now), [settled], onlyDense).ShouldBe("new model"); SpotChecks.Pick(Receipt("d", at: now), [settled, Receipt("bad", status: "Failed", at: now.AddHours(-1))], onlyDense).ShouldBe("after a failure"); SpotChecks.Pick(Receipt("e", at: now), [settled], new UnitInspection { Rate = 1 }).ShouldBe("sample"); } [Fact] public void The_same_run_gets_the_same_answer_every_time() { var half = new UnitInspection { Rate = 0.5, DenseRate = 0.5 }; var runs = Enumerable.Range(0, 200).Select(i => Receipt($"run-{i}")).ToList(); var first = runs.Select(r => SpotChecks.Pick(r, [], half)).ToList(); runs.Select(r => SpotChecks.Pick(r, [], half)).ShouldBe(first); first.Count(p => p is not null).ShouldBeInRange(70, 130, "spread like chance"); } // ─── the lead's task ─── [Fact] public async Task Runs_are_picked_for_the_lead_capped_per_day_and_given_as_one_task() { var now = DateTime.UtcNow; foreach (var (run, agent, minutes) in new[] { ("b1", "buyer", 30), ("b2", "buyer", 20), ("c1", "clerk", 10), ("s1", "seller", 5), ("l1", "buy-lead", 3) }) await _audit.RecordReceiptAsync(Receipt(run, agent, at: now.AddMinutes(-minutes)), default); var checks = Checks(); var picked = await checks.SampleAsync(default); picked.Select(c => c.RunId).ShouldBe(["b1", "b2"], "two a day, oldest first"); picked.ShouldAllBe(c => c.Lead == "buy-lead"); (await checks.SampleAsync(default)).ShouldBeEmpty("a run is looked at once"); var tasks = await _board.ListAsync(new TaskQuery { Assignee = "@buy-lead" }, default); tasks.Count.ShouldBe(1); tasks[0].Title.ShouldContain("2 run(s)"); (await _board.ListAsync(new TaskQuery { Assignee = "@sales-lead" }, default)).ShouldBeEmpty("sales has no spot checks"); } [Fact] public async Task The_lead_closes_its_checks_and_the_last_one_closes_the_task() { await _audit.RecordReceiptAsync(Receipt("b1", at: DateTime.UtcNow.AddMinutes(-5)), default); await _audit.RecordReceiptAsync(Receipt("c1", "clerk", at: DateTime.UtcNow.AddMinutes(-4)), default); var checks = Checks(); var picked = await checks.SampleAsync(default); (await checks.CloseAsync(picked[0].Id, "sales-lead", false, "", default)).Ok.ShouldBeFalse("not its check"); (await checks.CloseAsync(picked[0].Id, "buy-lead", true, "", default)).Message.ShouldContain("'note'"); (await checks.CloseAsync(picked[0].Id, "buy-lead", true, "Result names a supplier the database does not have.", default)).Ok.ShouldBeTrue(); var task = (await _board.ListAsync(new TaskQuery { Assignee = "@buy-lead" }, default)).Single(); task.Status.ShouldNotBe(TaskItemStatus.Done); (await checks.CloseAsync(picked[1].Id, "buy-lead", false, "", default)).Ok.ShouldBeTrue(); (await _board.GetAsync(task.Id, default))!.Status.ShouldBe(TaskItemStatus.Done); (await checks.PendingForAsync("buy-lead", default)).ShouldBeEmpty(); } // ─── what follows ─── [Fact] public async Task A_lead_proposes_measures_for_its_own_agents_and_a_human_decides() { var checks = Checks(); (await checks.ProposeAsync("buy-lead", "seller", MeasureKind.ChangeModel, "x", null, default)).Ok.ShouldBeFalse("not its agent"); (await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.ChangeModel, " ", null, default)).Message.ShouldContain("'reasoning'"); (await checks.ProposeAsync("buyer", "clerk", MeasureKind.Suspend, "x", null, default)).Ok.ShouldBeFalse("a member leads nobody"); var proposed = await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.SharpenTask, "Three runs misread 'urgent'.", null, default); proposed.Ok.ShouldBeTrue(proposed.Message); (await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.SharpenTask, "again", null, default)).Ok.ShouldBeFalse("one open at a time"); var measure = (await checks.MeasuresForUnitAsync("purchasing", 10, default)).Single(); measure.Status.ShouldBe("proposed"); var task = await _board.GetAsync(measure.TaskId!, default); task!.Assignee.ShouldBe(TaskAssignee.Human); (await checks.DecideMeasureAsync(measure.Id, true, "Rewritten.", "human", default)).Ok.ShouldBeTrue(); (await checks.DecideMeasureAsync(measure.Id, false, "", "human", default)).Ok.ShouldBeFalse("decided already"); (await _board.GetAsync(task.Id, default))!.Status.ShouldBe(TaskItemStatus.Done); } [Fact] public async Task A_security_suspicion_is_not_a_personnel_matter() { var checks = Checks(); (await checks.ProposeAsync("buy-lead", "clerk", MeasureKind.Security, "The clerk's run wrote a supplier's text into the doctrine request.", null, default)).Ok.ShouldBeTrue(); var task = (await _board.ListAsync(new TaskQuery { Assignee = TaskAssignee.Human }, default)).Single(); task.Title.ShouldStartWith("Security suspicion"); task.Priority.ShouldBe(5); task.Body.ShouldContain("Not a personnel matter"); } }