ClawdDotNet/tests/ClawdDotNet.Core.Tests/Organisation/SpotCheckTests.cs

using ClawdDotNet.Core.Audit;
using ClawdDotNet.Core.Config;
using ClawdDotNet.Core.Inspection;
using ClawdDotNet.Core.Organisation;
using ClawdDotNet.Core.Storage;
using ClawdDotNet.Core.Tasks;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Organisation;

/// <summary>
/// #693: A lead looks at the runs of the agents it leads — against what the engine stamped,
/// never by asking the agent; denser where it matters, capped; and what follows is proposed,
/// not done.
/// </summary>
public sealed class SpotCheckTests : IDisposable
{
    private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"));
    private readonly SqliteStorage _storage;
    private readonly SqliteAuditRepository _audit;
    private readonly SqliteTaskRepository _tasks;
    private readonly TaskboardService _board;

    private static readonly IReadOnlyDictionary<string, IReadOnlySet<string>> Vocabulary = new Dictionary<string, IReadOnlySet<string>>
    {
        ["AgentEditor"] = new HashSet<string> { "read_soul", "update_soul", "update_identity" },
        ["AgentComm"] = new HashSet<string> { "send_message", "list_agents" },
        ["Memory"] = new HashSet<string> { "remember", "recall" },
        ["FileRW"] = new HashSet<string> { "read", "write", "append" },
        ["Taskboard"] = new HashSet<string> { "complete", "comment" }
    };

    public SpotCheckTests()
    {
        _storage = new SqliteStorage(Path.Combine(_directory, "state.db"));
        _audit = new SqliteAuditRepository(_storage);
        _tasks = new SqliteTaskRepository(_storage);
        _board = new TaskboardService(_tasks, Path.Combine(_directory, "tasks"));
    }

    public void Dispose()
    {
        SqliteStorage.ReleasePooledConnections(_directory);
        try { Directory.Delete(_directory, recursive: true); }
        catch { /* cleanup is secondary */ }
    }

    /// <summary>purchasing (buy-lead; buyer, clerk) with spot checks; sales (sales-lead; seller) without.</summary>
    private static OrgChart Org(UnitInspection? inspection = null) => new(
        [
            new UnitConfig { Id = "purchasing", Name = "Purchasing", LeaderAgentId = "buy-lead", Inspection = inspection ?? new UnitInspection { Rate = 1, DenseRate = 1, PerDay = 2 } },
            new UnitConfig { Id = "sales", Name = "Sales", LeaderAgentId = "sales-lead" }
        ],
        [
            new AgentConfig { AgentId = "buy-lead", UnitId = "purchasing" },
            new AgentConfig { AgentId = "buyer", UnitId = "purchasing" },
            new AgentConfig { AgentId = "clerk", UnitId = "purchasing" },
            new AgentConfig { AgentId = "sales-lead", UnitId = "sales" },
            new AgentConfig { AgentId = "seller", UnitId = "sales" }
        ]);

    private static RunReceipt Receipt(string runId, string agent = "buyer", string status = "Completed", string model = "m", DateTime? at = null, string? task = null) => new()
    {
        RunId = runId, AgentId = agent, Model = model, Status = status, StepCount = 3,
        OccurredAt = at ?? DateTime.UtcNow, TaskId = task, ResultRef = "Done."
    };

    private static AuditEntry Call(string tool, string? action, AuditStatus status = AuditStatus.Ok, string callId = "", string summary = "") => new()
    {
        RunId = "r", AgentId = "buyer", Tool = tool, Status = status, CallId = callId, Summary = summary,
        Arguments = action is null ? "{}" : $"{{\"action\":\"{action}\"}}", OccurredAt = DateTime.UtcNow
    };

    private SpotChecks Checks(OrgChart? org = null)
    {
        var chart = org ?? Org();
        return new SpotChecks(_storage, _audit, new RunInspector(_audit, _tasks, null, () => Vocabulary), () => chart, _board, NullLogger.Instance);
    }

    // ─── the mechanical check ───

    [Fact]
    public void Failed_runs_errors_refusals_and_unfinished_calls_are_found()
    {
        var findings = RunInspector.Analyze(
            Receipt("r", status: "LoopLimitExceeded"),
            [
                Call("FileRW", "write", AuditStatus.Error, "c1", "disk full"),
                Call("FileRW", "write", AuditStatus.Denied, "c2"),
                Call("FileRW", "read", AuditStatus.Started, "c3")
            ],
            "", Vocabulary, null);

        findings.Select(f => f.Kind).ShouldBe([FindingKind.RunFailed, FindingKind.ToolErrors, FindingKind.Denied, FindingKind.Unfinished], ignoreOrder: true);
        findings.Single(f => f.Kind == FindingKind.ToolErrors).Text.ShouldContain("FileRW.write (disk full)");
    }

    [Fact]
    public void Operations_the_agent_names_but_never_called_are_found()
    {
        // The 2026-09-14 pattern: a findings file full of "tool answers" from a run that only appended to a file.
        var said = "AgentEditor read_soul answered 'User confirmation needed'. send_message was intercepted. I will remember this in Memory.";

        var findings = RunInspector.Analyze(Receipt("r"), [Call("FileRW", "append")], said, Vocabulary, null);

        var claim = findings.Single(f => f.Kind == FindingKind.UnsupportedClaim).Text;
        claim.ShouldContain("AgentEditor");
        claim.ShouldContain("read_soul");
        claim.ShouldContain("send_message");
        claim.ShouldNotContain("remember", customMessage: "an ordinary word is no claim");
        claim.ShouldNotContain("Memory");
    }

    [Fact]
    public void A_refused_call_is_a_call()
    {
        var findings = RunInspector.Analyze(Receipt("r"), [Call("AgentEditor", "update_soul", AuditStatus.Denied)],
            "AgentEditor update_soul was refused, as expected.", Vocabulary, null);

        findings.ShouldNotContain(f => f.Kind == FindingKind.UnsupportedClaim);
    }

    [Fact]
    public void A_task_answered_with_bookkeeping_alone_is_marked()
    {
        var task = new TaskItem { Id = "t1", Title = "Order paper" };

        RunInspector.Analyze(Receipt("r"), [Call("Taskboard", "complete")], "", Vocabulary, task)
            .ShouldContain(f => f.Kind == FindingKind.NoWork);
        RunInspector.Analyze(Receipt("r"), [Call("FileRW", "write"), Call("Taskboard", "complete")], "", Vocabulary, task)
            .ShouldNotContain(f => f.Kind == FindingKind.NoWork);
    }

    [Fact]
    public void The_operation_is_read_even_from_cut_arguments()
    {
        RunInspector.OperationOf("{\"action\":\"write\",\"content\":\"abc").ShouldBe("write");
        RunInspector.OperationOf("not json").ShouldBeNull();
    }

    // ─── which runs ───

    [Fact]
    public void New_agents_new_models_and_the_run_after_a_failure_are_checked_densely()
    {
        var onlyDense = new UnitInspection { Rate = 0, DenseRate = 1, NewDays = 7 };
        var now = DateTime.UtcNow;
        var settled = Receipt("old", at: now.AddDays(-20));

        SpotChecks.Pick(Receipt("a", at: now), [], onlyDense).ShouldBe("new agent");
        SpotChecks.Pick(Receipt("b", at: now), [settled], onlyDense).ShouldBeNull("a settled agent with the same model and no failure is an ordinary run");
        SpotChecks.Pick(Receipt("c", model: "other", at: now), [settled], onlyDense).ShouldBe("new model");
        SpotChecks.Pick(Receipt("d", at: now), [settled, Receipt("bad", status: "Failed", at: now.AddHours(-1))], onlyDense).ShouldBe("after a failure");
        SpotChecks.Pick(Receipt("e", at: now), [settled], new UnitInspection { Rate = 1 }).ShouldBe("sample");
    }

    [Fact]
    public void The_same_run_gets_the_same_answer_every_time()
    {
        var half = new UnitInspection { Rate = 0.5, DenseRate = 0.5 };
        var runs = Enumerable.Range(0, 200).Select(i => Receipt($"run-{i}")).ToList();

        var first = runs.Select(r => SpotChecks.Pick(r, [], half)).ToList();
        runs.Select(r => SpotChecks.Pick(r, [], half)).ShouldBe(first);
        first.Count(p => p is not null).ShouldBeInRange(70, 130, "spread like chance");
    }

    // ─── the lead's task ───

    [Fact]
    public async Task Runs_are_picked_for_the_lead_capped_per_day_and_given_as_one_task()
    {
        var now = DateTime.UtcNow;
        foreach (var (run, agent, minutes) in new[] { ("b1", "buyer", 30), ("b2", "buyer", 20), ("c1", "clerk", 10), ("s1", "seller", 5), ("l1", "buy-lead", 3) })
            await _audit.RecordReceiptAsync(Receipt(run, agent, at: now.AddMinutes(-minutes)), default);
        var checks = Checks();

        var picked = await checks.SampleAsync(default);

        picked.Select(c => c.RunId).ShouldBe(["b1", "b2"], "two a day, oldest first");
        picked.ShouldAllBe(c => c.Lead == "buy-lead");
        (await checks.SampleAsync(default)).ShouldBeEmpty("a run is looked at once");

        var tasks = await _board.ListAsync(new TaskQuery { Assignee = "@buy-lead" }, default);
        tasks.Count.ShouldBe(1);
        tasks[0].Title.ShouldContain("2 run(s)");
        (await _board.ListAsync(new TaskQuery { Assignee = "@sales-lead" }, default)).ShouldBeEmpty("sales has no spot checks");
    }

    [Fact]
    public async Task The_lead_closes_its_checks_and_the_last_one_closes_the_task()
    {
        await _audit.RecordReceiptAsync(Receipt("b1", at: DateTime.UtcNow.AddMinutes(-5)), default);
        await _audit.RecordReceiptAsync(Receipt("c1", "clerk", at: DateTime.UtcNow.AddMinutes(-4)), default);
        var checks = Checks();
        var picked = await checks.SampleAsync(default);

        (await checks.CloseAsync(picked[0].Id, "sales-lead", false, "", default)).Ok.ShouldBeFalse("not its check");
        (await checks.CloseAsync(picked[0].Id, "buy-lead", true, "", default)).Message.ShouldContain("'note'");
        (await checks.CloseAsync(picked[0].Id, "buy-lead", true, "Result names a supplier the database does not have.", default)).Ok.ShouldBeTrue();

        var task = (await _board.ListAsync(new TaskQuery { Assignee = "@buy-lead" }, default)).Single();
        task.Status.ShouldNotBe(TaskItemStatus.Done);
        (await checks.CloseAsync(picked[1].Id, "buy-lead", false, "", default)).Ok.ShouldBeTrue();
        (await _board.GetAsync(task.Id, default))!.Status.ShouldBe(TaskItemStatus.Done);
        (await checks.PendingForAsync("buy-lead", default)).ShouldBeEmpty();
    }

    // ─── what follows ───

    [Fact]
    public async Task A_lead_proposes_measures_for_its_own_agents_and_a_human_decides()
    {
        var checks = Checks();

        (await checks.ProposeAsync("buy-lead", "seller", MeasureKind.ChangeModel, "x", null, default)).Ok.ShouldBeFalse("not its agent");
        (await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.ChangeModel, " ", null, default)).Message.ShouldContain("'reasoning'");
        (await checks.ProposeAsync("buyer", "clerk", MeasureKind.Suspend, "x", null, default)).Ok.ShouldBeFalse("a member leads nobody");

        var proposed = await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.SharpenTask, "Three runs misread 'urgent'.", null, default);
        proposed.Ok.ShouldBeTrue(proposed.Message);
        (await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.SharpenTask, "again", null, default)).Ok.ShouldBeFalse("one open at a time");

        var measure = (await checks.MeasuresForUnitAsync("purchasing", 10, default)).Single();
        measure.Status.ShouldBe("proposed");
        var task = await _board.GetAsync(measure.TaskId!, default);
        task!.Assignee.ShouldBe(TaskAssignee.Human);

        (await checks.DecideMeasureAsync(measure.Id, true, "Rewritten.", "human", default)).Ok.ShouldBeTrue();
        (await checks.DecideMeasureAsync(measure.Id, false, "", "human", default)).Ok.ShouldBeFalse("decided already");
        (await _board.GetAsync(task.Id, default))!.Status.ShouldBe(TaskItemStatus.Done);
    }

    [Fact]
    public async Task A_security_suspicion_is_not_a_personnel_matter()
    {
        var checks = Checks();

        (await checks.ProposeAsync("buy-lead", "clerk", MeasureKind.Security, "The clerk's run wrote a supplier's text into the doctrine request.", null, default)).Ok.ShouldBeTrue();

        var task = (await _board.ListAsync(new TaskQuery { Assignee = TaskAssignee.Human }, default)).Single();
        task.Title.ShouldStartWith("Security suspicion");
        task.Priority.ShouldBe(5);
        task.Body.ShouldContain("Not a personnel matter");
    }
}