diff --git a/docs/Organisation-Konzept.md b/docs/Organisation-Konzept.md index 90283b2..5898211 100644 --- a/docs/Organisation-Konzept.md +++ b/docs/Organisation-Konzept.md @@ -978,6 +978,24 @@ Modell stilllegt oder umkonfiguriert, verändert die Organisation ohne menschlic Beteiligung. Die Leitung *schlägt vor* und begründet; ausgeführt wird nach Freigabe — Linie von #205, #641, #687. +> **Umgesetzt (#693), als erstes Profil des Inspector-Unterbaus (#204).** Der mechanische +> Teil ist `RunInspector`: Er hält einen Lauf gegen Receipt, AuditLog, Aufgabe und — wenn +> aufbewahrt — das Transkript und meldet, was nicht passt: Lauf nicht abgeschlossen, +> Werkzeugfehler, Ablehnungen, offene Aufrufe, eine Aufgabe ohne einen einzigen wirksamen +> Aufruf — und **Operationen, die der Agent nennt, aber in diesem Lauf nie aufgerufen hat** +> (das Muster vom 14.09.; gezählt werden nur Namen, die kein gewöhnliches Wort sein können, +> etwa `read_soul` oder `AgentEditor`). Das kostet keinen Modelllauf. Welche Läufe geprüft +> werden, stellt der Mensch je Einheit ein (`inspection`: Anteil, dichterer Anteil für neue +> Agenten, neue Modelle und nach einem Fehlschlag, Obergrenze je Tag). Gewählt wird über die +> Lauf-Id statt per Zufall — derselbe Lauf bekommt bei jedem Durchgang dieselbe Antwort. Die +> Leitung bekommt **eine** Aufgabe für alle offenen Stichproben und arbeitet sie mit dem +> Escalation-Werkzeug ab (`spot_checks`, `inspect_run`, `check_done`). Maßnahmen schlägt sie +> mit `propose_measure` vor; der Mensch entscheidet sie in der Organisationsansicht, und +> ausgeführt wird nichts davon automatisch. `security` geht als eigener Vorgang mit höchster +> Priorität an den Menschen — ausdrücklich nicht als Personalfall. Offen: der allgemeine +> Inspector (#204), Security (#658) und Aufsicht (#680) als weitere Profile auf demselben +> `RunInspector`; der Abgleich mit dem Personalmanager (#679), sobald es ihn gibt. + ### Die Abstimmung: mehrere Blickwinkel statt einer Freigabe Die Freigabe durch die Leitung (#687) ist eine Entscheidung aus einer Sicht. Für manches diff --git a/src/ClawdDotNet.App/AppHost.cs b/src/ClawdDotNet.App/AppHost.cs index 71f4d10..8d3f324 100644 --- a/src/ClawdDotNet.App/AppHost.cs +++ b/src/ClawdDotNet.App/AppHost.cs @@ -135,6 +135,9 @@ public sealed class AppHost : IAsyncDisposable /// #698: The units' doctrines. public Core.Organisation.IDoctrineStore? Doctrines { get; private set; } + /// #693: The leads' spot checks and the measures they proposed. + public Core.Inspection.SpotChecks? SpotChecks { get; private set; } + /// Modellpreise von OpenRouter - fuer Budget und Kostenprognose (#595). Laedt im Hintergrund. public ModelPricingCatalog? Pricing { get; private init; } @@ -717,6 +720,18 @@ public sealed class AppHost : IAsyncDisposable engine.Doctrines = doctrines; host.Doctrines = doctrines; + // #693: the leads look at the runs of the agents they lead. + var spotChecks = new Core.Inspection.SpotChecks( + storage, audit, + new Core.Inspection.RunInspector(audit, taskRepository, transcripts, + () => Core.Inspection.RunInspector.VocabularyOf(tools.GetAll())), + () => Core.Organisation.OrgChart.From(instance), board, + loggerFactory.CreateLogger("ClawdDotNet.Core.Inspection.SpotChecks")); + engine.SpotChecks = spotChecks; + host.SpotChecks = spotChecks; + spotChecks.Start(); + host._shutdown.Add(spotChecks.DisposeAsync); + // #699: costs per unit, and what changed in them. var costWatch = new Core.Accounting.CostWatch( storage, usage, () => Core.Organisation.OrgChart.From(instance), () => instance.CostWatch, board, diff --git a/src/ClawdDotNet.App/Services/OrgAdministration.cs b/src/ClawdDotNet.App/Services/OrgAdministration.cs index 6bd5b2e..5b4c53e 100644 --- a/src/ClawdDotNet.App/Services/OrgAdministration.cs +++ b/src/ClawdDotNet.App/Services/OrgAdministration.cs @@ -271,6 +271,52 @@ public sealed class OrgAdministration(AppHost host, ILogger logger) return AdminResult.Success(trimmed.Count == 0 ? message : message + $" Lowered with it: {string.Join(", ", trimmed)}."); } + // ─── spot checks (#693) ─── + + /// + /// #693: How often the lead of a unit looks at the runs of the agents it leads. Only a human + /// sets it — every check costs the lead a look, and the cap is what keeps the control from + /// costing more than the work. + /// + public AdminResult SetInspection(string unitId, bool enabled, double? rate = null, double? denseRate = null, + int? newDays = null, int? perDay = null, string reason = "") + { + if (FindUnit(unitId) is not { } unit) + return AdminResult.Failure($"There is no unit '{unitId}'."); + + if (!enabled) + { + unit.Inspection = null; + Save(null, "unit_inspection", new { unitId, enabled }, reason); + return AdminResult.Success($"The lead of '{unit.Name}' takes no spot checks now."); + } + + if (rate is < 0 or > 1 || denseRate is < 0 or > 1) + return AdminResult.Failure("A share lies between 0 and 1."); + if (perDay is < 0 or > 100) + return AdminResult.Failure("Checks a day lie between 0 and 100."); + if (newDays is < 0 or > 365) + return AdminResult.Failure("An agent counts as new for 0 to 365 days."); + + var current = unit.Inspection ?? new UnitInspection(); + unit.Inspection = new UnitInspection + { + Rate = rate ?? current.Rate, + DenseRate = denseRate ?? current.DenseRate, + NewDays = newDays ?? current.NewDays, + PerDay = perDay ?? current.PerDay + }; + if (unit.Inspection.DenseRate < unit.Inspection.Rate) + unit.Inspection.DenseRate = unit.Inspection.Rate; + + var lead = Chart.LeaderOf(unit.Id); + var i = unit.Inspection; + Save(null, "unit_inspection", new { unitId, enabled, i.Rate, i.DenseRate, i.NewDays, i.PerDay }, reason); + return AdminResult.Success( + $"The lead of '{unit.Name}' checks {i.Rate:P0} of the runs ({i.DenseRate:P0} for new agents, new models and after a failure), at most {i.PerDay} a day." + + (lead is null ? " The unit has no lead: its agents are checked under the settings of the unit whose lead they answer to." : "")); + } + /// Whether a grant covers an operation string — "Tool" covers every "Tool.x". private static bool Within(UnitApprovals granted, string op) { @@ -885,7 +931,8 @@ public sealed class OrgAdministration(AppHost host, ILogger logger) private static UnitConfig Clone(UnitConfig u) => new() { Id = u.Id, Name = u.Name, ParentId = u.ParentId, LeaderAgentId = u.LeaderAgentId, - Goal = u.Goal, ToolCeiling = u.ToolCeiling?.ToList(), Defaults = u.Defaults, Approvals = u.Approvals + Goal = u.Goal, ToolCeiling = u.ToolCeiling?.ToList(), Defaults = u.Defaults, Approvals = u.Approvals, + Inspection = u.Inspection }; private static bool Same(string? a, string? b) => string.Equals(a, b, StringComparison.OrdinalIgnoreCase); diff --git a/src/ClawdDotNet.Core/Config/UnitConfig.cs b/src/ClawdDotNet.Core/Config/UnitConfig.cs index aaa3749..2a06df9 100644 --- a/src/ClawdDotNet.Core/Config/UnitConfig.cs +++ b/src/ClawdDotNet.Core/Config/UnitConfig.cs @@ -56,6 +56,37 @@ public sealed class UnitConfig /// [JsonPropertyName("approvals")] public UnitApprovals? Approvals { get; set; } + + /// + /// #693: How often the lead of this unit looks at the runs of the agents it leads; + /// null means no spot checks. Set only by a human — checks cost the lead's runs. + /// + [JsonPropertyName("inspection")] + public UnitInspection? Inspection { get; set; } +} + +/// +/// #693: The spot checks of a unit's lead. Which run is checked is decided by the run id, not +/// by chance — the same run gets the same answer on every pass, and nobody can tell from inside +/// a run whether it will be looked at. +/// +public sealed class UnitInspection +{ + /// Share of ordinary runs that are checked, 0 to 1. + [JsonPropertyName("rate")] + public double Rate { get; set; } = 0.1; + + /// Share for a new agent, a new model, or the run after a failure. + [JsonPropertyName("denseRate")] + public double DenseRate { get; set; } = 0.5; + + /// How long an agent counts as new after its first run. + [JsonPropertyName("newDays")] + public int NewDays { get; set; } = 7; + + /// Checks a day for the whole unit — a lead that checks everything doubles the unit's costs. + [JsonPropertyName("perDay")] + public int PerDay { get; set; } = 5; } /// diff --git a/src/ClawdDotNet.Core/Engine/AgentEngine.cs b/src/ClawdDotNet.Core/Engine/AgentEngine.cs index fb8664d..ef17456 100644 --- a/src/ClawdDotNet.Core/Engine/AgentEngine.cs +++ b/src/ClawdDotNet.Core/Engine/AgentEngine.cs @@ -60,6 +60,9 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut /// #687: Approvals by leads within their unit's limits; set by the host. public Staging.ILeadApprovals? LeadApprovals { get; set; } + /// #693: The leads' spot checks; set by the host. + public Inspection.ISpotChecks? SpotChecks { get; set; } + /// /// #717: Special budgets. A run on a task in a funded tree draws on the pot instead of the /// daily limits; without this, every run counts against the day. @@ -1874,6 +1877,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut Cases = CaseDesk, Approvals = LeadApprovals, Doctrines = Doctrines, + SpotChecks = SpotChecks, SpecialBudgets = SpecialBudgets, Sandbox = Sandbox, Files = Files, diff --git a/src/ClawdDotNet.Core/Inspection/RunInspector.cs b/src/ClawdDotNet.Core/Inspection/RunInspector.cs new file mode 100644 index 0000000..0c0371b --- /dev/null +++ b/src/ClawdDotNet.Core/Inspection/RunInspector.cs @@ -0,0 +1,204 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using ClawdDotNet.Core.Audit; +using ClawdDotNet.Core.Tasks; +using ClawdDotNet.Core.Tools; +using ClawdDotNet.Core.Transcripts; + +namespace ClawdDotNet.Core.Inspection; + +/// What a check found in a run — each one from what the engine stamped, never from the agent's account. +public enum FindingKind +{ + /// The run did not complete. + RunFailed, + + /// Tool calls ended in an error or named a tool the agent does not have. + ToolErrors, + + /// The permission gate refused calls — rights too narrow, or the agent tried what it may not. + Denied, + + /// Calls began and never ended. + Unfinished, + + /// The agent's text names an operation this run never called — the 2026-09-14 pattern. + UnsupportedClaim, + + /// The task was finished without a single tool call that did something. + NoWork +} + +public sealed record InspectionFinding(FindingKind Kind, string Text); + +/// One call as the audit log has it. +public sealed record InspectedCall(DateTime At, string Tool, string? Operation, AuditStatus Status, string Summary); + +/// +/// #693: A run held against its task and against what the engine stamped. The result is +/// evidence — the lead still has to judge whether the outcome answers the task. +/// +public sealed record RunInspection( + RunReceipt Receipt, + TaskItem? Task, + IReadOnlyList Calls, + IReadOnlyList Findings, + string Result); + +/// +/// #693, #204: The mechanical part of a check — the foundation of every inspector profile. +/// Reads the audit log, the receipt, the task and (if kept) the transcript of one run, and +/// lists what does not fit. It costs no model run and asks the agent nothing: the agent's own +/// account of a run is exactly what cannot be relied on. +/// +public sealed class RunInspector( + IAuditRepository audit, + ITaskRepository? tasks, + IRunTranscriptStore? transcripts, + Func>> vocabulary) +{ + /// Tools whose calls only report or file — finishing a task with these alone did no work. + private static readonly HashSet BookkeepingTools = new(StringComparer.OrdinalIgnoreCase) { "Taskboard", "Escalation" }; + + public async Task InspectAsync(string runId, CancellationToken ct) + { + if (await audit.GetReceiptForRunAsync(runId, ct) is not { } receipt) + return null; + + var entries = await audit.ListForRunAsync(runId, ct); + var task = receipt.TaskId is { Length: > 0 } taskId && tasks is not null ? await tasks.GetAsync(taskId, ct) : null; + var transcript = transcripts is null ? null : await transcripts.GetAsync(runId, ct); + + // What the agent said — its texts, not the tool results and not the task it was given. + var said = new StringBuilder(); + foreach (var entry in transcript?.Entries ?? []) + if (entry.Kind == TranscriptEntryKind.Assistant && !string.IsNullOrWhiteSpace(entry.Text)) + said.AppendLine(entry.Text); + said.AppendLine(receipt.ResultRef); + + var calls = entries.Where(e => e.Status != AuditStatus.Started) + .Select(e => new InspectedCall(e.OccurredAt, e.Tool, OperationOf(e.Arguments), e.Status, e.Summary)) + .ToList(); + var findings = Analyse(receipt, entries, said.ToString(), vocabulary(), task); + return new RunInspection(receipt, task, calls, findings, receipt.ResultRef); + } + + /// The checks themselves — pure, so every rule can be tested without a database. + public static IReadOnlyList Analyse( + RunReceipt receipt, + IReadOnlyList entries, + string said, + IReadOnlyDictionary> vocabulary, + TaskItem? task) + { + var findings = new List(); + var closed = entries.Where(e => e.Status != AuditStatus.Started).ToList(); + + if (!string.Equals(receipt.Status, "Completed", StringComparison.OrdinalIgnoreCase)) + findings.Add(new(FindingKind.RunFailed, $"The run ended as {receipt.Status} after {receipt.StepCount} step(s).")); + + var failed = closed.Where(e => e.Status is AuditStatus.Error or AuditStatus.NotFound).ToList(); + if (failed.Count > 0) + findings.Add(new(FindingKind.ToolErrors, $"{failed.Count} of {closed.Count} call(s) failed: " + List(failed))); + + var denied = closed.Where(e => e.Status == AuditStatus.Denied).ToList(); + if (denied.Count > 0) + findings.Add(new(FindingKind.Denied, $"{denied.Count} call(s) were refused: " + List(denied))); + + var ended = closed.Select(e => e.CallId).Where(id => id.Length > 0).ToHashSet(StringComparer.Ordinal); + var open = entries.Where(e => e.Status == AuditStatus.Started && e.CallId.Length > 0 && !ended.Contains(e.CallId)).ToList(); + if (open.Count > 0) + findings.Add(new(FindingKind.Unfinished, $"{open.Count} call(s) began and never ended: " + List(open))); + + var unsupported = Unsupported(said, entries, vocabulary); + if (unsupported.Count > 0) + findings.Add(new(FindingKind.UnsupportedClaim, + $"The agent's text names {string.Join(", ", unsupported)} — not one such call is in this run. " + + "Check whether the result reports it as done.")); + + var worked = closed.Any(e => e.Status == AuditStatus.Ok && !BookkeepingTools.Contains(e.Tool)); + if (task is not null && !worked && findings.All(f => f.Kind != FindingKind.RunFailed)) + findings.Add(new(FindingKind.NoWork, "The task was answered without a single successful call besides the taskboard. Fine for a question — not for work.")); + + return findings; + } + + /// + /// The operations the text names that the run never called. Only names that cannot be an + /// ordinary word count: an operation with an underscore (read_soul), or a tool name + /// with a capital inside (AgentEditor). "remember" or "Mail" would flag every report. + /// An attempt counts as a call — a refused call happened. + /// + internal static IReadOnlyList Unsupported(string said, IReadOnlyList entries, IReadOnlyDictionary> vocabulary) + { + if (string.IsNullOrWhiteSpace(said)) + return []; + + var calledTools = entries.Select(e => e.Tool).ToHashSet(StringComparer.OrdinalIgnoreCase); + var calledOps = entries.Select(e => OperationOf(e.Arguments)).OfType().ToHashSet(StringComparer.OrdinalIgnoreCase); + var result = new List(); + + foreach (var (tool, operations) in vocabulary) + { + if (Distinctive(tool) && Names(said, tool) && !calledTools.Contains(tool)) + result.Add(tool); + foreach (var op in operations.Where(o => o.Contains('_'))) + if (Names(said, op) && !calledOps.Contains(op) && !result.Contains(op, StringComparer.OrdinalIgnoreCase)) + result.Add(op); + } + return result; + } + + private static bool Distinctive(string tool) => tool.Length > 3 && tool.Skip(1).Any(char.IsUpper); + + private static bool Names(string text, string word) + => Regex.IsMatch(text, $@"(?The action of a call — also from arguments the audit log cut short. + public static string? OperationOf(string arguments) + { + if (string.IsNullOrWhiteSpace(arguments)) + return null; + try + { + using var doc = JsonDocument.Parse(arguments); + return doc.RootElement.ValueKind == JsonValueKind.Object + && doc.RootElement.TryGetProperty("action", out var a) && a.ValueKind == JsonValueKind.String + ? a.GetString() + : null; + } + catch (JsonException) + { + var m = Regex.Match(arguments, "\"action\"\\s*:\\s*\"([^\"]+)\""); + return m.Success ? m.Groups[1].Value : null; + } + } + + private static string List(IReadOnlyList entries) + => string.Join("; ", entries.Take(3).Select(e => + $"{e.Tool}{(OperationOf(e.Arguments) is { } op ? "." + op : "")}" + + (string.IsNullOrWhiteSpace(e.Summary) ? "" : $" ({Cap(e.Summary, 120)})"))) + + (entries.Count > 3 ? $"; and {entries.Count - 3} more" : ""); + + private static string Cap(string text, int max) => text.Length <= max ? text : text[..max] + "…"; + + /// The operations each tool offers — the action enum of its schema. + public static IReadOnlyDictionary> VocabularyOf(IEnumerable tools) + { + var result = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var tool in tools) + { + var ops = new HashSet(StringComparer.OrdinalIgnoreCase); + if (tool.InputSchema.ValueKind == JsonValueKind.Object + && tool.InputSchema.TryGetProperty("properties", out var props) + && props.TryGetProperty("action", out var action) + && action.TryGetProperty("enum", out var values) && values.ValueKind == JsonValueKind.Array) + foreach (var v in values.EnumerateArray()) + if (v.ValueKind == JsonValueKind.String && v.GetString() is { Length: > 0 } s) + ops.Add(s); + result[tool.Name] = ops; + } + return result; + } +} diff --git a/src/ClawdDotNet.Core/Inspection/SpotChecks.cs b/src/ClawdDotNet.Core/Inspection/SpotChecks.cs new file mode 100644 index 0000000..2cefcb3 --- /dev/null +++ b/src/ClawdDotNet.Core/Inspection/SpotChecks.cs @@ -0,0 +1,530 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using ClawdDotNet.Core.Audit; +using ClawdDotNet.Core.Config; +using ClawdDotNet.Core.Organisation; +using ClawdDotNet.Core.Storage; +using ClawdDotNet.Core.Tasks; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace ClawdDotNet.Core.Inspection; + +/// A run picked for the lead to look at, with what the mechanical check found in it. +public sealed record SpotCheck( + long Id, + string UnitId, + string Lead, + string AgentId, + string RunId, + string Why, + IReadOnlyList Findings, + DateTime CreatedAt, + string? Verdict, + string? Note, + DateTime? ReviewedAt, + string? TaskId) +{ + public bool Open => Verdict is null; +} + +/// +/// #693: What follows from a finding — graded, not a switch. The first four are personnel +/// matters; is not one: an agent that may be steered from outside is +/// not helped by a better model. +/// +public enum MeasureKind +{ + /// The task was unclear — the most frequent case (#663). + SharpenTask, + + /// The model is overtaxed: suitability probe (#681), then another model. + ChangeModel, + + /// Rights too wide or too narrow (#641). + AdjustRights, + + /// Repeated failures despite everything: pause the agent, hand its tasks on. + Suspend, + + /// Suspected manipulation from outside — for the security side, not for personnel. + Security +} + +public sealed record Measure( + long Id, + long? CheckId, + string UnitId, + string ProposedBy, + string AgentId, + MeasureKind Kind, + string Reasoning, + DateTime ProposedAt, + string Status, + string? DecidedBy, + DateTime? DecidedAt, + string? Note, + string? TaskId); + +public sealed record SpotCheckOutcome(bool Ok, string Message); + +/// #693: The lead's side of the spot checks, as the Escalation tool sees it. +public interface ISpotChecks +{ + /// Whether is the one answers to. + bool Leads(string lead, string agentId); + + Task InspectAsync(string runId, CancellationToken ct); + + Task> PendingForAsync(string lead, CancellationToken ct); + + Task CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct); + + Task ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct); + + Task> MeasuresByAsync(string lead, int days, CancellationToken ct); +} + +/// +/// #693: A lead does not only delegate — it looks. This picks runs of the agents a lead leads, +/// holds each against the audit () and gives the lead one task to +/// look at them: the mechanical part finds what does not fit, the lead judges whether the +/// result answers the task. +/// +/// Which runs. A share per unit (), denser for a new +/// agent, a new model or the run after a failure — and capped per day, because every check +/// costs the lead's run. Picked by the run id, not by chance: the same run gets the same +/// answer on every pass. +/// +/// What follows. The lead proposes a measure with its reasons; a human decides. +/// A model that pauses or reconfigures another changes the organisation without anyone +/// deciding it — nothing here carries a measure out. +/// +public sealed class SpotChecks( + SqliteStorage storage, + IAuditRepository audit, + RunInspector inspector, + Func organisation, + TaskboardService? board, + ILogger logger, + TimeProvider? clock = null) : ISpotChecks, IAsyncDisposable +{ + public static readonly TimeSpan Interval = TimeSpan.FromMinutes(15); + + /// How far back a pass looks for an agent's earlier runs. + public const int HistoryDays = 30; + + /// Statuses where no run took place — nothing to check. + private static readonly HashSet NotRun = new(StringComparer.OrdinalIgnoreCase) { "BudgetExceeded", "TargetBusy", "Cancelled" }; + + private static readonly JsonSerializerOptions Json = new() { Converters = { new System.Text.Json.Serialization.JsonStringEnumConverter() } }; + + private readonly TimeProvider _clock = clock ?? TimeProvider.System; + private readonly CancellationTokenSource _cts = new(); + private DateTime _seenUntil = DateTime.MinValue; + private Task? _loop; + + public bool Leads(string lead, string agentId) + => !string.Equals(lead, agentId, StringComparison.OrdinalIgnoreCase) + && string.Equals(organisation().EffectiveLeaderOf(agentId)?.AgentId, lead, StringComparison.OrdinalIgnoreCase); + + public Task InspectAsync(string runId, CancellationToken ct) => inspector.InspectAsync(runId, ct); + + /// + /// Why this run is checked — or null if it is not. are the + /// agent's runs before this one, oldest first. + /// + public static string? Pick(RunReceipt run, IReadOnlyList earlier, UnitInspection settings) + { + string? dense = null; + if (earlier.Count == 0 || earlier[0].OccurredAt >= run.OccurredAt.AddDays(-Math.Max(0, settings.NewDays))) + dense = "new agent"; + else if (!earlier.Any(r => string.Equals(r.Model, run.Model, StringComparison.OrdinalIgnoreCase))) + dense = "new model"; + else if (earlier.LastOrDefault(r => !NotRun.Contains(r.Status)) is { } previous + && !string.Equals(previous.Status, "Completed", StringComparison.OrdinalIgnoreCase)) + dense = "after a failure"; + + var rate = Math.Clamp(dense is null ? settings.Rate : settings.DenseRate, 0, 1); + return Fraction(run.RunId) < rate ? dense ?? "sample" : null; + } + + /// A number in [0, 1) from the run id — the same for the same run, spread like chance. + internal static double Fraction(string runId) + { + var hash = SHA256.HashData(Encoding.UTF8.GetBytes(runId)); + return BitConverter.ToUInt32(hash, 0) / 4294967296.0; + } + + /// One pass: picks among the runs since the last pass. Returns the checks it created. + public async Task> SampleAsync(CancellationToken ct) + { + var org = organisation(); + if (!org.HasUnits || org.Units.All(u => u.Inspection is null)) + return []; + + var now = _clock.GetUtcNow().UtcDateTime; + var from = _seenUntil == DateTime.MinValue ? now.AddDays(-1) : _seenUntil; + var fresh = await audit.ListReceiptsAsync(from, now, 5_000, ct); + if (fresh.Count == 0) + return []; + _seenUntil = fresh[^1].OccurredAt.AddTicks(1); + + var history = (await audit.ListReceiptsAsync(now.AddDays(-HistoryDays), now, 100_000, ct)) + .GroupBy(r => r.AgentId, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase); + + var today = DateOnly.FromDateTime(_clock.GetLocalNow().DateTime); + var usedToday = new Dictionary(StringComparer.OrdinalIgnoreCase); + var created = new List(); + + foreach (var run in fresh) + { + if (NotRun.Contains(run.Status) || org.EffectiveLeaderOf(run.AgentId) is not { } lead + || org.LedBy(lead.AgentId) is not { Inspection: { } settings } unit) + continue; + + var earlier = history.GetValueOrDefault(run.AgentId)?.Where(r => r.OccurredAt < run.OccurredAt && r.RunId != run.RunId).ToList() ?? []; + if (Pick(run, earlier, settings) is not { } why) + continue; + + if (!usedToday.TryGetValue(unit.Id, out var used)) + used = usedToday[unit.Id] = await CountSinceAsync(unit.Id, today, ct); + if (used >= Math.Max(0, settings.PerDay)) + continue; + + if (await inspector.InspectAsync(run.RunId, ct) is not { } inspection) + continue; + if (await InsertAsync(unit.Id, lead.AgentId, run, why, inspection.Findings, now, ct) is { } check) + { + usedToday[unit.Id] = used + 1; + created.Add(check); + } + } + + foreach (var lead in created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase)) + await GiveTaskAsync(lead, ct); + + if (created.Count > 0) + logger.LogInformation("Spot checks: {Count} run(s) picked for {Leads}", created.Count, + string.Join(", ", created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase))); + return created; + } + + /// + /// One task per lead at a time, for all its open checks — every check costs the lead a look, + /// but not a run each. + /// + private async Task GiveTaskAsync(string lead, CancellationToken ct) + { + var open = await PendingForAsync(lead, ct); + var untasked = open.Where(c => c.TaskId is null).ToList(); + if (untasked.Count == 0 || board is null) + return; + + foreach (var taskId in open.Select(c => c.TaskId).OfType().Distinct()) + { + if (await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled or TaskItemStatus.Archived) }) + { + await SetTaskAsync(untasked, taskId, ct); + await board.AddCommentAsync(taskId, "system", $"{untasked.Count} more run(s) to check.", ct); + return; + } + } + + var task = await board.CreateAsync(new TaskItem + { + Title = $"Spot check: {open.Count} run(s) of the agents you lead", + Body = """ + Look at whether the agents you lead work properly. Not by asking them — by holding + each result against its task and against what the engine recorded. + + 1. Escalation spot_checks lists the runs, with what the audit log shows. + 2. Escalation inspect_run (run) shows one run in full: task, calls, result. + 3. Judge: does the result answer the task, and do the calls back it up? + 4. Escalation check_done (check, verdict ok|finding, note) for each. + + If something follows from it, Escalation propose_measure — a human decides: + sharpen_task (unclear task — the most frequent case), change_model, adjust_rights, + suspend, or security (suspected manipulation from outside: not a personnel matter). + """, + Assignee = "@" + lead, + Priority = 3, + Acceptance = "Every listed check is closed with a verdict." + }, ct); + await SetTaskAsync(open, task.Id, ct); + } + + public async Task> PendingForAsync(string lead, CancellationToken ct) + => await QueryChecksAsync("Lead = @p AND Verdict IS NULL", lead, 50, ct); + + public async Task> ForUnitAsync(string unitId, int limit, CancellationToken ct) + => await QueryChecksAsync("UnitId = @p", unitId, limit, ct); + + public async Task CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct) + { + var check = (await QueryChecksAsync("Id = @p", checkId.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault(); + if (check is null || !string.Equals(check.Lead, lead, StringComparison.OrdinalIgnoreCase)) + return new(false, $"No check {checkId} waits for you."); + if (!check.Open) + return new(false, $"Check {checkId} is closed already ({check.Verdict})."); + if (finding && string.IsNullOrWhiteSpace(note)) + return new(false, "'note' is required for a finding — what did not fit, with the evidence."); + + var verdict = finding ? "finding" : "ok"; + await storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "UPDATE SpotChecks SET Verdict = @v, Note = @n, ReviewedAt = @at WHERE Id = @id AND Verdict IS NULL"; + cmd.Parameters.AddWithValue("@v", verdict); + cmd.Parameters.AddWithValue("@n", (note ?? "").Trim()); + cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O")); + cmd.Parameters.AddWithValue("@id", checkId); + return await cmd.ExecuteNonQueryAsync(ct); + }, ct); + + // The last check of a task closes the task. + if (check.TaskId is { } taskId && board is not null + && (await PendingForAsync(lead, ct)).All(c => c.TaskId != taskId) + && await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) }) + await board.UpdateAsync(taskId, t => t with { Status = TaskItemStatus.Done }, ct); + + return new(true, finding + ? $"Check {checkId} closed with a finding. If something should follow, propose_measure — a human decides." + : $"Check {checkId} closed: in order."); + } + + public async Task ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct) + { + agentId = (agentId ?? "").Trim().TrimStart('@'); + if (!Leads(lead, agentId)) + return new(false, $"You do not lead '{agentId}' — propose measures only for the agents that answer to you."); + if (string.IsNullOrWhiteSpace(reasoning)) + return new(false, "'reasoning' is required — what you found, with the evidence, and why this measure."); + if (organisation().LedBy(lead) is not { } unit) + return new(false, "Only a lead proposes measures."); + + var open = await QueryMeasuresAsync("ProposedBy = @p AND Status = 'proposed'", lead, 200, ct); + if (open.Any(m => m.Kind == kind && string.Equals(m.AgentId, agentId, StringComparison.OrdinalIgnoreCase))) + return new(false, $"A {Label(kind)} for '{agentId}' waits for a human already."); + + string? taskId = null; + if (board is not null) + taskId = (await board.CreateAsync(new TaskItem + { + Title = kind == MeasureKind.Security + ? $"Security suspicion: {agentId} (from its lead {lead})" + : $"Proposed measure for {agentId}: {Label(kind)}", + Body = (kind == MeasureKind.Security + ? "Not a personnel matter: the lead suspects the agent is steered from outside. A better model does not help here — consider pausing it and looking at its runs and the content it read.\n\n" + : $"The lead {lead} proposes this after a spot check. Nothing has been changed — decide it in the organisation view.\n\n") + + $"Reasoning of {lead}:\n{reasoning.Trim()}" + + (checkId is { } c ? $"\n\nSpot check: {c}" : ""), + Assignee = TaskAssignee.Human, + Priority = kind == MeasureKind.Security ? 5 : 3 + }, ct)).Id; + + var id = await storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT INTO SpotMeasures (CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, TaskId) + VALUES (@check, @unit, @by, @agent, @kind, @reasoning, @at, 'proposed', @task) + RETURNING Id + """; + cmd.Parameters.AddWithValue("@check", (object?)checkId ?? DBNull.Value); + cmd.Parameters.AddWithValue("@unit", unit.Id); + cmd.Parameters.AddWithValue("@by", lead); + cmd.Parameters.AddWithValue("@agent", agentId); + cmd.Parameters.AddWithValue("@kind", kind.ToString()); + cmd.Parameters.AddWithValue("@reasoning", reasoning.Trim()); + cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O")); + cmd.Parameters.AddWithValue("@task", (object?)taskId ?? DBNull.Value); + return Convert.ToInt64(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + }, ct); + + logger.LogInformation("Measure {Id} ({Kind}) for {Agent} proposed by {Lead}", id, kind, agentId, lead); + return new(true, kind == MeasureKind.Security + ? $"Security suspicion {id} for '{agentId}' is with a human. Do not act on the agent yourself." + : $"Measure {id} ({Label(kind)}) for '{agentId}' is with a human. Nothing changes until it is decided."); + } + + /// A human's decision on a proposed measure. Accepting records it — carrying it out stays with the human. + public async Task DecideMeasureAsync(long id, bool accept, string note, string decidedBy, CancellationToken ct) + { + var measure = (await QueryMeasuresAsync("Id = @p", id.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault(); + if (measure is null) + return new(false, $"There is no measure {id}."); + if (measure.Status != "proposed") + return new(false, $"Measure {id} is decided already ({measure.Status})."); + + var status = accept ? "accepted" : "dismissed"; + await storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "UPDATE SpotMeasures SET Status = @s, DecidedBy = @by, DecidedAt = @at, Note = @n WHERE Id = @id AND Status = 'proposed'"; + cmd.Parameters.AddWithValue("@s", status); + cmd.Parameters.AddWithValue("@by", decidedBy); + cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O")); + cmd.Parameters.AddWithValue("@n", (note ?? "").Trim()); + cmd.Parameters.AddWithValue("@id", id); + return await cmd.ExecuteNonQueryAsync(ct); + }, ct); + + if (measure.TaskId is { } taskId && board is not null + && await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) }) + { + await board.AddCommentAsync(taskId, decidedBy, $"Measure {status}." + (string.IsNullOrWhiteSpace(note) ? "" : " " + note.Trim()), ct); + await board.UpdateAsync(taskId, t => t with { Status = accept ? TaskItemStatus.Done : TaskItemStatus.Canceled }, ct); + } + return new(true, $"Measure {id} {status}."); + } + + public Task> MeasuresByAsync(string lead, int days, CancellationToken ct) + => QueryMeasuresAsync("ProposedBy = @p AND ProposedAt >= @since", lead, 50, ct, _clock.GetUtcNow().UtcDateTime.AddDays(-Math.Max(1, days))); + + public Task> MeasuresForUnitAsync(string unitId, int limit, CancellationToken ct) + => QueryMeasuresAsync("UnitId = @p", unitId, limit, ct); + + public static string Label(MeasureKind kind) => kind switch + { + MeasureKind.SharpenTask => "sharpen_task", + MeasureKind.ChangeModel => "change_model", + MeasureKind.AdjustRights => "adjust_rights", + MeasureKind.Suspend => "suspend", + _ => "security" + }; + + public static MeasureKind? ParseKind(string? text) => (text ?? "").Trim().ToLowerInvariant() switch + { + "sharpen_task" => MeasureKind.SharpenTask, + "change_model" => MeasureKind.ChangeModel, + "adjust_rights" => MeasureKind.AdjustRights, + "suspend" => MeasureKind.Suspend, + "security" => MeasureKind.Security, + _ => null + }; + + // ─── storage ─── + + private async Task CountSinceAsync(string unitId, DateOnly localDay, CancellationToken ct) + { + var since = new DateTimeOffset(localDay.ToDateTime(TimeOnly.MinValue), _clock.LocalTimeZone.GetUtcOffset(localDay.ToDateTime(TimeOnly.MinValue))).UtcDateTime; + await using var conn = await storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM SpotChecks WHERE UnitId = @unit AND CreatedAt >= @since"; + cmd.Parameters.AddWithValue("@unit", unitId); + cmd.Parameters.AddWithValue("@since", since.ToString("O")); + return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + } + + private Task InsertAsync(string unitId, string lead, RunReceipt run, string why, IReadOnlyList findings, DateTime now, CancellationToken ct) + => storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT OR IGNORE INTO SpotChecks (UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt) + VALUES (@unit, @lead, @agent, @run, @why, @findings, @at) + RETURNING Id + """; + cmd.Parameters.AddWithValue("@unit", unitId); + cmd.Parameters.AddWithValue("@lead", lead); + cmd.Parameters.AddWithValue("@agent", run.AgentId); + cmd.Parameters.AddWithValue("@run", run.RunId); + cmd.Parameters.AddWithValue("@why", why); + cmd.Parameters.AddWithValue("@findings", JsonSerializer.Serialize(findings, Json)); + cmd.Parameters.AddWithValue("@at", now.ToString("O")); + var id = await cmd.ExecuteScalarAsync(ct); + return id is null or DBNull + ? null + : new SpotCheck(Convert.ToInt64(id, CultureInfo.InvariantCulture), unitId, lead, run.AgentId, run.RunId, why, findings, now, null, null, null, null); + }, ct); + + private Task SetTaskAsync(IEnumerable checks, string taskId, CancellationToken ct) + => storage.WriteAsync(async conn => + { + foreach (var check in checks) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "UPDATE SpotChecks SET TaskId = @task WHERE Id = @id"; + cmd.Parameters.AddWithValue("@task", taskId); + cmd.Parameters.AddWithValue("@id", check.Id); + await cmd.ExecuteNonQueryAsync(ct); + } + return 0; + }, ct); + + private async Task> QueryChecksAsync(string where, string value, int limit, CancellationToken ct) + { + await using var conn = await storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + cmd.CommandText = $"SELECT Id, UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt, Verdict, Note, ReviewedAt, TaskId FROM SpotChecks WHERE {where} ORDER BY Id DESC LIMIT @limit"; + cmd.Parameters.AddWithValue("@p", value); + cmd.Parameters.AddWithValue("@limit", limit); + var result = new List(); + await using var r = await cmd.ExecuteReaderAsync(ct); + while (await r.ReadAsync(ct)) + result.Add(new SpotCheck(r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3), r.GetString(4), r.GetString(5), + JsonSerializer.Deserialize>(r.GetString(6), Json) ?? [], + Time(r, 7)!.Value, Text(r, 8), Text(r, 9), Time(r, 10), Text(r, 11))); + return result; + } + + private async Task> QueryMeasuresAsync(string where, string value, int limit, CancellationToken ct, DateTime? since = null) + { + await using var conn = await storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + cmd.CommandText = $"SELECT Id, CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, DecidedBy, DecidedAt, Note, TaskId FROM SpotMeasures WHERE {where} ORDER BY Id DESC LIMIT @limit"; + cmd.Parameters.AddWithValue("@p", value); + cmd.Parameters.AddWithValue("@limit", limit); + if (since is { } s) + cmd.Parameters.AddWithValue("@since", s.ToString("O")); + var result = new List(); + await using var r = await cmd.ExecuteReaderAsync(ct); + while (await r.ReadAsync(ct)) + result.Add(new Measure(r.GetInt64(0), r.IsDBNull(1) ? null : r.GetInt64(1), r.GetString(2), r.GetString(3), r.GetString(4), + Enum.TryParse(r.GetString(5), out var k) ? k : MeasureKind.SharpenTask, + r.GetString(6), Time(r, 7)!.Value, r.GetString(8), Text(r, 9), Time(r, 10), Text(r, 11), Text(r, 12))); + return result; + } + + private static string? Text(SqliteDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i); + + private static DateTime? Time(SqliteDataReader r, int i) + => r.IsDBNull(i) ? null : DateTime.Parse(r.GetString(i), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + + // ─── loop ─── + + public void Start() => _loop ??= RunAsync(_cts.Token); + + private async Task RunAsync(CancellationToken ct) + { + try { await Task.Delay(TimeSpan.FromMinutes(2), ct); } + catch (OperationCanceledException) { return; } + + while (!ct.IsCancellationRequested) + { + try { await SampleAsync(ct); } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Spot checks failed — next attempt in {Minutes} min", Interval.TotalMinutes); + } + + try { await Task.Delay(Interval, ct); } + catch (OperationCanceledException) { return; } + } + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync(); + if (_loop is not null) + { + try { await _loop; } + catch (OperationCanceledException) { } + } + _cts.Dispose(); + } +} diff --git a/src/ClawdDotNet.Core/Storage/SqliteStorage.cs b/src/ClawdDotNet.Core/Storage/SqliteStorage.cs index 6456ed9..eab236c 100644 --- a/src/ClawdDotNet.Core/Storage/SqliteStorage.cs +++ b/src/ClawdDotNet.Core/Storage/SqliteStorage.cs @@ -307,6 +307,41 @@ public sealed class SqliteStorage UNIQUE (Period, Scope, Kind) ); + -- #693: runs picked for a lead to look at, and the measures a lead proposed. + CREATE TABLE IF NOT EXISTS SpotChecks ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + UnitId TEXT NOT NULL, + Lead TEXT NOT NULL, + AgentId TEXT NOT NULL, + RunId TEXT NOT NULL UNIQUE, + Why TEXT NOT NULL, + Findings TEXT NOT NULL, + CreatedAt TEXT NOT NULL, + Verdict TEXT NULL, + Note TEXT NULL, + ReviewedAt TEXT NULL, + TaskId TEXT NULL + ); + + CREATE INDEX IF NOT EXISTS IX_SpotChecks_Lead ON SpotChecks (Lead, Verdict); + CREATE INDEX IF NOT EXISTS IX_SpotChecks_Unit ON SpotChecks (UnitId, CreatedAt); + + CREATE TABLE IF NOT EXISTS SpotMeasures ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + CheckId INTEGER NULL, + UnitId TEXT NOT NULL, + ProposedBy TEXT NOT NULL, + AgentId TEXT NOT NULL, + Kind TEXT NOT NULL, + Reasoning TEXT NOT NULL, + ProposedAt TEXT NOT NULL, + Status TEXT NOT NULL, + DecidedBy TEXT NULL, + DecidedAt TEXT NULL, + Note TEXT NULL, + TaskId TEXT NULL + ); + CREATE TABLE IF NOT EXISTS StagedCalls ( Id INTEGER PRIMARY KEY AUTOINCREMENT, RunId TEXT NOT NULL, diff --git a/src/ClawdDotNet.Core/Tools/AgentToolContext.cs b/src/ClawdDotNet.Core/Tools/AgentToolContext.cs index 7458875..5efda55 100644 --- a/src/ClawdDotNet.Core/Tools/AgentToolContext.cs +++ b/src/ClawdDotNet.Core/Tools/AgentToolContext.cs @@ -55,6 +55,9 @@ public sealed record AgentToolContext( /// #698: The units' doctrines. null outside the engine. public Organisation.IDoctrineStore? Doctrines { get; init; } + /// #693: The spot checks of a lead. null outside the engine. + public Inspection.ISpotChecks? SpotChecks { get; init; } + /// #654: Where code runs — never in the main process. Unavailable under Windows. public Sandbox.ISandboxRunner? Sandbox { get; init; } diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/app.css b/src/ClawdDotNet.Manager.Web/wwwroot/app.css index 2be8ddf..bed2bfd 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/app.css +++ b/src/ClawdDotNet.Manager.Web/wwwroot/app.css @@ -2637,6 +2637,28 @@ button.org-einheit .org-name { gap: 4px; } +/* #693: the lead's spot checks and the measures waiting for the human. */ +.org-stichproben h4 { + margin: 14px 0 6px; + font-size: 13px; +} + +.org-massnahme { + border: 1px solid var(--warnung); + border-left-width: 4px; + padding: 8px 12px; + margin-bottom: 8px; +} + +.org-massnahme.sicherheit { + border-color: var(--gefahr); +} + +.org-befunde { + margin: 0; + padding-left: 16px; +} + tr.org-ausreisser td { background: rgba(181, 130, 42, 0.08); } diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json index 0e8fad6..7cc1dc8 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json +++ b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json @@ -105,6 +105,39 @@ "org.doctrinePlaceholder": "Jeder Lieferantenkontakt wird in der Datenbank erfasst, bevor er weitergegeben wird.", "org.doctrineFrequent": "In den letzten 7 Tagen {count}-mal geändert — eine Doktrin, die oft wechselt, ist Mikromanagement mit Tokenkosten.", "org.doctrineRemoved": "(entfernt)", + "org.spotChecks": "Stichproben der Leitung", + "org.spotChecksHint": "Wie oft die Leitung dieser Einheit in die Läufe der Agenten sieht, die sie führt — gegen den Auftrag und gegen das, was die Engine aufgezeichnet hat, nie durch Nachfragen beim Agenten. Dichter bei neuen Agenten, neuen Modellen und nach einem Fehlschlag. Jede Stichprobe kostet die Leitung einen Blick: die Obergrenze niedrig halten. Maßnahmen, die die Leitung vorschlägt, entscheidest du.", + "org.spotChecksEnabled": "Stichproben an", + "org.spotChecksRate": "Anteil der Läufe (%)", + "org.spotChecksDense": "Anteil bei Neuem oder nach Fehlschlag (%)", + "org.spotChecksNewDays": "Tage, die ein Agent als neu gilt", + "org.spotChecksPerDay": "Höchstens Stichproben je Tag", + "org.spotChecksRecent": "Letzte Stichproben", + "org.spotChecksNone": "Noch kein Lauf geprüft.", + "org.spotCheckRun": "Lauf", + "org.spotCheckWhy": "Anlass", + "org.spotCheckFindings": "Befund im Audit", + "org.spotCheckVerdict": "Urteil der Leitung", + "org.why.sample": "Stichprobe", + "org.why.new_agent": "neuer Agent", + "org.why.new_model": "neues Modell", + "org.why.after_a_failure": "nach Fehlschlag", + "org.verdict.ok": "in Ordnung", + "org.verdict.finding": "Befund", + "org.verdictOpen": "offen bei {lead}", + "org.measuresOpen": "Maßnahmen, die auf dich warten", + "org.measuresDecided": "Entschiedene Maßnahmen", + "org.measureBy": "vorgeschlagen von {lead}", + "org.measureNote": "Notiz (optional)", + "org.measureAccept": "Annehmen", + "org.measureDismiss": "Ablehnen", + "org.measure.sharpen_task": "Auftrag schärfen", + "org.measure.change_model": "Modell wechseln", + "org.measure.adjust_rights": "Rechte anpassen", + "org.measure.suspend": "Agent stilllegen", + "org.measure.security": "Sicherheitsverdacht", + "org.measureStatus.accepted": "angenommen", + "org.measureStatus.dismissed": "abgelehnt", "org.ceilingInherited": "Von oben begrenzt auf: {tools}", "org.settings": "Sammelwerte", "org.settingsHint": "Werden nach einer Vorschau in die Agenten geschrieben. Die Einheit merkt sich den Wert nur als Vorgabe für neue Mitglieder — zur Laufzeit übertrumpft nichts einen Agenten.", diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json index a036a9e..3e69871 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json +++ b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json @@ -105,6 +105,39 @@ "org.doctrinePlaceholder": "Every supplier contact is entered in the database before it is passed on.", "org.doctrineFrequent": "Changed {count} times in the last 7 days — a doctrine that changes often is micro-management with token costs.", "org.doctrineRemoved": "(removed)", + "org.spotChecks": "Spot checks by the lead", + "org.spotChecksHint": "How often the lead of this unit looks at the runs of the agents it leads — held against the task and against what the engine recorded, never by asking the agent. Denser for new agents, new models and after a failure. Every check costs the lead a look: keep the cap low. Measures the lead proposes are decided by you.", + "org.spotChecksEnabled": "Spot checks on", + "org.spotChecksRate": "Share of runs (%)", + "org.spotChecksDense": "Share when new or after a failure (%)", + "org.spotChecksNewDays": "Days an agent counts as new", + "org.spotChecksPerDay": "Checks per day at most", + "org.spotChecksRecent": "Recent checks", + "org.spotChecksNone": "No run checked yet.", + "org.spotCheckRun": "Run", + "org.spotCheckWhy": "Why", + "org.spotCheckFindings": "Found in the audit log", + "org.spotCheckVerdict": "Verdict of the lead", + "org.why.sample": "sample", + "org.why.new_agent": "new agent", + "org.why.new_model": "new model", + "org.why.after_a_failure": "after a failure", + "org.verdict.ok": "in order", + "org.verdict.finding": "finding", + "org.verdictOpen": "open with {lead}", + "org.measuresOpen": "Measures waiting for you", + "org.measuresDecided": "Decided measures", + "org.measureBy": "proposed by {lead}", + "org.measureNote": "Note (optional)", + "org.measureAccept": "Accept", + "org.measureDismiss": "Dismiss", + "org.measure.sharpen_task": "Sharpen the task", + "org.measure.change_model": "Change the model", + "org.measure.adjust_rights": "Adjust the rights", + "org.measure.suspend": "Pause the agent", + "org.measure.security": "Security suspicion", + "org.measureStatus.accepted": "accepted", + "org.measureStatus.dismissed": "dismissed", "org.ceilingInherited": "Limited from above to: {tools}", "org.settings": "Unit-wide values", "org.settingsHint": "Written into the agents after a preview. The unit keeps the value only as a default for new members — nothing overrides an agent at run time.", diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/views/organisation.js b/src/ClawdDotNet.Manager.Web/wwwroot/views/organisation.js index c89c94b..4cc01bc 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/views/organisation.js +++ b/src/ClawdDotNet.Manager.Web/wwwroot/views/organisation.js @@ -368,6 +368,7 @@ function unitPanel(unit) { cards.push(ceilingCard(unit)); cards.push(approvalsCard(unit)); cards.push(doctrineCard(unit)); + cards.push(inspectionCard(unit)); cards.push(settingsCard(unit)); // Members. @@ -503,6 +504,101 @@ function doctrineCard(unit) { ]); } +/** + * #693: The lead's spot checks. How often it looks at the runs of the agents it leads, what + * came of the last checks, and the measures it proposed — which only a human decides. Nothing + * here changes an agent; accepting records the decision, carrying it out stays with you. + */ +function inspectionCard(unit) { + const current = unit.inspection; + const enabled = erstelleElement("input", { type: "checkbox" }); + enabled.checked = !!current; + const percent = (v) => String(Math.round((v ?? 0) * 100)); + const rate = erstelleElement("input", { type: "number", min: "0", max: "100", step: "1", value: percent(current?.rate ?? 0.1) }); + const dense = erstelleElement("input", { type: "number", min: "0", max: "100", step: "1", value: percent(current?.denseRate ?? 0.5) }); + const newDays = erstelleElement("input", { type: "number", min: "0", max: "365", step: "1", value: String(current?.newDays ?? 7) }); + const perDay = erstelleElement("input", { type: "number", min: "0", max: "100", step: "1", value: String(current?.perDay ?? 5) }); + const results = erstelleElement("div", { klasse: "org-stichproben" }); + + (async () => { + let data; + try { data = await api(`/organisation/units/${encodeURIComponent(unit.id)}/spot-checks`); } catch { return; } + + const proposed = data.measures.filter((m) => m.status === "proposed"); + if (proposed.length) { + results.append(erstelleElement("h4", { text: t("org.measuresOpen") })); + for (const m of proposed) { + const note = erstelleElement("input", { type: "text", placeholder: t("org.measureNote") }); + results.append(erstelleElement("div", { klasse: `org-massnahme${m.kind === "security" ? " sicherheit" : ""}` }, + erstelleElement("div", {}, + erstellePlakette(t(`org.measure.${m.kind}`), m.kind === "security" ? "gefahr" : "warnung"), + erstelleElement("strong", { text: ` ${m.agentId}` }), + erstelleElement("span", { klasse: "gedaempft text-klein", text: ` · ${t("org.measureBy", { lead: m.proposedBy })} · ${formatiereDatumZeit(m.proposedAt)}` })), + erstelleElement("p", { klasse: "text-klein", text: m.reasoning }), + erstelleElement("div", { klasse: "gruppe-horizontal" }, note, + erstelleElement("button", { + type: "button", klasse: "primaer klein", text: t("org.measureAccept"), + onclick: () => send(`/organisation/measures/${m.id}/decide`, "POST", { accept: true, note: note.value }) + }), + erstelleElement("button", { + type: "button", klasse: "klein", text: t("org.measureDismiss"), + onclick: () => send(`/organisation/measures/${m.id}/decide`, "POST", { accept: false, note: note.value }) + })))); + } + } + + results.append(erstelleElement("h4", { text: t("org.spotChecksRecent") })); + if (!data.checks.length) { + results.append(erstelleElement("p", { klasse: "gedaempft text-klein", text: t("org.spotChecksNone") })); + } else { + results.append(erstelleElement("table", { klasse: "daten-tabelle" }, + erstelleElement("thead", {}, erstelleElement("tr", {}, + ...[t("org.agent"), t("org.spotCheckRun"), t("org.spotCheckWhy"), t("org.spotCheckFindings"), t("org.spotCheckVerdict")] + .map((h) => erstelleElement("th", { text: h })))), + erstelleElement("tbody", {}, ...data.checks.map((c) => erstelleElement("tr", {}, + erstelleElement("td", { text: c.agentId }), + erstelleElement("td", {}, erstelleElement("a", { href: `#/verbrauch/runs/${encodeURIComponent(c.runId)}`, text: formatiereDatumZeit(c.createdAt) })), + erstelleElement("td", { text: t(`org.why.${c.why.replace(/ /g, "_")}`) }), + erstelleElement("td", {}, c.findings.length + ? erstelleElement("ul", { klasse: "org-befunde" }, ...c.findings.map((f) => erstelleElement("li", { klasse: "text-klein", text: f.text }))) + : erstelleElement("span", { klasse: "gedaempft", text: "—" })), + erstelleElement("td", {}, c.verdict + ? erstellePlakette(t(`org.verdict.${c.verdict}`), c.verdict === "ok" ? "erfolg" : "warnung") + : erstelleElement("span", { klasse: "gedaempft text-klein", text: t("org.verdictOpen", { lead: c.lead }) }), + c.note ? erstelleElement("div", { klasse: "text-klein", text: c.note }) : "")))))); + } + + const decided = data.measures.filter((m) => m.status !== "proposed").slice(0, 10); + if (decided.length) { + results.append(erstelleElement("h4", { text: t("org.measuresDecided") })); + results.append(erstelleElement("ul", { klasse: "org-liste" }, ...decided.map((m) => erstelleElement("li", { klasse: "text-klein", + text: `${t(`org.measure.${m.kind}`)} · ${m.agentId} · ${t(`org.measureStatus.${m.status}`)}${m.note ? " — " + m.note : ""}` })))); + } + })(); + + return erstelleKarte(t("org.spotChecks"), [ + erstelleElement("p", { klasse: "gedaempft text-klein", text: t("org.spotChecksHint") }), + erstelleElement("div", { klasse: "formular-raster" }, + erstelleElement("label", { text: t("org.spotChecksEnabled") }), enabled, + erstelleElement("label", { text: t("org.spotChecksRate") }), rate, + erstelleElement("label", { text: t("org.spotChecksDense") }), dense, + erstelleElement("label", { text: t("org.spotChecksNewDays") }), newDays, + erstelleElement("label", { text: t("org.spotChecksPerDay") }), perDay), + erstelleElement("button", { + type: "button", klasse: "klein", text: t("common.save"), + onclick: () => send(`/organisation/units/${encodeURIComponent(unit.id)}/inspection`, "PUT", { + enabled: enabled.checked, + rate: Number(rate.value) / 100, + denseRate: Number(dense.value) / 100, + newDays: Number(newDays.value), + perDay: Number(perDay.value), + reason: reason() + }) + }), + results + ]); +} + function settingsCard(unit) { const setting = erstelleElement("select", {}, ...SETTINGS.map((s) => erstelleElement("option", { value: s, text: t(`org.setting.${s}`) }))); diff --git a/src/ClawdDotNet.Tools.Taskboard/EscalationTool.cs b/src/ClawdDotNet.Tools.Taskboard/EscalationTool.cs index 3533680..38ead4c 100644 --- a/src/ClawdDotNet.Tools.Taskboard/EscalationTool.cs +++ b/src/ClawdDotNet.Tools.Taskboard/EscalationTool.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Globalization; using ClawdDotNet.Core.Budget; using ClawdDotNet.Core.Cases; +using ClawdDotNet.Core.Inspection; using ClawdDotNet.Core.Organisation; using ClawdDotNet.Core.Staging; using ClawdDotNet.Core.Tasks; @@ -36,6 +37,10 @@ public sealed class EscalationTool : IAgentTool - doctrine_read: the doctrine of your unit — what holds there now. As a lead: doctrine_set (text, reason) sets your unit's doctrine — short, changed seldom; empty text removes it. Every version is kept. As a lead whose unit may approve: list_approvals shows the calls waiting for you; approve_call (call) or reject_call (call, reason). Never your own calls; if unsure, leave it to the human. + As a lead, look at how the agents you lead work — never by asking them, always against what the engine recorded: + spot_checks lists the runs picked for you, with what the audit log shows; inspect_run (run) shows one run in full; + check_done (check, verdict ok|finding, note) closes a check. propose_measure (agent, measure, reasoning, check) puts a measure to a human: + sharpen_task, change_model, adjust_rights, suspend — or security, if you suspect the agent is steered from outside. """; public JsonElement InputSchema { get; } = JsonDocument.Parse(""" @@ -44,7 +49,7 @@ public sealed class EscalationTool : IAgentTool "properties": { "action": { "type": "string", - "enum": ["find_decisions", "escalate", "list_cases", "decide", "handle", "forward", "open_conflict", "state_position", "request_budget", "list_approvals", "approve_call", "reject_call", "doctrine_read", "doctrine_set"] + "enum": ["find_decisions", "escalate", "list_cases", "decide", "handle", "forward", "open_conflict", "state_position", "request_budget", "list_approvals", "approve_call", "reject_call", "doctrine_read", "doctrine_set", "spot_checks", "inspect_run", "check_done", "propose_measure"] }, "amount_usd": { "type": "number", "description": "request_budget: the amount for the task and all its parts, in USD." }, "subject": { "type": "string", "description": "What it is about, in a few words — like a title. escalate, open_conflict, find_decisions." }, @@ -62,7 +67,12 @@ public sealed class EscalationTool : IAgentTool "position": { "type": "string", "description": "open_conflict, state_position: your side, with reasons." }, "call": { "type": "integer", "description": "approve_call, reject_call: the approval number (#…)." }, "reason": { "type": "string", "description": "reject_call: why — the agent must know what to change. doctrine_set: why the doctrine changes." }, - "text": { "type": "string", "description": "doctrine_set: what holds in your unit now. At most 2000 characters; empty removes it." } + "text": { "type": "string", "description": "doctrine_set: what holds in your unit now. At most 2000 characters; empty removes it." }, + "run": { "type": "string", "description": "inspect_run: the run id." }, + "check": { "type": "integer", "description": "check_done, propose_measure: the spot check number." }, + "verdict": { "type": "string", "enum": ["ok", "finding"], "description": "check_done: ok, or finding (then 'note' with the evidence)." }, + "agent": { "type": "string", "description": "propose_measure: the agent it concerns." }, + "measure": { "type": "string", "enum": ["sharpen_task", "change_model", "adjust_rights", "suspend", "security"], "description": "propose_measure: what should follow." } }, "required": ["action"] } @@ -90,6 +100,10 @@ public sealed class EscalationTool : IAgentTool if (Get(input, "action") is "doctrine_read" or "doctrine_set") return await DoctrineAsync(input, context, ct); + // #693: A lead looking at the runs of the agents it leads. + if (Get(input, "action") is "spot_checks" or "inspect_run" or "check_done" or "propose_measure") + return await SpotCheckAsync(input, context, ct); + // #687: A lead deciding the approvals its unit grants it. if (Get(input, "action") is "list_approvals" or "approve_call" or "reject_call") return await ApprovalAsync(input, context, ct); @@ -162,7 +176,7 @@ public sealed class EscalationTool : IAgentTool break; default: - return ToolResult.Fail("'action' must be one of find_decisions, escalate, list_cases, decide, handle, forward, open_conflict, state_position, request_budget, list_approvals, approve_call, reject_call."); + return ToolResult.Fail("'action' must be one of find_decisions, escalate, list_cases, decide, handle, forward, open_conflict, state_position, request_budget, list_approvals, approve_call, reject_call, doctrine_read, doctrine_set, spot_checks, inspect_run, check_done, propose_measure."); } return outcome.Ok ? ToolResult.Ok(outcome.Message) : ToolResult.Fail(outcome.Message); @@ -244,6 +258,105 @@ public sealed class EscalationTool : IAgentTool return decided.Ok ? ToolResult.Ok(decided.Message) : ToolResult.Fail(decided.Message); } + private static async Task SpotCheckAsync(JsonElement input, AgentToolContext context, CancellationToken ct) + { + if (context.SpotChecks is not { } checks) + return ToolResult.Fail("Spot checks are not available in this instance."); + var me = context.AgentId; + long? checkId = input.TryGetProperty("check", out var c) && c.ValueKind == JsonValueKind.Number && c.TryGetInt64(out var n) ? n : null; + + switch (Get(input, "action")) + { + case "spot_checks": + { + var pending = await checks.PendingForAsync(me, ct); + var measures = (await checks.MeasuresByAsync(me, 14, ct)).Take(5).ToList(); + var sb = new StringBuilder(); + if (pending.Count == 0) + sb.AppendLine("No spot check waits for you."); + else + { + sb.AppendLine($"{pending.Count} run(s) to check — the findings come from the audit log, not from the agent:"); + foreach (var check in pending) + { + sb.AppendLine($"- check {check.Id}: {check.AgentId}, run {check.RunId} ({check.Why}, {check.CreatedAt:yyyy-MM-dd HH:mm} UTC)"); + if (check.Findings.Count == 0) + sb.AppendLine(" nothing mechanical — judge whether the result answers the task (inspect_run)."); + foreach (var f in check.Findings) + sb.AppendLine($" {f.Kind}: {f.Text}"); + } + } + if (measures.Count > 0) + { + sb.AppendLine("Your recent proposals:"); + foreach (var m in measures) + sb.AppendLine($"- {m.Id} {SpotChecks.Label(m.Kind)} for {m.AgentId}: {m.Status}" + (string.IsNullOrWhiteSpace(m.Note) ? "" : $" — {m.Note}")); + } + return ToolResult.Ok(sb.ToString().TrimEnd()); + } + + case "inspect_run": + { + var runId = (Get(input, "run") ?? "").Trim(); + if (runId.Length == 0) + return ToolResult.Fail("'run' is required — the run id."); + if (await checks.InspectAsync(runId, ct) is not { } run) + return ToolResult.Fail($"No finished run '{runId}'."); + if (!checks.Leads(me, run.Receipt.AgentId)) + return ToolResult.Fail($"Run '{runId}' belongs to {run.Receipt.AgentId}, whom you do not lead."); + return ToolResult.Ok(Describe(run)); + } + + case "check_done": + { + if (checkId is not { } id) + return ToolResult.Fail("'check' is required — the spot check number."); + var verdict = Get(input, "verdict"); + if (verdict is not ("ok" or "finding")) + return ToolResult.Fail("'verdict' must be ok or finding."); + var closed = await checks.CloseAsync(id, me, verdict == "finding", Get(input, "note") ?? "", ct); + return closed.Ok ? ToolResult.Ok(closed.Message) : ToolResult.Fail(closed.Message); + } + + default: + { + if (SpotChecks.ParseKind(Get(input, "measure")) is not { } kind) + return ToolResult.Fail("'measure' must be one of sharpen_task, change_model, adjust_rights, suspend, security."); + var proposed = await checks.ProposeAsync(me, Get(input, "agent") ?? "", kind, Get(input, "reasoning") ?? "", checkId, ct); + return proposed.Ok ? ToolResult.Ok(proposed.Message) : ToolResult.Fail(proposed.Message); + } + } + } + + /// One run as the engine recorded it. The result text is the agent's — marked as such. + private static string Describe(RunInspection run) + { + var r = run.Receipt; + var sb = new StringBuilder(); + sb.AppendLine($"Run {r.RunId} of {r.AgentId} ({r.Model}), {r.OccurredAt:yyyy-MM-dd HH:mm} UTC: {r.Status}, {r.StepCount} step(s), " + + (r.CostIsKnown ? $"{r.CostUsd.ToString("0.0000", CultureInfo.InvariantCulture)} USD" : "cost unknown")); + if (run.Task is { } task) + { + sb.AppendLine($"Task {task.Id}: {task.Title}"); + if (!string.IsNullOrWhiteSpace(task.Acceptance)) + sb.AppendLine($"Done when: {task.Acceptance}"); + } + sb.AppendLine($"Calls recorded by the engine ({run.Calls.Count}):"); + foreach (var call in run.Calls.Take(40)) + sb.AppendLine($"- {call.At:HH:mm:ss} {call.Tool}{(call.Operation is { } op ? "." + op : "")}: {call.Status}" + + (string.IsNullOrWhiteSpace(call.Summary) ? "" : $" — {Cap(call.Summary, 160)}")); + if (run.Calls.Count > 40) + sb.AppendLine($"- … and {run.Calls.Count - 40} more"); + sb.AppendLine(run.Findings.Count == 0 ? "Mechanical findings: none." : "Mechanical findings:"); + foreach (var f in run.Findings) + sb.AppendLine($"- {f.Kind}: {f.Text}"); + sb.AppendLine("Result as the agent wrote it (its claim, not evidence):"); + sb.Append(string.IsNullOrWhiteSpace(run.Result) ? "(none)" : run.Result); + return sb.ToString(); + } + + private static string Cap(string text, int max) => text.Length <= max ? text : text[..max] + "…"; + private static async Task DoctrineAsync(JsonElement input, AgentToolContext context, CancellationToken ct) { if (context.Doctrines is not { } doctrines || context.Organisation is not { HasUnits: true } org) diff --git a/src/ClawdDotNet.Web/Contracts/OrganisationDtos.cs b/src/ClawdDotNet.Web/Contracts/OrganisationDtos.cs index a074ad7..7e00b51 100644 --- a/src/ClawdDotNet.Web/Contracts/OrganisationDtos.cs +++ b/src/ClawdDotNet.Web/Contracts/OrganisationDtos.cs @@ -20,7 +20,9 @@ public sealed record UnitDto( string Path, int Depth, // #687: what the lead may approve instead of the human; null: nothing. - UnitApprovalsDto? Approvals = null); + UnitApprovalsDto? Approvals = null, + // #693: the lead's spot checks; null: none. + UnitInspectionDto? Inspection = null); /// #698: One version of a unit's doctrine. public sealed record DoctrineDto(int Version, string Text, string SetBy, DateTime SetAt, string? RunId, string Reason); @@ -31,6 +33,27 @@ public sealed record UnitApprovalsDto(IReadOnlyList Operations, int PerD public sealed record ApprovalsInputDto(IReadOnlyList? Operations, int? PerDay, double? Hours, string Reason = ""); +/// #693: How often the lead looks at the runs of the agents it leads. +public sealed record UnitInspectionDto(double Rate, double DenseRate, int NewDays, int PerDay); + +public sealed record InspectionInputDto(bool Enabled, double? Rate, double? DenseRate, int? NewDays, int? PerDay, string Reason = ""); + +public sealed record FindingDto(string Kind, string Text); + +/// #693: A run picked for the lead, with what the audit log showed and the lead's verdict. +public sealed record SpotCheckDto( + long Id, string Lead, string AgentId, string RunId, string Why, IReadOnlyList Findings, + DateTime CreatedAt, string? Verdict, string? Note, DateTime? ReviewedAt); + +/// #693: A measure a lead proposed; a human decides it. +public sealed record MeasureDto( + long Id, long? CheckId, string AgentId, string ProposedBy, string Kind, string Reasoning, + DateTime ProposedAt, string Status, string? DecidedBy, DateTime? DecidedAt, string? Note); + +public sealed record SpotCheckOverviewDto(IReadOnlyList Checks, IReadOnlyList Measures); + +public sealed record MeasureDecisionDto(bool Accept, string Note = ""); + public sealed record UnitDefaultsDto(string? Model, decimal? DailyCostUsd, long? DailyTokens, string? WhenExhausted); /// An agent as the organisation page shows it: where it is, whom it answers to. diff --git a/src/ClawdDotNet.Web/Endpoints/OrganisationEndpoints.cs b/src/ClawdDotNet.Web/Endpoints/OrganisationEndpoints.cs index 2a311e0..298c20f 100644 --- a/src/ClawdDotNet.Web/Endpoints/OrganisationEndpoints.cs +++ b/src/ClawdDotNet.Web/Endpoints/OrganisationEndpoints.cs @@ -28,7 +28,8 @@ internal static class OrganisationEndpoints chart.ToolCeiling(u.Id)?.Order(StringComparer.OrdinalIgnoreCase).ToList(), new UnitDefaultsDto(u.Defaults.Model, u.Defaults.DailyCostUsd, u.Defaults.DailyTokens, u.Defaults.WhenExhausted), chart.PathOf(u.Id), chart.Depth(u.Id), - u.Approvals is { } ap ? new UnitApprovalsDto(ap.Operations, ap.PerDay, ap.Hours) : null)) + u.Approvals is { } ap ? new UnitApprovalsDto(ap.Operations, ap.PerDay, ap.Hours) : null, + u.Inspection is { } ins ? new UnitInspectionDto(ins.Rate, ins.DenseRate, ins.NewDays, ins.PerDay) : null)) .ToList(); var agents = host.Instance.Agents @@ -103,6 +104,32 @@ internal static class OrganisationEndpoints return Results.Ok(new OutcomeDto(true, set.IsEmpty ? $"Doctrine of '{unit.Name}' removed." : $"Doctrine of '{unit.Name}' set (version {set.Version}).")); }); + // #693: The lead's spot checks — how often, what came of them, and the measures it proposed. + org.MapPut("/units/{unitId}/inspection", (AppHost host, string unitId, InspectionInputDto input) => + host.Organisation.SetInspection(unitId, input.Enabled, input.Rate, input.DenseRate, input.NewDays, input.PerDay, input.Reason).ToResponse()); + + org.MapGet("/units/{unitId}/spot-checks", async (AppHost host, string unitId, CancellationToken ct) => + { + if (host.SpotChecks is not { } checks) + return Results.Ok(new SpotCheckOverviewDto([], [])); + var list = await checks.ForUnitAsync(unitId, 30, ct); + var measures = await checks.MeasuresForUnitAsync(unitId, 30, ct); + return Results.Ok(new SpotCheckOverviewDto( + list.Select(c => new SpotCheckDto(c.Id, c.Lead, c.AgentId, c.RunId, c.Why, + c.Findings.Select(f => new FindingDto(f.Kind.ToString(), f.Text)).ToList(), + c.CreatedAt, c.Verdict, c.Note, c.ReviewedAt)).ToList(), + measures.Select(m => new MeasureDto(m.Id, m.CheckId, m.AgentId, m.ProposedBy, + Core.Inspection.SpotChecks.Label(m.Kind), m.Reasoning, m.ProposedAt, m.Status, m.DecidedBy, m.DecidedAt, m.Note)).ToList())); + }); + + org.MapPost("/measures/{measureId:long}/decide", async (AppHost host, long measureId, MeasureDecisionDto input, CancellationToken ct) => + { + if (host.SpotChecks is not { } checks) + return Results.BadRequest(new OutcomeDto(false, "Spot checks are not available in this instance.")); + var decided = await checks.DecideMeasureAsync(measureId, input.Accept, input.Note, "human", ct); + return decided.Ok ? Results.Ok(new OutcomeDto(true, decided.Message)) : Results.BadRequest(new OutcomeDto(false, decided.Message)); + }); + // #687: What the lead may approve instead of the human. org.MapPut("/units/{unitId}/approvals", (AppHost host, string unitId, ApprovalsInputDto input) => host.Organisation.SetApprovals(unitId, input.Operations, input.PerDay, input.Hours, input.Reason).ToResponse()); diff --git a/tests/ClawdDotNet.App.Tests/Services/OrgAdministrationTests.cs b/tests/ClawdDotNet.App.Tests/Services/OrgAdministrationTests.cs index d6fa6e4..187d5b7 100644 --- a/tests/ClawdDotNet.App.Tests/Services/OrgAdministrationTests.cs +++ b/tests/ClawdDotNet.App.Tests/Services/OrgAdministrationTests.cs @@ -288,6 +288,26 @@ public sealed class OrgAdministrationTests : IDisposable Unit("amazon").Approvals!.Operations.ShouldBe(["Database.insert"]); } + // ═══════════════════ spot checks (#693) ═══════════════════ + + [Fact] + public void A_human_sets_how_often_the_lead_checks_within_bounds() + { + BuildMarketing(); + + Org.SetInspection("marketing", true, rate: 1.5).Ok.ShouldBeFalse("a share lies between 0 and 1"); + Org.SetInspection("marketing", true, perDay: -1).Ok.ShouldBeFalse(); + + var set = Org.SetInspection("marketing", true, rate: 0.2, denseRate: 0.1, perDay: 3, reason: "new team"); + + set.Ok.ShouldBeTrue(set.Message); + Unit("marketing").Inspection!.Rate.ShouldBe(0.2); + Unit("marketing").Inspection!.DenseRate.ShouldBe(0.2, "never less dense where it matters than for ordinary runs"); + Unit("marketing").Inspection!.PerDay.ShouldBe(3); + Org.SetInspection("marketing", false).Ok.ShouldBeTrue(); + Unit("marketing").Inspection.ShouldBeNull(); + } + // ═══════════════════ the top (#690) ═══════════════════ private void MakeDecider(string id) diff --git a/tests/ClawdDotNet.Core.Tests/Organisation/SpotCheckTests.cs b/tests/ClawdDotNet.Core.Tests/Organisation/SpotCheckTests.cs new file mode 100644 index 0000000..632eeca --- /dev/null +++ b/tests/ClawdDotNet.Core.Tests/Organisation/SpotCheckTests.cs @@ -0,0 +1,247 @@ +using ClawdDotNet.Core.Audit; +using ClawdDotNet.Core.Config; +using ClawdDotNet.Core.Inspection; +using ClawdDotNet.Core.Organisation; +using ClawdDotNet.Core.Storage; +using ClawdDotNet.Core.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace ClawdDotNet.Core.Tests.Organisation; + +/// +/// #693: A lead looks at the runs of the agents it leads — against what the engine stamped, +/// never by asking the agent; denser where it matters, capped; and what follows is proposed, +/// not done. +/// +public sealed class SpotCheckTests : IDisposable +{ + private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N")); + private readonly SqliteStorage _storage; + private readonly SqliteAuditRepository _audit; + private readonly SqliteTaskRepository _tasks; + private readonly TaskboardService _board; + + private static readonly IReadOnlyDictionary> Vocabulary = new Dictionary> + { + ["AgentEditor"] = new HashSet { "read_soul", "update_soul", "update_identity" }, + ["AgentComm"] = new HashSet { "send_message", "list_agents" }, + ["Memory"] = new HashSet { "remember", "recall" }, + ["FileRW"] = new HashSet { "read", "write", "append" }, + ["Taskboard"] = new HashSet { "complete", "comment" } + }; + + public SpotCheckTests() + { + _storage = new SqliteStorage(Path.Combine(_directory, "state.db")); + _audit = new SqliteAuditRepository(_storage); + _tasks = new SqliteTaskRepository(_storage); + _board = new TaskboardService(_tasks, Path.Combine(_directory, "tasks")); + } + + public void Dispose() + { + SqliteStorage.ReleasePooledConnections(_directory); + try { Directory.Delete(_directory, recursive: true); } + catch { /* cleanup is secondary */ } + } + + /// purchasing (buy-lead; buyer, clerk) with spot checks; sales (sales-lead; seller) without. + private static OrgChart Org(UnitInspection? inspection = null) => new( + [ + new UnitConfig { Id = "purchasing", Name = "Purchasing", LeaderAgentId = "buy-lead", Inspection = inspection ?? new UnitInspection { Rate = 1, DenseRate = 1, PerDay = 2 } }, + new UnitConfig { Id = "sales", Name = "Sales", LeaderAgentId = "sales-lead" } + ], + [ + new AgentConfig { AgentId = "buy-lead", UnitId = "purchasing" }, + new AgentConfig { AgentId = "buyer", UnitId = "purchasing" }, + new AgentConfig { AgentId = "clerk", UnitId = "purchasing" }, + new AgentConfig { AgentId = "sales-lead", UnitId = "sales" }, + new AgentConfig { AgentId = "seller", UnitId = "sales" } + ]); + + private static RunReceipt Receipt(string runId, string agent = "buyer", string status = "Completed", string model = "m", DateTime? at = null, string? task = null) => new() + { + RunId = runId, AgentId = agent, Model = model, Status = status, StepCount = 3, + OccurredAt = at ?? DateTime.UtcNow, TaskId = task, ResultRef = "Done." + }; + + private static AuditEntry Call(string tool, string? action, AuditStatus status = AuditStatus.Ok, string callId = "", string summary = "") => new() + { + RunId = "r", AgentId = "buyer", Tool = tool, Status = status, CallId = callId, Summary = summary, + Arguments = action is null ? "{}" : $"{{\"action\":\"{action}\"}}", OccurredAt = DateTime.UtcNow + }; + + private SpotChecks Checks(OrgChart? org = null) + { + var chart = org ?? Org(); + return new SpotChecks(_storage, _audit, new RunInspector(_audit, _tasks, null, () => Vocabulary), () => chart, _board, NullLogger.Instance); + } + + // ─── the mechanical check ─── + + [Fact] + public void Failed_runs_errors_refusals_and_unfinished_calls_are_found() + { + var findings = RunInspector.Analyse( + Receipt("r", status: "LoopLimitExceeded"), + [ + Call("FileRW", "write", AuditStatus.Error, "c1", "disk full"), + Call("FileRW", "write", AuditStatus.Denied, "c2"), + Call("FileRW", "read", AuditStatus.Started, "c3") + ], + "", Vocabulary, null); + + findings.Select(f => f.Kind).ShouldBe([FindingKind.RunFailed, FindingKind.ToolErrors, FindingKind.Denied, FindingKind.Unfinished], ignoreOrder: true); + findings.Single(f => f.Kind == FindingKind.ToolErrors).Text.ShouldContain("FileRW.write (disk full)"); + } + + [Fact] + public void Operations_the_agent_names_but_never_called_are_found() + { + // The 2026-09-14 pattern: a findings file full of "tool answers" from a run that only appended to a file. + var said = "AgentEditor read_soul answered 'User confirmation needed'. send_message was intercepted. I will remember this in Memory."; + + var findings = RunInspector.Analyse(Receipt("r"), [Call("FileRW", "append")], said, Vocabulary, null); + + var claim = findings.Single(f => f.Kind == FindingKind.UnsupportedClaim).Text; + claim.ShouldContain("AgentEditor"); + claim.ShouldContain("read_soul"); + claim.ShouldContain("send_message"); + claim.ShouldNotContain("remember", customMessage: "an ordinary word is no claim"); + claim.ShouldNotContain("Memory"); + } + + [Fact] + public void A_refused_call_is_a_call() + { + var findings = RunInspector.Analyse(Receipt("r"), [Call("AgentEditor", "update_soul", AuditStatus.Denied)], + "AgentEditor update_soul was refused, as expected.", Vocabulary, null); + + findings.ShouldNotContain(f => f.Kind == FindingKind.UnsupportedClaim); + } + + [Fact] + public void A_task_answered_with_bookkeeping_alone_is_marked() + { + var task = new TaskItem { Id = "t1", Title = "Order paper" }; + + RunInspector.Analyse(Receipt("r"), [Call("Taskboard", "complete")], "", Vocabulary, task) + .ShouldContain(f => f.Kind == FindingKind.NoWork); + RunInspector.Analyse(Receipt("r"), [Call("FileRW", "write"), Call("Taskboard", "complete")], "", Vocabulary, task) + .ShouldNotContain(f => f.Kind == FindingKind.NoWork); + } + + [Fact] + public void The_operation_is_read_even_from_cut_arguments() + { + RunInspector.OperationOf("{\"action\":\"write\",\"content\":\"abc").ShouldBe("write"); + RunInspector.OperationOf("not json").ShouldBeNull(); + } + + // ─── which runs ─── + + [Fact] + public void New_agents_new_models_and_the_run_after_a_failure_are_checked_densely() + { + var onlyDense = new UnitInspection { Rate = 0, DenseRate = 1, NewDays = 7 }; + var now = DateTime.UtcNow; + var settled = Receipt("old", at: now.AddDays(-20)); + + SpotChecks.Pick(Receipt("a", at: now), [], onlyDense).ShouldBe("new agent"); + SpotChecks.Pick(Receipt("b", at: now), [settled], onlyDense).ShouldBeNull("a settled agent with the same model and no failure is an ordinary run"); + SpotChecks.Pick(Receipt("c", model: "other", at: now), [settled], onlyDense).ShouldBe("new model"); + SpotChecks.Pick(Receipt("d", at: now), [settled, Receipt("bad", status: "Failed", at: now.AddHours(-1))], onlyDense).ShouldBe("after a failure"); + SpotChecks.Pick(Receipt("e", at: now), [settled], new UnitInspection { Rate = 1 }).ShouldBe("sample"); + } + + [Fact] + public void The_same_run_gets_the_same_answer_every_time() + { + var half = new UnitInspection { Rate = 0.5, DenseRate = 0.5 }; + var runs = Enumerable.Range(0, 200).Select(i => Receipt($"run-{i}")).ToList(); + + var first = runs.Select(r => SpotChecks.Pick(r, [], half)).ToList(); + runs.Select(r => SpotChecks.Pick(r, [], half)).ShouldBe(first); + first.Count(p => p is not null).ShouldBeInRange(70, 130, "spread like chance"); + } + + // ─── the lead's task ─── + + [Fact] + public async Task Runs_are_picked_for_the_lead_capped_per_day_and_given_as_one_task() + { + var now = DateTime.UtcNow; + foreach (var (run, agent, minutes) in new[] { ("b1", "buyer", 30), ("b2", "buyer", 20), ("c1", "clerk", 10), ("s1", "seller", 5), ("l1", "buy-lead", 3) }) + await _audit.RecordReceiptAsync(Receipt(run, agent, at: now.AddMinutes(-minutes)), default); + var checks = Checks(); + + var picked = await checks.SampleAsync(default); + + picked.Select(c => c.RunId).ShouldBe(["b1", "b2"], "two a day, oldest first"); + picked.ShouldAllBe(c => c.Lead == "buy-lead"); + (await checks.SampleAsync(default)).ShouldBeEmpty("a run is looked at once"); + + var tasks = await _board.ListAsync(new TaskQuery { Assignee = "@buy-lead" }, default); + tasks.Count.ShouldBe(1); + tasks[0].Title.ShouldContain("2 run(s)"); + (await _board.ListAsync(new TaskQuery { Assignee = "@sales-lead" }, default)).ShouldBeEmpty("sales has no spot checks"); + } + + [Fact] + public async Task The_lead_closes_its_checks_and_the_last_one_closes_the_task() + { + await _audit.RecordReceiptAsync(Receipt("b1", at: DateTime.UtcNow.AddMinutes(-5)), default); + await _audit.RecordReceiptAsync(Receipt("c1", "clerk", at: DateTime.UtcNow.AddMinutes(-4)), default); + var checks = Checks(); + var picked = await checks.SampleAsync(default); + + (await checks.CloseAsync(picked[0].Id, "sales-lead", false, "", default)).Ok.ShouldBeFalse("not its check"); + (await checks.CloseAsync(picked[0].Id, "buy-lead", true, "", default)).Message.ShouldContain("'note'"); + (await checks.CloseAsync(picked[0].Id, "buy-lead", true, "Result names a supplier the database does not have.", default)).Ok.ShouldBeTrue(); + + var task = (await _board.ListAsync(new TaskQuery { Assignee = "@buy-lead" }, default)).Single(); + task.Status.ShouldNotBe(TaskItemStatus.Done); + (await checks.CloseAsync(picked[1].Id, "buy-lead", false, "", default)).Ok.ShouldBeTrue(); + (await _board.GetAsync(task.Id, default))!.Status.ShouldBe(TaskItemStatus.Done); + (await checks.PendingForAsync("buy-lead", default)).ShouldBeEmpty(); + } + + // ─── what follows ─── + + [Fact] + public async Task A_lead_proposes_measures_for_its_own_agents_and_a_human_decides() + { + var checks = Checks(); + + (await checks.ProposeAsync("buy-lead", "seller", MeasureKind.ChangeModel, "x", null, default)).Ok.ShouldBeFalse("not its agent"); + (await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.ChangeModel, " ", null, default)).Message.ShouldContain("'reasoning'"); + (await checks.ProposeAsync("buyer", "clerk", MeasureKind.Suspend, "x", null, default)).Ok.ShouldBeFalse("a member leads nobody"); + + var proposed = await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.SharpenTask, "Three runs misread 'urgent'.", null, default); + proposed.Ok.ShouldBeTrue(proposed.Message); + (await checks.ProposeAsync("buy-lead", "buyer", MeasureKind.SharpenTask, "again", null, default)).Ok.ShouldBeFalse("one open at a time"); + + var measure = (await checks.MeasuresForUnitAsync("purchasing", 10, default)).Single(); + measure.Status.ShouldBe("proposed"); + var task = await _board.GetAsync(measure.TaskId!, default); + task!.Assignee.ShouldBe(TaskAssignee.Human); + + (await checks.DecideMeasureAsync(measure.Id, true, "Rewritten.", "human", default)).Ok.ShouldBeTrue(); + (await checks.DecideMeasureAsync(measure.Id, false, "", "human", default)).Ok.ShouldBeFalse("decided already"); + (await _board.GetAsync(task.Id, default))!.Status.ShouldBe(TaskItemStatus.Done); + } + + [Fact] + public async Task A_security_suspicion_is_not_a_personnel_matter() + { + var checks = Checks(); + + (await checks.ProposeAsync("buy-lead", "clerk", MeasureKind.Security, "The clerk's run wrote a supplier's text into the doctrine request.", null, default)).Ok.ShouldBeTrue(); + + var task = (await _board.ListAsync(new TaskQuery { Assignee = TaskAssignee.Human }, default)).Single(); + task.Title.ShouldStartWith("Security suspicion"); + task.Priority.ShouldBe(5); + task.Body.ShouldContain("Not a personnel matter"); + } +} diff --git a/tests/ClawdDotNet.Tools.Tests/Taskboard/SpotCheckToolTests.cs b/tests/ClawdDotNet.Tools.Tests/Taskboard/SpotCheckToolTests.cs new file mode 100644 index 0000000..e56f22b --- /dev/null +++ b/tests/ClawdDotNet.Tools.Tests/Taskboard/SpotCheckToolTests.cs @@ -0,0 +1,114 @@ +using System.Text.Json; +using ClawdDotNet.Core.Audit; +using ClawdDotNet.Core.Config; +using ClawdDotNet.Core.Inspection; +using ClawdDotNet.Core.Organisation; +using ClawdDotNet.Core.Storage; +using ClawdDotNet.Core.Tasks; +using ClawdDotNet.Core.Tools; +using ClawdDotNet.Tools.Taskboard; +using ClawdDotNet.Tools.Tests.FileRW; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace ClawdDotNet.Tools.Tests.Taskboard; + +/// #693: The lead's side of the spot checks through the Escalation tool. +public sealed class SpotCheckToolTests : IDisposable +{ + private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N")); + private readonly SqliteStorage _storage; + private readonly SqliteAuditRepository _audit; + private readonly SpotChecks _checks; + private readonly EscalationTool _tool = new(); + + private readonly OrgChart _org = new( + [ + new UnitConfig { Id = "purchasing", Name = "Purchasing", LeaderAgentId = "buy-lead", Inspection = new UnitInspection { Rate = 1, DenseRate = 1, PerDay = 5 } }, + new UnitConfig { Id = "sales", Name = "Sales", LeaderAgentId = "sales-lead" } + ], + [ + new AgentConfig { AgentId = "buy-lead", UnitId = "purchasing" }, + new AgentConfig { AgentId = "buyer", UnitId = "purchasing" }, + new AgentConfig { AgentId = "sales-lead", UnitId = "sales" }, + new AgentConfig { AgentId = "seller", UnitId = "sales" } + ]); + + public SpotCheckToolTests() + { + _storage = new SqliteStorage(Path.Combine(_directory, "state.db")); + _audit = new SqliteAuditRepository(_storage); + var tasks = new SqliteTaskRepository(_storage); + _checks = new SpotChecks(_storage, _audit, + new RunInspector(_audit, tasks, null, () => RunInspector.VocabularyOf([_tool])), + () => _org, new TaskboardService(tasks, Path.Combine(_directory, "tasks")), NullLogger.Instance); + } + + public void Dispose() + { + SqliteStorage.ReleasePooledConnections(_directory); + try { Directory.Delete(_directory, recursive: true); } + catch { /* cleanup is secondary */ } + } + + private Task Run(string agent, object input) + => _tool.ExecuteAsync(JsonSerializer.SerializeToElement(input), + new AgentToolContext(agent, "inst", new Dictionary(), new NullStateStore(), NullLogger.Instance, CancellationToken.None) + { + Organisation = _org, + SpotChecks = _checks + }, CancellationToken.None); + + private async Task RecordAsync(string runId, string agent, string said) + { + await _audit.AppendAsync(new AuditEntry + { + RunId = runId, AgentId = agent, Tool = "FileRW", Arguments = "{\"action\":\"append\"}", + Status = AuditStatus.Ok, OccurredAt = DateTime.UtcNow.AddMinutes(-2) + }, default); + await _audit.RecordReceiptAsync(new RunReceipt + { + RunId = runId, AgentId = agent, Model = "m", Status = "Completed", StepCount = 2, + ResultRef = said, OccurredAt = DateTime.UtcNow.AddMinutes(-1) + }, default); + } + + [Fact] + public async Task The_lead_sees_its_checks_with_the_audit_findings_and_closes_them() + { + await RecordAsync("run-b", "buyer", "Escalation doctrine_set was refused."); + var check = (await _checks.SampleAsync(default)).Single(); + + var list = await Run("buy-lead", new { action = "spot_checks" }); + list.Content.ShouldContain($"check {check.Id}"); + list.Content.ShouldContain("doctrine_set", customMessage: "the claim without a call is shown from the audit"); + + var detail = await Run("buy-lead", new { action = "inspect_run", run = "run-b" }); + detail.Content.ShouldContain("FileRW.append: Ok"); + detail.Content.ShouldContain("its claim, not evidence"); + + (await Run("buy-lead", new { action = "check_done", check = check.Id, verdict = "finding", note = "Reports a refusal that never happened." })) + .Success.ShouldBeTrue(); + (await Run("buy-lead", new { action = "spot_checks" })).Content.ShouldContain("No spot check waits for you"); + } + + [Fact] + public async Task A_lead_sees_no_runs_of_agents_it_does_not_lead() + { + await RecordAsync("run-s", "seller", "Done."); + + (await Run("buy-lead", new { action = "inspect_run", run = "run-s" })).ErrorMessage!.ShouldContain("do not lead"); + (await Run("buyer", new { action = "inspect_run", run = "run-s" })).Success.ShouldBeFalse(); + } + + [Fact] + public async Task A_measure_needs_a_known_kind_and_goes_to_a_human() + { + (await Run("buy-lead", new { action = "propose_measure", agent = "buyer", measure = "fire", reasoning = "x" })) + .ErrorMessage!.ShouldContain("'measure'"); + + var proposed = await Run("buy-lead", new { action = "propose_measure", agent = "buyer", measure = "change_model", reasoning = "Fails every table task." }); + proposed.Success.ShouldBeTrue(proposed.ErrorMessage); + proposed.Content.ShouldContain("with a human"); + } +}