Richard/ClawdDotNet
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Kontrollpflicht der Leitung: Stichproben gegen das Audit (#693)

Browse Source 
A lead looks at the runs of the agents it leads - held against the task and
against what the engine stamped, never by asking the agent.

- RunInspector (Core/Inspection): the mechanical check and first profile of
  the inspector foundation (#204). Flags failed runs, tool errors, refusals,
  unfinished calls, tasks answered with bookkeeping alone, and operations the
  agent names but never called in the run (the 2026-09-14 pattern).
- SpotChecks: picks runs per unit (rate, denser for new agents, new models and
  after a failure, daily cap), deterministic by run id; one task per lead.
- Escalation tool: spot_checks, inspect_run, check_done, propose_measure.
  Measures go to a human; nothing is carried out automatically. A security
  suspicion is its own case, not a personnel matter.
- Organisation view: settings per unit, recent checks, measures to decide.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
2026-10-01 11:40:20 +02:00
co-authored by Claude Opus 5.5
parent ab1bd67464
commit 68e723a9c0
19 changed files with 1621 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ClawdDotNet.Tools.Taskboard/EscalationTool.cs
+116 -3
View File
@@ -3,6 +3,7 @@ using System.Text.Json;
using System.Globalization;
using ClawdDotNet.Core.Budget;
using ClawdDotNet.Core.Cases;
using ClawdDotNet.Core.Inspection;
using ClawdDotNet.Core.Organisation;
using ClawdDotNet.Core.Staging;
using ClawdDotNet.Core.Tasks;
@@ -36,6 +37,10 @@ public sealed class EscalationTool : IAgentTool
- doctrine_read: the doctrine of your unit — what holds there now.
As a lead: doctrine_set (text, reason) sets your unit's doctrine — short, changed seldom; empty text removes it. Every version is kept.
As a lead whose unit may approve: list_approvals shows the calls waiting for you; approve_call (call) or reject_call (call, reason). Never your own calls; if unsure, leave it to the human.
As a lead, look at how the agents you lead work — never by asking them, always against what the engine recorded:
spot_checks lists the runs picked for you, with what the audit log shows; inspect_run (run) shows one run in full;
check_done (check, verdict ok|finding, note) closes a check. propose_measure (agent, measure, reasoning, check) puts a measure to a human:
sharpen_task, change_model, adjust_rights, suspend — or security, if you suspect the agent is steered from outside.
""";
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
@@ -44,7 +49,7 @@ public sealed class EscalationTool : IAgentTool
"properties": {
"action": {
"type": "string",
"enum": ["find_decisions", "escalate", "list_cases", "decide", "handle", "forward", "open_conflict", "state_position", "request_budget", "list_approvals", "approve_call", "reject_call", "doctrine_read", "doctrine_set"]
"enum": ["find_decisions", "escalate", "list_cases", "decide", "handle", "forward", "open_conflict", "state_position", "request_budget", "list_approvals", "approve_call", "reject_call", "doctrine_read", "doctrine_set", "spot_checks", "inspect_run", "check_done", "propose_measure"]
},
"amount_usd": { "type": "number", "description": "request_budget: the amount for the task and all its parts, in USD." },
"subject": { "type": "string", "description": "What it is about, in a few words — like a title. escalate, open_conflict, find_decisions." },
@@ -62,7 +67,12 @@ public sealed class EscalationTool : IAgentTool
"position": { "type": "string", "description": "open_conflict, state_position: your side, with reasons." },
"call": { "type": "integer", "description": "approve_call, reject_call: the approval number (#…)." },
"reason": { "type": "string", "description": "reject_call: why — the agent must know what to change. doctrine_set: why the doctrine changes." },
"text": { "type": "string", "description": "doctrine_set: what holds in your unit now. At most 2000 characters; empty removes it." }
"text": { "type": "string", "description": "doctrine_set: what holds in your unit now. At most 2000 characters; empty removes it." },
"run": { "type": "string", "description": "inspect_run: the run id." },
"check": { "type": "integer", "description": "check_done, propose_measure: the spot check number." },
"verdict": { "type": "string", "enum": ["ok", "finding"], "description": "check_done: ok, or finding (then 'note' with the evidence)." },
"agent": { "type": "string", "description": "propose_measure: the agent it concerns." },
"measure": { "type": "string", "enum": ["sharpen_task", "change_model", "adjust_rights", "suspend", "security"], "description": "propose_measure: what should follow." }
},
"required": ["action"]
}
@@ -90,6 +100,10 @@ public sealed class EscalationTool : IAgentTool
if (Get(input, "action") is "doctrine_read" or "doctrine_set")
return await DoctrineAsync(input, context, ct);
// #693: A lead looking at the runs of the agents it leads.
if (Get(input, "action") is "spot_checks" or "inspect_run" or "check_done" or "propose_measure")
return await SpotCheckAsync(input, context, ct);
// #687: A lead deciding the approvals its unit grants it.
if (Get(input, "action") is "list_approvals" or "approve_call" or "reject_call")
return await ApprovalAsync(input, context, ct);
@@ -162,7 +176,7 @@ public sealed class EscalationTool : IAgentTool
break;
default:
return ToolResult.Fail("'action' must be one of find_decisions, escalate, list_cases, decide, handle, forward, open_conflict, state_position, request_budget, list_approvals, approve_call, reject_call.");
return ToolResult.Fail("'action' must be one of find_decisions, escalate, list_cases, decide, handle, forward, open_conflict, state_position, request_budget, list_approvals, approve_call, reject_call, doctrine_read, doctrine_set, spot_checks, inspect_run, check_done, propose_measure.");
}
return outcome.Ok ? ToolResult.Ok(outcome.Message) : ToolResult.Fail(outcome.Message);
@@ -244,6 +258,105 @@ public sealed class EscalationTool : IAgentTool
return decided.Ok ? ToolResult.Ok(decided.Message) : ToolResult.Fail(decided.Message);
}
private static async Task<ToolResult> SpotCheckAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
{
if (context.SpotChecks is not { } checks)
return ToolResult.Fail("Spot checks are not available in this instance.");
var me = context.AgentId;
long? checkId = input.TryGetProperty("check", out var c) && c.ValueKind == JsonValueKind.Number && c.TryGetInt64(out var n) ? n : null;
switch (Get(input, "action"))
{
case "spot_checks":
{
var pending = await checks.PendingForAsync(me, ct);
var measures = (await checks.MeasuresByAsync(me, 14, ct)).Take(5).ToList();
var sb = new StringBuilder();
if (pending.Count == 0)
sb.AppendLine("No spot check waits for you.");
else
{
sb.AppendLine($"{pending.Count} run(s) to check — the findings come from the audit log, not from the agent:");
foreach (var check in pending)
{
sb.AppendLine($"- check {check.Id}: {check.AgentId}, run {check.RunId} ({check.Why}, {check.CreatedAt:yyyy-MM-dd HH:mm} UTC)");
if (check.Findings.Count == 0)
sb.AppendLine(" nothing mechanical — judge whether the result answers the task (inspect_run).");
foreach (var f in check.Findings)
sb.AppendLine($" {f.Kind}: {f.Text}");
}
}
if (measures.Count > 0)
{
sb.AppendLine("Your recent proposals:");
foreach (var m in measures)
sb.AppendLine($"- {m.Id} {SpotChecks.Label(m.Kind)} for {m.AgentId}: {m.Status}" + (string.IsNullOrWhiteSpace(m.Note) ? "" : $" — {m.Note}"));
}
return ToolResult.Ok(sb.ToString().TrimEnd());
}
case "inspect_run":
{
var runId = (Get(input, "run") ?? "").Trim();
if (runId.Length == 0)
return ToolResult.Fail("'run' is required — the run id.");
if (await checks.InspectAsync(runId, ct) is not { } run)
return ToolResult.Fail($"No finished run '{runId}'.");
if (!checks.Leads(me, run.Receipt.AgentId))
return ToolResult.Fail($"Run '{runId}' belongs to {run.Receipt.AgentId}, whom you do not lead.");
return ToolResult.Ok(Describe(run));
}
case "check_done":
{
if (checkId is not { } id)
return ToolResult.Fail("'check' is required — the spot check number.");
var verdict = Get(input, "verdict");
if (verdict is not ("ok" or "finding"))
return ToolResult.Fail("'verdict' must be ok or finding.");
var closed = await checks.CloseAsync(id, me, verdict == "finding", Get(input, "note") ?? "", ct);
return closed.Ok ? ToolResult.Ok(closed.Message) : ToolResult.Fail(closed.Message);
}
default:
{
if (SpotChecks.ParseKind(Get(input, "measure")) is not { } kind)
return ToolResult.Fail("'measure' must be one of sharpen_task, change_model, adjust_rights, suspend, security.");
var proposed = await checks.ProposeAsync(me, Get(input, "agent") ?? "", kind, Get(input, "reasoning") ?? "", checkId, ct);
return proposed.Ok ? ToolResult.Ok(proposed.Message) : ToolResult.Fail(proposed.Message);
}
}
}
/// <summary>One run as the engine recorded it. The result text is the agent's — marked as such.</summary>
private static string Describe(RunInspection run)
{
var r = run.Receipt;
var sb = new StringBuilder();
sb.AppendLine($"Run {r.RunId} of {r.AgentId} ({r.Model}), {r.OccurredAt:yyyy-MM-dd HH:mm} UTC: {r.Status}, {r.StepCount} step(s), " +
(r.CostIsKnown ? $"{r.CostUsd.ToString("0.0000", CultureInfo.InvariantCulture)} USD" : "cost unknown"));
if (run.Task is { } task)
{
sb.AppendLine($"Task {task.Id}: {task.Title}");
if (!string.IsNullOrWhiteSpace(task.Acceptance))
sb.AppendLine($"Done when: {task.Acceptance}");
}
sb.AppendLine($"Calls recorded by the engine ({run.Calls.Count}):");
foreach (var call in run.Calls.Take(40))
sb.AppendLine($"- {call.At:HH:mm:ss} {call.Tool}{(call.Operation is { } op ? "." + op : "")}: {call.Status}" +
(string.IsNullOrWhiteSpace(call.Summary) ? "" : $" — {Cap(call.Summary, 160)}"));
if (run.Calls.Count > 40)
sb.AppendLine($"- … and {run.Calls.Count - 40} more");
sb.AppendLine(run.Findings.Count == 0 ? "Mechanical findings: none." : "Mechanical findings:");
foreach (var f in run.Findings)
sb.AppendLine($"- {f.Kind}: {f.Text}");
sb.AppendLine("Result as the agent wrote it (its claim, not evidence):");
sb.Append(string.IsNullOrWhiteSpace(run.Result) ? "(none)" : run.Result);
return sb.ToString();
}
private static string Cap(string text, int max) => text.Length <= max ? text : text[..max] + "…";
private static async Task<ToolResult> DoctrineAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
{
if (context.Doctrines is not { } doctrines || context.Organisation is not { HasUnits: true } org)