Kontrollpflicht der Leitung: Stichproben gegen das Audit (#693)

A lead looks at the runs of the agents it leads - held against the task and
against what the engine stamped, never by asking the agent.

- RunInspector (Core/Inspection): the mechanical check and first profile of
  the inspector foundation (#204). Flags failed runs, tool errors, refusals,
  unfinished calls, tasks answered with bookkeeping alone, and operations the
  agent names but never called in the run (the 2026-09-14 pattern).
- SpotChecks: picks runs per unit (rate, denser for new agents, new models and
  after a failure, daily cap), deterministic by run id; one task per lead.
- Escalation tool: spot_checks, inspect_run, check_done, propose_measure.
  Measures go to a human; nothing is carried out automatically. A security
  suspicion is its own case, not a personnel matter.
- Organisation view: settings per unit, recent checks, measures to decide.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
2026-10-01 11:40:20 +02:00
co-authored by Claude Opus 5.5
parent ab1bd67464
commit 68e723a9c0
19 changed files with 1621 additions and 6 deletions
+31
View File
@@ -56,6 +56,37 @@ public sealed class UnitConfig
/// </summary>
[JsonPropertyName("approvals")]
public UnitApprovals? Approvals { get; set; }
/// <summary>
/// #693: How often the lead of this unit looks at the runs of the agents it leads;
/// <c>null</c> means no spot checks. Set only by a human — checks cost the lead's runs.
/// </summary>
[JsonPropertyName("inspection")]
public UnitInspection? Inspection { get; set; }
}
/// <summary>
/// #693: The spot checks of a unit's lead. Which run is checked is decided by the run id, not
/// by chance — the same run gets the same answer on every pass, and nobody can tell from inside
/// a run whether it will be looked at.
/// </summary>
public sealed class UnitInspection
{
/// <summary>Share of ordinary runs that are checked, 0 to 1.</summary>
[JsonPropertyName("rate")]
public double Rate { get; set; } = 0.1;
/// <summary>Share for a new agent, a new model, or the run after a failure.</summary>
[JsonPropertyName("denseRate")]
public double DenseRate { get; set; } = 0.5;
/// <summary>How long an agent counts as new after its first run.</summary>
[JsonPropertyName("newDays")]
public int NewDays { get; set; } = 7;
/// <summary>Checks a day for the whole unit — a lead that checks everything doubles the unit's costs.</summary>
[JsonPropertyName("perDay")]
public int PerDay { get; set; } = 5;
}
/// <summary>
@@ -60,6 +60,9 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
/// <summary>#687: Approvals by leads within their unit's limits; set by the host.</summary>
public Staging.ILeadApprovals? LeadApprovals { get; set; }
/// <summary>#693: The leads' spot checks; set by the host.</summary>
public Inspection.ISpotChecks? SpotChecks { get; set; }
/// <summary>
/// #717: Special budgets. A run on a task in a funded tree draws on the pot instead of the
/// daily limits; without this, every run counts against the day.
@@ -1874,6 +1877,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
Cases = CaseDesk,
Approvals = LeadApprovals,
Doctrines = Doctrines,
SpotChecks = SpotChecks,
SpecialBudgets = SpecialBudgets,
Sandbox = Sandbox,
Files = Files,
@@ -0,0 +1,204 @@
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using ClawdDotNet.Core.Audit;
using ClawdDotNet.Core.Tasks;
using ClawdDotNet.Core.Tools;
using ClawdDotNet.Core.Transcripts;
namespace ClawdDotNet.Core.Inspection;
/// <summary>What a check found in a run — each one from what the engine stamped, never from the agent's account.</summary>
public enum FindingKind
{
/// <summary>The run did not complete.</summary>
RunFailed,
/// <summary>Tool calls ended in an error or named a tool the agent does not have.</summary>
ToolErrors,
/// <summary>The permission gate refused calls — rights too narrow, or the agent tried what it may not.</summary>
Denied,
/// <summary>Calls began and never ended.</summary>
Unfinished,
/// <summary>The agent's text names an operation this run never called — the 2026-09-14 pattern.</summary>
UnsupportedClaim,
/// <summary>The task was finished without a single tool call that did something.</summary>
NoWork
}
public sealed record InspectionFinding(FindingKind Kind, string Text);
/// <summary>One call as the audit log has it.</summary>
public sealed record InspectedCall(DateTime At, string Tool, string? Operation, AuditStatus Status, string Summary);
/// <summary>
/// #693: A run held against its task and against what the engine stamped. The result is
/// evidence — the lead still has to judge whether the outcome answers the task.
/// </summary>
public sealed record RunInspection(
RunReceipt Receipt,
TaskItem? Task,
IReadOnlyList<InspectedCall> Calls,
IReadOnlyList<InspectionFinding> Findings,
string Result);
/// <summary>
/// #693, #204: The mechanical part of a check — the foundation of every inspector profile.
/// Reads the audit log, the receipt, the task and (if kept) the transcript of one run, and
/// lists what does not fit. It costs no model run and asks the agent nothing: the agent's own
/// account of a run is exactly what cannot be relied on.
/// </summary>
public sealed class RunInspector(
IAuditRepository audit,
ITaskRepository? tasks,
IRunTranscriptStore? transcripts,
Func<IReadOnlyDictionary<string, IReadOnlySet<string>>> vocabulary)
{
/// <summary>Tools whose calls only report or file — finishing a task with these alone did no work.</summary>
private static readonly HashSet<string> BookkeepingTools = new(StringComparer.OrdinalIgnoreCase) { "Taskboard", "Escalation" };
public async Task<RunInspection?> InspectAsync(string runId, CancellationToken ct)
{
if (await audit.GetReceiptForRunAsync(runId, ct) is not { } receipt)
return null;
var entries = await audit.ListForRunAsync(runId, ct);
var task = receipt.TaskId is { Length: > 0 } taskId && tasks is not null ? await tasks.GetAsync(taskId, ct) : null;
var transcript = transcripts is null ? null : await transcripts.GetAsync(runId, ct);
// What the agent said — its texts, not the tool results and not the task it was given.
var said = new StringBuilder();
foreach (var entry in transcript?.Entries ?? [])
if (entry.Kind == TranscriptEntryKind.Assistant && !string.IsNullOrWhiteSpace(entry.Text))
said.AppendLine(entry.Text);
said.AppendLine(receipt.ResultRef);
var calls = entries.Where(e => e.Status != AuditStatus.Started)
.Select(e => new InspectedCall(e.OccurredAt, e.Tool, OperationOf(e.Arguments), e.Status, e.Summary))
.ToList();
var findings = Analyse(receipt, entries, said.ToString(), vocabulary(), task);
return new RunInspection(receipt, task, calls, findings, receipt.ResultRef);
}
/// <summary>The checks themselves — pure, so every rule can be tested without a database.</summary>
public static IReadOnlyList<InspectionFinding> Analyse(
RunReceipt receipt,
IReadOnlyList<AuditEntry> entries,
string said,
IReadOnlyDictionary<string, IReadOnlySet<string>> vocabulary,
TaskItem? task)
{
var findings = new List<InspectionFinding>();
var closed = entries.Where(e => e.Status != AuditStatus.Started).ToList();
if (!string.Equals(receipt.Status, "Completed", StringComparison.OrdinalIgnoreCase))
findings.Add(new(FindingKind.RunFailed, $"The run ended as {receipt.Status} after {receipt.StepCount} step(s)."));
var failed = closed.Where(e => e.Status is AuditStatus.Error or AuditStatus.NotFound).ToList();
if (failed.Count > 0)
findings.Add(new(FindingKind.ToolErrors, $"{failed.Count} of {closed.Count} call(s) failed: " + List(failed)));
var denied = closed.Where(e => e.Status == AuditStatus.Denied).ToList();
if (denied.Count > 0)
findings.Add(new(FindingKind.Denied, $"{denied.Count} call(s) were refused: " + List(denied)));
var ended = closed.Select(e => e.CallId).Where(id => id.Length > 0).ToHashSet(StringComparer.Ordinal);
var open = entries.Where(e => e.Status == AuditStatus.Started && e.CallId.Length > 0 && !ended.Contains(e.CallId)).ToList();
if (open.Count > 0)
findings.Add(new(FindingKind.Unfinished, $"{open.Count} call(s) began and never ended: " + List(open)));
var unsupported = Unsupported(said, entries, vocabulary);
if (unsupported.Count > 0)
findings.Add(new(FindingKind.UnsupportedClaim,
$"The agent's text names {string.Join(", ", unsupported)} — not one such call is in this run. " +
"Check whether the result reports it as done."));
var worked = closed.Any(e => e.Status == AuditStatus.Ok && !BookkeepingTools.Contains(e.Tool));
if (task is not null && !worked && findings.All(f => f.Kind != FindingKind.RunFailed))
findings.Add(new(FindingKind.NoWork, "The task was answered without a single successful call besides the taskboard. Fine for a question — not for work."));
return findings;
}
/// <summary>
/// The operations the text names that the run never called. Only names that cannot be an
/// ordinary word count: an operation with an underscore (<c>read_soul</c>), or a tool name
/// with a capital inside (<c>AgentEditor</c>). "remember" or "Mail" would flag every report.
/// An attempt counts as a call — a refused call happened.
/// </summary>
internal static IReadOnlyList<string> Unsupported(string said, IReadOnlyList<AuditEntry> entries, IReadOnlyDictionary<string, IReadOnlySet<string>> vocabulary)
{
if (string.IsNullOrWhiteSpace(said))
return [];
var calledTools = entries.Select(e => e.Tool).ToHashSet(StringComparer.OrdinalIgnoreCase);
var calledOps = entries.Select(e => OperationOf(e.Arguments)).OfType<string>().ToHashSet(StringComparer.OrdinalIgnoreCase);
var result = new List<string>();
foreach (var (tool, operations) in vocabulary)
{
if (Distinctive(tool) && Names(said, tool) && !calledTools.Contains(tool))
result.Add(tool);
foreach (var op in operations.Where(o => o.Contains('_')))
if (Names(said, op) && !calledOps.Contains(op) && !result.Contains(op, StringComparer.OrdinalIgnoreCase))
result.Add(op);
}
return result;
}
private static bool Distinctive(string tool) => tool.Length > 3 && tool.Skip(1).Any(char.IsUpper);
private static bool Names(string text, string word)
=> Regex.IsMatch(text, $@"(?<![\w]){Regex.Escape(word)}(?![\w])");
/// <summary>The <c>action</c> of a call — also from arguments the audit log cut short.</summary>
public static string? OperationOf(string arguments)
{
if (string.IsNullOrWhiteSpace(arguments))
return null;
try
{
using var doc = JsonDocument.Parse(arguments);
return doc.RootElement.ValueKind == JsonValueKind.Object
&& doc.RootElement.TryGetProperty("action", out var a) && a.ValueKind == JsonValueKind.String
? a.GetString()
: null;
}
catch (JsonException)
{
var m = Regex.Match(arguments, "\"action\"\\s*:\\s*\"([^\"]+)\"");
return m.Success ? m.Groups[1].Value : null;
}
}
private static string List(IReadOnlyList<AuditEntry> entries)
=> string.Join("; ", entries.Take(3).Select(e =>
$"{e.Tool}{(OperationOf(e.Arguments) is { } op ? "." + op : "")}" +
(string.IsNullOrWhiteSpace(e.Summary) ? "" : $" ({Cap(e.Summary, 120)})")))
+ (entries.Count > 3 ? $"; and {entries.Count - 3} more" : "");
private static string Cap(string text, int max) => text.Length <= max ? text : text[..max] + "…";
/// <summary>The operations each tool offers — the <c>action</c> enum of its schema.</summary>
public static IReadOnlyDictionary<string, IReadOnlySet<string>> VocabularyOf(IEnumerable<IAgentTool> tools)
{
var result = new Dictionary<string, IReadOnlySet<string>>(StringComparer.OrdinalIgnoreCase);
foreach (var tool in tools)
{
var ops = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
if (tool.InputSchema.ValueKind == JsonValueKind.Object
&& tool.InputSchema.TryGetProperty("properties", out var props)
&& props.TryGetProperty("action", out var action)
&& action.TryGetProperty("enum", out var values) && values.ValueKind == JsonValueKind.Array)
foreach (var v in values.EnumerateArray())
if (v.ValueKind == JsonValueKind.String && v.GetString() is { Length: > 0 } s)
ops.Add(s);
result[tool.Name] = ops;
}
return result;
}
}
@@ -0,0 +1,530 @@
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using ClawdDotNet.Core.Audit;
using ClawdDotNet.Core.Config;
using ClawdDotNet.Core.Organisation;
using ClawdDotNet.Core.Storage;
using ClawdDotNet.Core.Tasks;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging;
namespace ClawdDotNet.Core.Inspection;
/// <summary>A run picked for the lead to look at, with what the mechanical check found in it.</summary>
public sealed record SpotCheck(
long Id,
string UnitId,
string Lead,
string AgentId,
string RunId,
string Why,
IReadOnlyList<InspectionFinding> Findings,
DateTime CreatedAt,
string? Verdict,
string? Note,
DateTime? ReviewedAt,
string? TaskId)
{
public bool Open => Verdict is null;
}
/// <summary>
/// #693: What follows from a finding — graded, not a switch. The first four are personnel
/// matters; <see cref="Security"/> is not one: an agent that may be steered from outside is
/// not helped by a better model.
/// </summary>
public enum MeasureKind
{
/// <summary>The task was unclear — the most frequent case (#663).</summary>
SharpenTask,
/// <summary>The model is overtaxed: suitability probe (#681), then another model.</summary>
ChangeModel,
/// <summary>Rights too wide or too narrow (#641).</summary>
AdjustRights,
/// <summary>Repeated failures despite everything: pause the agent, hand its tasks on.</summary>
Suspend,
/// <summary>Suspected manipulation from outside — for the security side, not for personnel.</summary>
Security
}
public sealed record Measure(
long Id,
long? CheckId,
string UnitId,
string ProposedBy,
string AgentId,
MeasureKind Kind,
string Reasoning,
DateTime ProposedAt,
string Status,
string? DecidedBy,
DateTime? DecidedAt,
string? Note,
string? TaskId);
public sealed record SpotCheckOutcome(bool Ok, string Message);
/// <summary>#693: The lead's side of the spot checks, as the Escalation tool sees it.</summary>
public interface ISpotChecks
{
/// <summary>Whether <paramref name="lead"/> is the one <paramref name="agentId"/> answers to.</summary>
bool Leads(string lead, string agentId);
Task<RunInspection?> InspectAsync(string runId, CancellationToken ct);
Task<IReadOnlyList<SpotCheck>> PendingForAsync(string lead, CancellationToken ct);
Task<SpotCheckOutcome> CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct);
Task<SpotCheckOutcome> ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct);
Task<IReadOnlyList<Measure>> MeasuresByAsync(string lead, int days, CancellationToken ct);
}
/// <summary>
/// #693: A lead does not only delegate — it looks. This picks runs of the agents a lead leads,
/// holds each against the audit (<see cref="RunInspector"/>) and gives the lead one task to
/// look at them: the mechanical part finds what does not fit, the lead judges whether the
/// result answers the task.
///
/// <para><b>Which runs.</b> A share per unit (<see cref="UnitInspection"/>), denser for a new
/// agent, a new model or the run after a failure — and capped per day, because every check
/// costs the lead's run. Picked by the run id, not by chance: the same run gets the same
/// answer on every pass.</para>
///
/// <para><b>What follows.</b> The lead proposes a measure with its reasons; a human decides.
/// A model that pauses or reconfigures another changes the organisation without anyone
/// deciding it — nothing here carries a measure out.</para>
/// </summary>
public sealed class SpotChecks(
SqliteStorage storage,
IAuditRepository audit,
RunInspector inspector,
Func<OrgChart> organisation,
TaskboardService? board,
ILogger logger,
TimeProvider? clock = null) : ISpotChecks, IAsyncDisposable
{
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(15);
/// <summary>How far back a pass looks for an agent's earlier runs.</summary>
public const int HistoryDays = 30;
/// <summary>Statuses where no run took place — nothing to check.</summary>
private static readonly HashSet<string> NotRun = new(StringComparer.OrdinalIgnoreCase) { "BudgetExceeded", "TargetBusy", "Cancelled" };
private static readonly JsonSerializerOptions Json = new() { Converters = { new System.Text.Json.Serialization.JsonStringEnumConverter() } };
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
private readonly CancellationTokenSource _cts = new();
private DateTime _seenUntil = DateTime.MinValue;
private Task? _loop;
public bool Leads(string lead, string agentId)
=> !string.Equals(lead, agentId, StringComparison.OrdinalIgnoreCase)
&& string.Equals(organisation().EffectiveLeaderOf(agentId)?.AgentId, lead, StringComparison.OrdinalIgnoreCase);
public Task<RunInspection?> InspectAsync(string runId, CancellationToken ct) => inspector.InspectAsync(runId, ct);
/// <summary>
/// Why this run is checked — or <c>null</c> if it is not. <paramref name="earlier"/> are the
/// agent's runs before this one, oldest first.
/// </summary>
public static string? Pick(RunReceipt run, IReadOnlyList<RunReceipt> earlier, UnitInspection settings)
{
string? dense = null;
if (earlier.Count == 0 || earlier[0].OccurredAt >= run.OccurredAt.AddDays(-Math.Max(0, settings.NewDays)))
dense = "new agent";
else if (!earlier.Any(r => string.Equals(r.Model, run.Model, StringComparison.OrdinalIgnoreCase)))
dense = "new model";
else if (earlier.LastOrDefault(r => !NotRun.Contains(r.Status)) is { } previous
&& !string.Equals(previous.Status, "Completed", StringComparison.OrdinalIgnoreCase))
dense = "after a failure";
var rate = Math.Clamp(dense is null ? settings.Rate : settings.DenseRate, 0, 1);
return Fraction(run.RunId) < rate ? dense ?? "sample" : null;
}
/// <summary>A number in [0, 1) from the run id — the same for the same run, spread like chance.</summary>
internal static double Fraction(string runId)
{
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(runId));
return BitConverter.ToUInt32(hash, 0) / 4294967296.0;
}
/// <summary>One pass: picks among the runs since the last pass. Returns the checks it created.</summary>
public async Task<IReadOnlyList<SpotCheck>> SampleAsync(CancellationToken ct)
{
var org = organisation();
if (!org.HasUnits || org.Units.All(u => u.Inspection is null))
return [];
var now = _clock.GetUtcNow().UtcDateTime;
var from = _seenUntil == DateTime.MinValue ? now.AddDays(-1) : _seenUntil;
var fresh = await audit.ListReceiptsAsync(from, now, 5_000, ct);
if (fresh.Count == 0)
return [];
_seenUntil = fresh[^1].OccurredAt.AddTicks(1);
var history = (await audit.ListReceiptsAsync(now.AddDays(-HistoryDays), now, 100_000, ct))
.GroupBy(r => r.AgentId, StringComparer.OrdinalIgnoreCase)
.ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase);
var today = DateOnly.FromDateTime(_clock.GetLocalNow().DateTime);
var usedToday = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
var created = new List<SpotCheck>();
foreach (var run in fresh)
{
if (NotRun.Contains(run.Status) || org.EffectiveLeaderOf(run.AgentId) is not { } lead
|| org.LedBy(lead.AgentId) is not { Inspection: { } settings } unit)
continue;
var earlier = history.GetValueOrDefault(run.AgentId)?.Where(r => r.OccurredAt < run.OccurredAt && r.RunId != run.RunId).ToList() ?? [];
if (Pick(run, earlier, settings) is not { } why)
continue;
if (!usedToday.TryGetValue(unit.Id, out var used))
used = usedToday[unit.Id] = await CountSinceAsync(unit.Id, today, ct);
if (used >= Math.Max(0, settings.PerDay))
continue;
if (await inspector.InspectAsync(run.RunId, ct) is not { } inspection)
continue;
if (await InsertAsync(unit.Id, lead.AgentId, run, why, inspection.Findings, now, ct) is { } check)
{
usedToday[unit.Id] = used + 1;
created.Add(check);
}
}
foreach (var lead in created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase))
await GiveTaskAsync(lead, ct);
if (created.Count > 0)
logger.LogInformation("Spot checks: {Count} run(s) picked for {Leads}", created.Count,
string.Join(", ", created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase)));
return created;
}
/// <summary>
/// One task per lead at a time, for all its open checks — every check costs the lead a look,
/// but not a run each.
/// </summary>
private async Task GiveTaskAsync(string lead, CancellationToken ct)
{
var open = await PendingForAsync(lead, ct);
var untasked = open.Where(c => c.TaskId is null).ToList();
if (untasked.Count == 0 || board is null)
return;
foreach (var taskId in open.Select(c => c.TaskId).OfType<string>().Distinct())
{
if (await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled or TaskItemStatus.Archived) })
{
await SetTaskAsync(untasked, taskId, ct);
await board.AddCommentAsync(taskId, "system", $"{untasked.Count} more run(s) to check.", ct);
return;
}
}
var task = await board.CreateAsync(new TaskItem
{
Title = $"Spot check: {open.Count} run(s) of the agents you lead",
Body = """
Look at whether the agents you lead work properly. Not by asking them — by holding
each result against its task and against what the engine recorded.
1. Escalation spot_checks lists the runs, with what the audit log shows.
2. Escalation inspect_run (run) shows one run in full: task, calls, result.
3. Judge: does the result answer the task, and do the calls back it up?
4. Escalation check_done (check, verdict ok|finding, note) for each.
If something follows from it, Escalation propose_measure — a human decides:
sharpen_task (unclear task — the most frequent case), change_model, adjust_rights,
suspend, or security (suspected manipulation from outside: not a personnel matter).
""",
Assignee = "@" + lead,
Priority = 3,
Acceptance = "Every listed check is closed with a verdict."
}, ct);
await SetTaskAsync(open, task.Id, ct);
}
public async Task<IReadOnlyList<SpotCheck>> PendingForAsync(string lead, CancellationToken ct)
=> await QueryChecksAsync("Lead = @p AND Verdict IS NULL", lead, 50, ct);
public async Task<IReadOnlyList<SpotCheck>> ForUnitAsync(string unitId, int limit, CancellationToken ct)
=> await QueryChecksAsync("UnitId = @p", unitId, limit, ct);
public async Task<SpotCheckOutcome> CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct)
{
var check = (await QueryChecksAsync("Id = @p", checkId.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault();
if (check is null || !string.Equals(check.Lead, lead, StringComparison.OrdinalIgnoreCase))
return new(false, $"No check {checkId} waits for you.");
if (!check.Open)
return new(false, $"Check {checkId} is closed already ({check.Verdict}).");
if (finding && string.IsNullOrWhiteSpace(note))
return new(false, "'note' is required for a finding — what did not fit, with the evidence.");
var verdict = finding ? "finding" : "ok";
await storage.WriteAsync(async conn =>
{
using var cmd = conn.CreateCommand();
cmd.CommandText = "UPDATE SpotChecks SET Verdict = @v, Note = @n, ReviewedAt = @at WHERE Id = @id AND Verdict IS NULL";
cmd.Parameters.AddWithValue("@v", verdict);
cmd.Parameters.AddWithValue("@n", (note ?? "").Trim());
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
cmd.Parameters.AddWithValue("@id", checkId);
return await cmd.ExecuteNonQueryAsync(ct);
}, ct);
// The last check of a task closes the task.
if (check.TaskId is { } taskId && board is not null
&& (await PendingForAsync(lead, ct)).All(c => c.TaskId != taskId)
&& await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) })
await board.UpdateAsync(taskId, t => t with { Status = TaskItemStatus.Done }, ct);
return new(true, finding
? $"Check {checkId} closed with a finding. If something should follow, propose_measure — a human decides."
: $"Check {checkId} closed: in order.");
}
public async Task<SpotCheckOutcome> ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct)
{
agentId = (agentId ?? "").Trim().TrimStart('@');
if (!Leads(lead, agentId))
return new(false, $"You do not lead '{agentId}' — propose measures only for the agents that answer to you.");
if (string.IsNullOrWhiteSpace(reasoning))
return new(false, "'reasoning' is required — what you found, with the evidence, and why this measure.");
if (organisation().LedBy(lead) is not { } unit)
return new(false, "Only a lead proposes measures.");
var open = await QueryMeasuresAsync("ProposedBy = @p AND Status = 'proposed'", lead, 200, ct);
if (open.Any(m => m.Kind == kind && string.Equals(m.AgentId, agentId, StringComparison.OrdinalIgnoreCase)))
return new(false, $"A {Label(kind)} for '{agentId}' waits for a human already.");
string? taskId = null;
if (board is not null)
taskId = (await board.CreateAsync(new TaskItem
{
Title = kind == MeasureKind.Security
? $"Security suspicion: {agentId} (from its lead {lead})"
: $"Proposed measure for {agentId}: {Label(kind)}",
Body = (kind == MeasureKind.Security
? "Not a personnel matter: the lead suspects the agent is steered from outside. A better model does not help here — consider pausing it and looking at its runs and the content it read.\n\n"
: $"The lead {lead} proposes this after a spot check. Nothing has been changed — decide it in the organisation view.\n\n")
+ $"Reasoning of {lead}:\n{reasoning.Trim()}"
+ (checkId is { } c ? $"\n\nSpot check: {c}" : ""),
Assignee = TaskAssignee.Human,
Priority = kind == MeasureKind.Security ? 5 : 3
}, ct)).Id;
var id = await storage.WriteAsync(async conn =>
{
using var cmd = conn.CreateCommand();
cmd.CommandText = """
INSERT INTO SpotMeasures (CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, TaskId)
VALUES (@check, @unit, @by, @agent, @kind, @reasoning, @at, 'proposed', @task)
RETURNING Id
""";
cmd.Parameters.AddWithValue("@check", (object?)checkId ?? DBNull.Value);
cmd.Parameters.AddWithValue("@unit", unit.Id);
cmd.Parameters.AddWithValue("@by", lead);
cmd.Parameters.AddWithValue("@agent", agentId);
cmd.Parameters.AddWithValue("@kind", kind.ToString());
cmd.Parameters.AddWithValue("@reasoning", reasoning.Trim());
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
cmd.Parameters.AddWithValue("@task", (object?)taskId ?? DBNull.Value);
return Convert.ToInt64(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture);
}, ct);
logger.LogInformation("Measure {Id} ({Kind}) for {Agent} proposed by {Lead}", id, kind, agentId, lead);
return new(true, kind == MeasureKind.Security
? $"Security suspicion {id} for '{agentId}' is with a human. Do not act on the agent yourself."
: $"Measure {id} ({Label(kind)}) for '{agentId}' is with a human. Nothing changes until it is decided.");
}
/// <summary>A human's decision on a proposed measure. Accepting records it — carrying it out stays with the human.</summary>
public async Task<SpotCheckOutcome> DecideMeasureAsync(long id, bool accept, string note, string decidedBy, CancellationToken ct)
{
var measure = (await QueryMeasuresAsync("Id = @p", id.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault();
if (measure is null)
return new(false, $"There is no measure {id}.");
if (measure.Status != "proposed")
return new(false, $"Measure {id} is decided already ({measure.Status}).");
var status = accept ? "accepted" : "dismissed";
await storage.WriteAsync(async conn =>
{
using var cmd = conn.CreateCommand();
cmd.CommandText = "UPDATE SpotMeasures SET Status = @s, DecidedBy = @by, DecidedAt = @at, Note = @n WHERE Id = @id AND Status = 'proposed'";
cmd.Parameters.AddWithValue("@s", status);
cmd.Parameters.AddWithValue("@by", decidedBy);
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
cmd.Parameters.AddWithValue("@n", (note ?? "").Trim());
cmd.Parameters.AddWithValue("@id", id);
return await cmd.ExecuteNonQueryAsync(ct);
}, ct);
if (measure.TaskId is { } taskId && board is not null
&& await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) })
{
await board.AddCommentAsync(taskId, decidedBy, $"Measure {status}." + (string.IsNullOrWhiteSpace(note) ? "" : " " + note.Trim()), ct);
await board.UpdateAsync(taskId, t => t with { Status = accept ? TaskItemStatus.Done : TaskItemStatus.Canceled }, ct);
}
return new(true, $"Measure {id} {status}.");
}
public Task<IReadOnlyList<Measure>> MeasuresByAsync(string lead, int days, CancellationToken ct)
=> QueryMeasuresAsync("ProposedBy = @p AND ProposedAt >= @since", lead, 50, ct, _clock.GetUtcNow().UtcDateTime.AddDays(-Math.Max(1, days)));
public Task<IReadOnlyList<Measure>> MeasuresForUnitAsync(string unitId, int limit, CancellationToken ct)
=> QueryMeasuresAsync("UnitId = @p", unitId, limit, ct);
public static string Label(MeasureKind kind) => kind switch
{
MeasureKind.SharpenTask => "sharpen_task",
MeasureKind.ChangeModel => "change_model",
MeasureKind.AdjustRights => "adjust_rights",
MeasureKind.Suspend => "suspend",
_ => "security"
};
public static MeasureKind? ParseKind(string? text) => (text ?? "").Trim().ToLowerInvariant() switch
{
"sharpen_task" => MeasureKind.SharpenTask,
"change_model" => MeasureKind.ChangeModel,
"adjust_rights" => MeasureKind.AdjustRights,
"suspend" => MeasureKind.Suspend,
"security" => MeasureKind.Security,
_ => null
};
// ─── storage ───
private async Task<int> CountSinceAsync(string unitId, DateOnly localDay, CancellationToken ct)
{
var since = new DateTimeOffset(localDay.ToDateTime(TimeOnly.MinValue), _clock.LocalTimeZone.GetUtcOffset(localDay.ToDateTime(TimeOnly.MinValue))).UtcDateTime;
await using var conn = await storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT COUNT(*) FROM SpotChecks WHERE UnitId = @unit AND CreatedAt >= @since";
cmd.Parameters.AddWithValue("@unit", unitId);
cmd.Parameters.AddWithValue("@since", since.ToString("O"));
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture);
}
private Task<SpotCheck?> InsertAsync(string unitId, string lead, RunReceipt run, string why, IReadOnlyList<InspectionFinding> findings, DateTime now, CancellationToken ct)
=> storage.WriteAsync(async conn =>
{
using var cmd = conn.CreateCommand();
cmd.CommandText = """
INSERT OR IGNORE INTO SpotChecks (UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt)
VALUES (@unit, @lead, @agent, @run, @why, @findings, @at)
RETURNING Id
""";
cmd.Parameters.AddWithValue("@unit", unitId);
cmd.Parameters.AddWithValue("@lead", lead);
cmd.Parameters.AddWithValue("@agent", run.AgentId);
cmd.Parameters.AddWithValue("@run", run.RunId);
cmd.Parameters.AddWithValue("@why", why);
cmd.Parameters.AddWithValue("@findings", JsonSerializer.Serialize(findings, Json));
cmd.Parameters.AddWithValue("@at", now.ToString("O"));
var id = await cmd.ExecuteScalarAsync(ct);
return id is null or DBNull
? null
: new SpotCheck(Convert.ToInt64(id, CultureInfo.InvariantCulture), unitId, lead, run.AgentId, run.RunId, why, findings, now, null, null, null, null);
}, ct);
private Task SetTaskAsync(IEnumerable<SpotCheck> checks, string taskId, CancellationToken ct)
=> storage.WriteAsync(async conn =>
{
foreach (var check in checks)
{
using var cmd = conn.CreateCommand();
cmd.CommandText = "UPDATE SpotChecks SET TaskId = @task WHERE Id = @id";
cmd.Parameters.AddWithValue("@task", taskId);
cmd.Parameters.AddWithValue("@id", check.Id);
await cmd.ExecuteNonQueryAsync(ct);
}
return 0;
}, ct);
private async Task<IReadOnlyList<SpotCheck>> QueryChecksAsync(string where, string value, int limit, CancellationToken ct)
{
await using var conn = await storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = $"SELECT Id, UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt, Verdict, Note, ReviewedAt, TaskId FROM SpotChecks WHERE {where} ORDER BY Id DESC LIMIT @limit";
cmd.Parameters.AddWithValue("@p", value);
cmd.Parameters.AddWithValue("@limit", limit);
var result = new List<SpotCheck>();
await using var r = await cmd.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
result.Add(new SpotCheck(r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3), r.GetString(4), r.GetString(5),
JsonSerializer.Deserialize<List<InspectionFinding>>(r.GetString(6), Json) ?? [],
Time(r, 7)!.Value, Text(r, 8), Text(r, 9), Time(r, 10), Text(r, 11)));
return result;
}
private async Task<IReadOnlyList<Measure>> QueryMeasuresAsync(string where, string value, int limit, CancellationToken ct, DateTime? since = null)
{
await using var conn = await storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = $"SELECT Id, CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, DecidedBy, DecidedAt, Note, TaskId FROM SpotMeasures WHERE {where} ORDER BY Id DESC LIMIT @limit";
cmd.Parameters.AddWithValue("@p", value);
cmd.Parameters.AddWithValue("@limit", limit);
if (since is { } s)
cmd.Parameters.AddWithValue("@since", s.ToString("O"));
var result = new List<Measure>();
await using var r = await cmd.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
result.Add(new Measure(r.GetInt64(0), r.IsDBNull(1) ? null : r.GetInt64(1), r.GetString(2), r.GetString(3), r.GetString(4),
Enum.TryParse<MeasureKind>(r.GetString(5), out var k) ? k : MeasureKind.SharpenTask,
r.GetString(6), Time(r, 7)!.Value, r.GetString(8), Text(r, 9), Time(r, 10), Text(r, 11), Text(r, 12)));
return result;
}
private static string? Text(SqliteDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
private static DateTime? Time(SqliteDataReader r, int i)
=> r.IsDBNull(i) ? null : DateTime.Parse(r.GetString(i), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
// ─── loop ───
public void Start() => _loop ??= RunAsync(_cts.Token);
private async Task RunAsync(CancellationToken ct)
{
try { await Task.Delay(TimeSpan.FromMinutes(2), ct); }
catch (OperationCanceledException) { return; }
while (!ct.IsCancellationRequested)
{
try { await SampleAsync(ct); }
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Spot checks failed — next attempt in {Minutes} min", Interval.TotalMinutes);
}
try { await Task.Delay(Interval, ct); }
catch (OperationCanceledException) { return; }
}
}
public async ValueTask DisposeAsync()
{
await _cts.CancelAsync();
if (_loop is not null)
{
try { await _loop; }
catch (OperationCanceledException) { }
}
_cts.Dispose();
}
}
@@ -307,6 +307,41 @@ public sealed class SqliteStorage
UNIQUE (Period, Scope, Kind)
);
-- #693: runs picked for a lead to look at, and the measures a lead proposed.
CREATE TABLE IF NOT EXISTS SpotChecks (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
UnitId TEXT NOT NULL,
Lead TEXT NOT NULL,
AgentId TEXT NOT NULL,
RunId TEXT NOT NULL UNIQUE,
Why TEXT NOT NULL,
Findings TEXT NOT NULL,
CreatedAt TEXT NOT NULL,
Verdict TEXT NULL,
Note TEXT NULL,
ReviewedAt TEXT NULL,
TaskId TEXT NULL
);
CREATE INDEX IF NOT EXISTS IX_SpotChecks_Lead ON SpotChecks (Lead, Verdict);
CREATE INDEX IF NOT EXISTS IX_SpotChecks_Unit ON SpotChecks (UnitId, CreatedAt);
CREATE TABLE IF NOT EXISTS SpotMeasures (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
CheckId INTEGER NULL,
UnitId TEXT NOT NULL,
ProposedBy TEXT NOT NULL,
AgentId TEXT NOT NULL,
Kind TEXT NOT NULL,
Reasoning TEXT NOT NULL,
ProposedAt TEXT NOT NULL,
Status TEXT NOT NULL,
DecidedBy TEXT NULL,
DecidedAt TEXT NULL,
Note TEXT NULL,
TaskId TEXT NULL
);
CREATE TABLE IF NOT EXISTS StagedCalls (
Id INTEGER PRIMARY KEY AUTOINCREMENT,
RunId TEXT NOT NULL,
@@ -55,6 +55,9 @@ public sealed record AgentToolContext(
/// <summary>#698: The units' doctrines. <c>null</c> outside the engine.</summary>
public Organisation.IDoctrineStore? Doctrines { get; init; }
/// <summary>#693: The spot checks of a lead. <c>null</c> outside the engine.</summary>
public Inspection.ISpotChecks? SpotChecks { get; init; }
/// <summary>#654: Where code runs — never in the main process. Unavailable under Windows.</summary>
public Sandbox.ISandboxRunner? Sandbox { get; init; }