Kontrollpflicht der Leitung: Stichproben gegen das Audit (#693)
A lead looks at the runs of the agents it leads - held against the task and against what the engine stamped, never by asking the agent. - RunInspector (Core/Inspection): the mechanical check and first profile of the inspector foundation (#204). Flags failed runs, tool errors, refusals, unfinished calls, tasks answered with bookkeeping alone, and operations the agent names but never called in the run (the 2026-09-14 pattern). - SpotChecks: picks runs per unit (rate, denser for new agents, new models and after a failure, daily cap), deterministic by run id; one task per lead. - Escalation tool: spot_checks, inspect_run, check_done, propose_measure. Measures go to a human; nothing is carried out automatically. A security suspicion is its own case, not a personnel matter. - Organisation view: settings per unit, recent checks, measures to decide. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
ab1bd67464
commit
68e723a9c0
@@ -135,6 +135,9 @@ public sealed class AppHost : IAsyncDisposable
|
||||
/// <summary>#698: The units' doctrines.</summary>
|
||||
public Core.Organisation.IDoctrineStore? Doctrines { get; private set; }
|
||||
|
||||
/// <summary>#693: The leads' spot checks and the measures they proposed.</summary>
|
||||
public Core.Inspection.SpotChecks? SpotChecks { get; private set; }
|
||||
|
||||
/// <summary>Modellpreise von OpenRouter - fuer Budget und Kostenprognose (#595). Laedt im Hintergrund.</summary>
|
||||
public ModelPricingCatalog? Pricing { get; private init; }
|
||||
|
||||
@@ -717,6 +720,18 @@ public sealed class AppHost : IAsyncDisposable
|
||||
engine.Doctrines = doctrines;
|
||||
host.Doctrines = doctrines;
|
||||
|
||||
// #693: the leads look at the runs of the agents they lead.
|
||||
var spotChecks = new Core.Inspection.SpotChecks(
|
||||
storage, audit,
|
||||
new Core.Inspection.RunInspector(audit, taskRepository, transcripts,
|
||||
() => Core.Inspection.RunInspector.VocabularyOf(tools.GetAll())),
|
||||
() => Core.Organisation.OrgChart.From(instance), board,
|
||||
loggerFactory.CreateLogger("ClawdDotNet.Core.Inspection.SpotChecks"));
|
||||
engine.SpotChecks = spotChecks;
|
||||
host.SpotChecks = spotChecks;
|
||||
spotChecks.Start();
|
||||
host._shutdown.Add(spotChecks.DisposeAsync);
|
||||
|
||||
// #699: costs per unit, and what changed in them.
|
||||
var costWatch = new Core.Accounting.CostWatch(
|
||||
storage, usage, () => Core.Organisation.OrgChart.From(instance), () => instance.CostWatch, board,
|
||||
|
||||
@@ -271,6 +271,52 @@ public sealed class OrgAdministration(AppHost host, ILogger logger)
|
||||
return AdminResult.Success(trimmed.Count == 0 ? message : message + $" Lowered with it: {string.Join(", ", trimmed)}.");
|
||||
}
|
||||
|
||||
// ─── spot checks (#693) ───
|
||||
|
||||
/// <summary>
|
||||
/// #693: How often the lead of a unit looks at the runs of the agents it leads. Only a human
|
||||
/// sets it — every check costs the lead a look, and the cap is what keeps the control from
|
||||
/// costing more than the work.
|
||||
/// </summary>
|
||||
public AdminResult SetInspection(string unitId, bool enabled, double? rate = null, double? denseRate = null,
|
||||
int? newDays = null, int? perDay = null, string reason = "")
|
||||
{
|
||||
if (FindUnit(unitId) is not { } unit)
|
||||
return AdminResult.Failure($"There is no unit '{unitId}'.");
|
||||
|
||||
if (!enabled)
|
||||
{
|
||||
unit.Inspection = null;
|
||||
Save(null, "unit_inspection", new { unitId, enabled }, reason);
|
||||
return AdminResult.Success($"The lead of '{unit.Name}' takes no spot checks now.");
|
||||
}
|
||||
|
||||
if (rate is < 0 or > 1 || denseRate is < 0 or > 1)
|
||||
return AdminResult.Failure("A share lies between 0 and 1.");
|
||||
if (perDay is < 0 or > 100)
|
||||
return AdminResult.Failure("Checks a day lie between 0 and 100.");
|
||||
if (newDays is < 0 or > 365)
|
||||
return AdminResult.Failure("An agent counts as new for 0 to 365 days.");
|
||||
|
||||
var current = unit.Inspection ?? new UnitInspection();
|
||||
unit.Inspection = new UnitInspection
|
||||
{
|
||||
Rate = rate ?? current.Rate,
|
||||
DenseRate = denseRate ?? current.DenseRate,
|
||||
NewDays = newDays ?? current.NewDays,
|
||||
PerDay = perDay ?? current.PerDay
|
||||
};
|
||||
if (unit.Inspection.DenseRate < unit.Inspection.Rate)
|
||||
unit.Inspection.DenseRate = unit.Inspection.Rate;
|
||||
|
||||
var lead = Chart.LeaderOf(unit.Id);
|
||||
var i = unit.Inspection;
|
||||
Save(null, "unit_inspection", new { unitId, enabled, i.Rate, i.DenseRate, i.NewDays, i.PerDay }, reason);
|
||||
return AdminResult.Success(
|
||||
$"The lead of '{unit.Name}' checks {i.Rate:P0} of the runs ({i.DenseRate:P0} for new agents, new models and after a failure), at most {i.PerDay} a day."
|
||||
+ (lead is null ? " The unit has no lead: its agents are checked under the settings of the unit whose lead they answer to." : ""));
|
||||
}
|
||||
|
||||
/// <summary>Whether a grant covers an operation string — "Tool" covers every "Tool.x".</summary>
|
||||
private static bool Within(UnitApprovals granted, string op)
|
||||
{
|
||||
@@ -885,7 +931,8 @@ public sealed class OrgAdministration(AppHost host, ILogger logger)
|
||||
private static UnitConfig Clone(UnitConfig u) => new()
|
||||
{
|
||||
Id = u.Id, Name = u.Name, ParentId = u.ParentId, LeaderAgentId = u.LeaderAgentId,
|
||||
Goal = u.Goal, ToolCeiling = u.ToolCeiling?.ToList(), Defaults = u.Defaults, Approvals = u.Approvals
|
||||
Goal = u.Goal, ToolCeiling = u.ToolCeiling?.ToList(), Defaults = u.Defaults, Approvals = u.Approvals,
|
||||
Inspection = u.Inspection
|
||||
};
|
||||
|
||||
private static bool Same(string? a, string? b) => string.Equals(a, b, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
@@ -56,6 +56,37 @@ public sealed class UnitConfig
|
||||
/// </summary>
|
||||
[JsonPropertyName("approvals")]
|
||||
public UnitApprovals? Approvals { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// #693: How often the lead of this unit looks at the runs of the agents it leads;
|
||||
/// <c>null</c> means no spot checks. Set only by a human — checks cost the lead's runs.
|
||||
/// </summary>
|
||||
[JsonPropertyName("inspection")]
|
||||
public UnitInspection? Inspection { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #693: The spot checks of a unit's lead. Which run is checked is decided by the run id, not
|
||||
/// by chance — the same run gets the same answer on every pass, and nobody can tell from inside
|
||||
/// a run whether it will be looked at.
|
||||
/// </summary>
|
||||
public sealed class UnitInspection
|
||||
{
|
||||
/// <summary>Share of ordinary runs that are checked, 0 to 1.</summary>
|
||||
[JsonPropertyName("rate")]
|
||||
public double Rate { get; set; } = 0.1;
|
||||
|
||||
/// <summary>Share for a new agent, a new model, or the run after a failure.</summary>
|
||||
[JsonPropertyName("denseRate")]
|
||||
public double DenseRate { get; set; } = 0.5;
|
||||
|
||||
/// <summary>How long an agent counts as new after its first run.</summary>
|
||||
[JsonPropertyName("newDays")]
|
||||
public int NewDays { get; set; } = 7;
|
||||
|
||||
/// <summary>Checks a day for the whole unit — a lead that checks everything doubles the unit's costs.</summary>
|
||||
[JsonPropertyName("perDay")]
|
||||
public int PerDay { get; set; } = 5;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -60,6 +60,9 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
|
||||
/// <summary>#687: Approvals by leads within their unit's limits; set by the host.</summary>
|
||||
public Staging.ILeadApprovals? LeadApprovals { get; set; }
|
||||
|
||||
/// <summary>#693: The leads' spot checks; set by the host.</summary>
|
||||
public Inspection.ISpotChecks? SpotChecks { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// #717: Special budgets. A run on a task in a funded tree draws on the pot instead of the
|
||||
/// daily limits; without this, every run counts against the day.
|
||||
@@ -1874,6 +1877,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
|
||||
Cases = CaseDesk,
|
||||
Approvals = LeadApprovals,
|
||||
Doctrines = Doctrines,
|
||||
SpotChecks = SpotChecks,
|
||||
SpecialBudgets = SpecialBudgets,
|
||||
Sandbox = Sandbox,
|
||||
Files = Files,
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using ClawdDotNet.Core.Audit;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
using ClawdDotNet.Core.Tools;
|
||||
using ClawdDotNet.Core.Transcripts;
|
||||
|
||||
namespace ClawdDotNet.Core.Inspection;
|
||||
|
||||
/// <summary>What a check found in a run — each one from what the engine stamped, never from the agent's account.</summary>
|
||||
public enum FindingKind
|
||||
{
|
||||
/// <summary>The run did not complete.</summary>
|
||||
RunFailed,
|
||||
|
||||
/// <summary>Tool calls ended in an error or named a tool the agent does not have.</summary>
|
||||
ToolErrors,
|
||||
|
||||
/// <summary>The permission gate refused calls — rights too narrow, or the agent tried what it may not.</summary>
|
||||
Denied,
|
||||
|
||||
/// <summary>Calls began and never ended.</summary>
|
||||
Unfinished,
|
||||
|
||||
/// <summary>The agent's text names an operation this run never called — the 2026-09-14 pattern.</summary>
|
||||
UnsupportedClaim,
|
||||
|
||||
/// <summary>The task was finished without a single tool call that did something.</summary>
|
||||
NoWork
|
||||
}
|
||||
|
||||
public sealed record InspectionFinding(FindingKind Kind, string Text);
|
||||
|
||||
/// <summary>One call as the audit log has it.</summary>
|
||||
public sealed record InspectedCall(DateTime At, string Tool, string? Operation, AuditStatus Status, string Summary);
|
||||
|
||||
/// <summary>
|
||||
/// #693: A run held against its task and against what the engine stamped. The result is
|
||||
/// evidence — the lead still has to judge whether the outcome answers the task.
|
||||
/// </summary>
|
||||
public sealed record RunInspection(
|
||||
RunReceipt Receipt,
|
||||
TaskItem? Task,
|
||||
IReadOnlyList<InspectedCall> Calls,
|
||||
IReadOnlyList<InspectionFinding> Findings,
|
||||
string Result);
|
||||
|
||||
/// <summary>
|
||||
/// #693, #204: The mechanical part of a check — the foundation of every inspector profile.
|
||||
/// Reads the audit log, the receipt, the task and (if kept) the transcript of one run, and
|
||||
/// lists what does not fit. It costs no model run and asks the agent nothing: the agent's own
|
||||
/// account of a run is exactly what cannot be relied on.
|
||||
/// </summary>
|
||||
public sealed class RunInspector(
|
||||
IAuditRepository audit,
|
||||
ITaskRepository? tasks,
|
||||
IRunTranscriptStore? transcripts,
|
||||
Func<IReadOnlyDictionary<string, IReadOnlySet<string>>> vocabulary)
|
||||
{
|
||||
/// <summary>Tools whose calls only report or file — finishing a task with these alone did no work.</summary>
|
||||
private static readonly HashSet<string> BookkeepingTools = new(StringComparer.OrdinalIgnoreCase) { "Taskboard", "Escalation" };
|
||||
|
||||
public async Task<RunInspection?> InspectAsync(string runId, CancellationToken ct)
|
||||
{
|
||||
if (await audit.GetReceiptForRunAsync(runId, ct) is not { } receipt)
|
||||
return null;
|
||||
|
||||
var entries = await audit.ListForRunAsync(runId, ct);
|
||||
var task = receipt.TaskId is { Length: > 0 } taskId && tasks is not null ? await tasks.GetAsync(taskId, ct) : null;
|
||||
var transcript = transcripts is null ? null : await transcripts.GetAsync(runId, ct);
|
||||
|
||||
// What the agent said — its texts, not the tool results and not the task it was given.
|
||||
var said = new StringBuilder();
|
||||
foreach (var entry in transcript?.Entries ?? [])
|
||||
if (entry.Kind == TranscriptEntryKind.Assistant && !string.IsNullOrWhiteSpace(entry.Text))
|
||||
said.AppendLine(entry.Text);
|
||||
said.AppendLine(receipt.ResultRef);
|
||||
|
||||
var calls = entries.Where(e => e.Status != AuditStatus.Started)
|
||||
.Select(e => new InspectedCall(e.OccurredAt, e.Tool, OperationOf(e.Arguments), e.Status, e.Summary))
|
||||
.ToList();
|
||||
var findings = Analyse(receipt, entries, said.ToString(), vocabulary(), task);
|
||||
return new RunInspection(receipt, task, calls, findings, receipt.ResultRef);
|
||||
}
|
||||
|
||||
/// <summary>The checks themselves — pure, so every rule can be tested without a database.</summary>
|
||||
public static IReadOnlyList<InspectionFinding> Analyse(
|
||||
RunReceipt receipt,
|
||||
IReadOnlyList<AuditEntry> entries,
|
||||
string said,
|
||||
IReadOnlyDictionary<string, IReadOnlySet<string>> vocabulary,
|
||||
TaskItem? task)
|
||||
{
|
||||
var findings = new List<InspectionFinding>();
|
||||
var closed = entries.Where(e => e.Status != AuditStatus.Started).ToList();
|
||||
|
||||
if (!string.Equals(receipt.Status, "Completed", StringComparison.OrdinalIgnoreCase))
|
||||
findings.Add(new(FindingKind.RunFailed, $"The run ended as {receipt.Status} after {receipt.StepCount} step(s)."));
|
||||
|
||||
var failed = closed.Where(e => e.Status is AuditStatus.Error or AuditStatus.NotFound).ToList();
|
||||
if (failed.Count > 0)
|
||||
findings.Add(new(FindingKind.ToolErrors, $"{failed.Count} of {closed.Count} call(s) failed: " + List(failed)));
|
||||
|
||||
var denied = closed.Where(e => e.Status == AuditStatus.Denied).ToList();
|
||||
if (denied.Count > 0)
|
||||
findings.Add(new(FindingKind.Denied, $"{denied.Count} call(s) were refused: " + List(denied)));
|
||||
|
||||
var ended = closed.Select(e => e.CallId).Where(id => id.Length > 0).ToHashSet(StringComparer.Ordinal);
|
||||
var open = entries.Where(e => e.Status == AuditStatus.Started && e.CallId.Length > 0 && !ended.Contains(e.CallId)).ToList();
|
||||
if (open.Count > 0)
|
||||
findings.Add(new(FindingKind.Unfinished, $"{open.Count} call(s) began and never ended: " + List(open)));
|
||||
|
||||
var unsupported = Unsupported(said, entries, vocabulary);
|
||||
if (unsupported.Count > 0)
|
||||
findings.Add(new(FindingKind.UnsupportedClaim,
|
||||
$"The agent's text names {string.Join(", ", unsupported)} — not one such call is in this run. " +
|
||||
"Check whether the result reports it as done."));
|
||||
|
||||
var worked = closed.Any(e => e.Status == AuditStatus.Ok && !BookkeepingTools.Contains(e.Tool));
|
||||
if (task is not null && !worked && findings.All(f => f.Kind != FindingKind.RunFailed))
|
||||
findings.Add(new(FindingKind.NoWork, "The task was answered without a single successful call besides the taskboard. Fine for a question — not for work."));
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The operations the text names that the run never called. Only names that cannot be an
|
||||
/// ordinary word count: an operation with an underscore (<c>read_soul</c>), or a tool name
|
||||
/// with a capital inside (<c>AgentEditor</c>). "remember" or "Mail" would flag every report.
|
||||
/// An attempt counts as a call — a refused call happened.
|
||||
/// </summary>
|
||||
internal static IReadOnlyList<string> Unsupported(string said, IReadOnlyList<AuditEntry> entries, IReadOnlyDictionary<string, IReadOnlySet<string>> vocabulary)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(said))
|
||||
return [];
|
||||
|
||||
var calledTools = entries.Select(e => e.Tool).ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var calledOps = entries.Select(e => OperationOf(e.Arguments)).OfType<string>().ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var result = new List<string>();
|
||||
|
||||
foreach (var (tool, operations) in vocabulary)
|
||||
{
|
||||
if (Distinctive(tool) && Names(said, tool) && !calledTools.Contains(tool))
|
||||
result.Add(tool);
|
||||
foreach (var op in operations.Where(o => o.Contains('_')))
|
||||
if (Names(said, op) && !calledOps.Contains(op) && !result.Contains(op, StringComparer.OrdinalIgnoreCase))
|
||||
result.Add(op);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static bool Distinctive(string tool) => tool.Length > 3 && tool.Skip(1).Any(char.IsUpper);
|
||||
|
||||
private static bool Names(string text, string word)
|
||||
=> Regex.IsMatch(text, $@"(?<![\w]){Regex.Escape(word)}(?![\w])");
|
||||
|
||||
/// <summary>The <c>action</c> of a call — also from arguments the audit log cut short.</summary>
|
||||
public static string? OperationOf(string arguments)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(arguments))
|
||||
return null;
|
||||
try
|
||||
{
|
||||
using var doc = JsonDocument.Parse(arguments);
|
||||
return doc.RootElement.ValueKind == JsonValueKind.Object
|
||||
&& doc.RootElement.TryGetProperty("action", out var a) && a.ValueKind == JsonValueKind.String
|
||||
? a.GetString()
|
||||
: null;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
var m = Regex.Match(arguments, "\"action\"\\s*:\\s*\"([^\"]+)\"");
|
||||
return m.Success ? m.Groups[1].Value : null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string List(IReadOnlyList<AuditEntry> entries)
|
||||
=> string.Join("; ", entries.Take(3).Select(e =>
|
||||
$"{e.Tool}{(OperationOf(e.Arguments) is { } op ? "." + op : "")}" +
|
||||
(string.IsNullOrWhiteSpace(e.Summary) ? "" : $" ({Cap(e.Summary, 120)})")))
|
||||
+ (entries.Count > 3 ? $"; and {entries.Count - 3} more" : "");
|
||||
|
||||
private static string Cap(string text, int max) => text.Length <= max ? text : text[..max] + "…";
|
||||
|
||||
/// <summary>The operations each tool offers — the <c>action</c> enum of its schema.</summary>
|
||||
public static IReadOnlyDictionary<string, IReadOnlySet<string>> VocabularyOf(IEnumerable<IAgentTool> tools)
|
||||
{
|
||||
var result = new Dictionary<string, IReadOnlySet<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var tool in tools)
|
||||
{
|
||||
var ops = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (tool.InputSchema.ValueKind == JsonValueKind.Object
|
||||
&& tool.InputSchema.TryGetProperty("properties", out var props)
|
||||
&& props.TryGetProperty("action", out var action)
|
||||
&& action.TryGetProperty("enum", out var values) && values.ValueKind == JsonValueKind.Array)
|
||||
foreach (var v in values.EnumerateArray())
|
||||
if (v.ValueKind == JsonValueKind.String && v.GetString() is { Length: > 0 } s)
|
||||
ops.Add(s);
|
||||
result[tool.Name] = ops;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,530 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using ClawdDotNet.Core.Audit;
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Core.Inspection;
|
||||
|
||||
/// <summary>A run picked for the lead to look at, with what the mechanical check found in it.</summary>
|
||||
public sealed record SpotCheck(
|
||||
long Id,
|
||||
string UnitId,
|
||||
string Lead,
|
||||
string AgentId,
|
||||
string RunId,
|
||||
string Why,
|
||||
IReadOnlyList<InspectionFinding> Findings,
|
||||
DateTime CreatedAt,
|
||||
string? Verdict,
|
||||
string? Note,
|
||||
DateTime? ReviewedAt,
|
||||
string? TaskId)
|
||||
{
|
||||
public bool Open => Verdict is null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #693: What follows from a finding — graded, not a switch. The first four are personnel
|
||||
/// matters; <see cref="Security"/> is not one: an agent that may be steered from outside is
|
||||
/// not helped by a better model.
|
||||
/// </summary>
|
||||
public enum MeasureKind
|
||||
{
|
||||
/// <summary>The task was unclear — the most frequent case (#663).</summary>
|
||||
SharpenTask,
|
||||
|
||||
/// <summary>The model is overtaxed: suitability probe (#681), then another model.</summary>
|
||||
ChangeModel,
|
||||
|
||||
/// <summary>Rights too wide or too narrow (#641).</summary>
|
||||
AdjustRights,
|
||||
|
||||
/// <summary>Repeated failures despite everything: pause the agent, hand its tasks on.</summary>
|
||||
Suspend,
|
||||
|
||||
/// <summary>Suspected manipulation from outside — for the security side, not for personnel.</summary>
|
||||
Security
|
||||
}
|
||||
|
||||
public sealed record Measure(
|
||||
long Id,
|
||||
long? CheckId,
|
||||
string UnitId,
|
||||
string ProposedBy,
|
||||
string AgentId,
|
||||
MeasureKind Kind,
|
||||
string Reasoning,
|
||||
DateTime ProposedAt,
|
||||
string Status,
|
||||
string? DecidedBy,
|
||||
DateTime? DecidedAt,
|
||||
string? Note,
|
||||
string? TaskId);
|
||||
|
||||
public sealed record SpotCheckOutcome(bool Ok, string Message);
|
||||
|
||||
/// <summary>#693: The lead's side of the spot checks, as the Escalation tool sees it.</summary>
|
||||
public interface ISpotChecks
|
||||
{
|
||||
/// <summary>Whether <paramref name="lead"/> is the one <paramref name="agentId"/> answers to.</summary>
|
||||
bool Leads(string lead, string agentId);
|
||||
|
||||
Task<RunInspection?> InspectAsync(string runId, CancellationToken ct);
|
||||
|
||||
Task<IReadOnlyList<SpotCheck>> PendingForAsync(string lead, CancellationToken ct);
|
||||
|
||||
Task<SpotCheckOutcome> CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct);
|
||||
|
||||
Task<SpotCheckOutcome> ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct);
|
||||
|
||||
Task<IReadOnlyList<Measure>> MeasuresByAsync(string lead, int days, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// #693: A lead does not only delegate — it looks. This picks runs of the agents a lead leads,
|
||||
/// holds each against the audit (<see cref="RunInspector"/>) and gives the lead one task to
|
||||
/// look at them: the mechanical part finds what does not fit, the lead judges whether the
|
||||
/// result answers the task.
|
||||
///
|
||||
/// <para><b>Which runs.</b> A share per unit (<see cref="UnitInspection"/>), denser for a new
|
||||
/// agent, a new model or the run after a failure — and capped per day, because every check
|
||||
/// costs the lead's run. Picked by the run id, not by chance: the same run gets the same
|
||||
/// answer on every pass.</para>
|
||||
///
|
||||
/// <para><b>What follows.</b> The lead proposes a measure with its reasons; a human decides.
|
||||
/// A model that pauses or reconfigures another changes the organisation without anyone
|
||||
/// deciding it — nothing here carries a measure out.</para>
|
||||
/// </summary>
|
||||
public sealed class SpotChecks(
|
||||
SqliteStorage storage,
|
||||
IAuditRepository audit,
|
||||
RunInspector inspector,
|
||||
Func<OrgChart> organisation,
|
||||
TaskboardService? board,
|
||||
ILogger logger,
|
||||
TimeProvider? clock = null) : ISpotChecks, IAsyncDisposable
|
||||
{
|
||||
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(15);
|
||||
|
||||
/// <summary>How far back a pass looks for an agent's earlier runs.</summary>
|
||||
public const int HistoryDays = 30;
|
||||
|
||||
/// <summary>Statuses where no run took place — nothing to check.</summary>
|
||||
private static readonly HashSet<string> NotRun = new(StringComparer.OrdinalIgnoreCase) { "BudgetExceeded", "TargetBusy", "Cancelled" };
|
||||
|
||||
private static readonly JsonSerializerOptions Json = new() { Converters = { new System.Text.Json.Serialization.JsonStringEnumConverter() } };
|
||||
|
||||
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private DateTime _seenUntil = DateTime.MinValue;
|
||||
private Task? _loop;
|
||||
|
||||
public bool Leads(string lead, string agentId)
|
||||
=> !string.Equals(lead, agentId, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(organisation().EffectiveLeaderOf(agentId)?.AgentId, lead, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
public Task<RunInspection?> InspectAsync(string runId, CancellationToken ct) => inspector.InspectAsync(runId, ct);
|
||||
|
||||
/// <summary>
|
||||
/// Why this run is checked — or <c>null</c> if it is not. <paramref name="earlier"/> are the
|
||||
/// agent's runs before this one, oldest first.
|
||||
/// </summary>
|
||||
public static string? Pick(RunReceipt run, IReadOnlyList<RunReceipt> earlier, UnitInspection settings)
|
||||
{
|
||||
string? dense = null;
|
||||
if (earlier.Count == 0 || earlier[0].OccurredAt >= run.OccurredAt.AddDays(-Math.Max(0, settings.NewDays)))
|
||||
dense = "new agent";
|
||||
else if (!earlier.Any(r => string.Equals(r.Model, run.Model, StringComparison.OrdinalIgnoreCase)))
|
||||
dense = "new model";
|
||||
else if (earlier.LastOrDefault(r => !NotRun.Contains(r.Status)) is { } previous
|
||||
&& !string.Equals(previous.Status, "Completed", StringComparison.OrdinalIgnoreCase))
|
||||
dense = "after a failure";
|
||||
|
||||
var rate = Math.Clamp(dense is null ? settings.Rate : settings.DenseRate, 0, 1);
|
||||
return Fraction(run.RunId) < rate ? dense ?? "sample" : null;
|
||||
}
|
||||
|
||||
/// <summary>A number in [0, 1) from the run id — the same for the same run, spread like chance.</summary>
|
||||
internal static double Fraction(string runId)
|
||||
{
|
||||
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(runId));
|
||||
return BitConverter.ToUInt32(hash, 0) / 4294967296.0;
|
||||
}
|
||||
|
||||
/// <summary>One pass: picks among the runs since the last pass. Returns the checks it created.</summary>
|
||||
public async Task<IReadOnlyList<SpotCheck>> SampleAsync(CancellationToken ct)
|
||||
{
|
||||
var org = organisation();
|
||||
if (!org.HasUnits || org.Units.All(u => u.Inspection is null))
|
||||
return [];
|
||||
|
||||
var now = _clock.GetUtcNow().UtcDateTime;
|
||||
var from = _seenUntil == DateTime.MinValue ? now.AddDays(-1) : _seenUntil;
|
||||
var fresh = await audit.ListReceiptsAsync(from, now, 5_000, ct);
|
||||
if (fresh.Count == 0)
|
||||
return [];
|
||||
_seenUntil = fresh[^1].OccurredAt.AddTicks(1);
|
||||
|
||||
var history = (await audit.ListReceiptsAsync(now.AddDays(-HistoryDays), now, 100_000, ct))
|
||||
.GroupBy(r => r.AgentId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var today = DateOnly.FromDateTime(_clock.GetLocalNow().DateTime);
|
||||
var usedToday = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
|
||||
var created = new List<SpotCheck>();
|
||||
|
||||
foreach (var run in fresh)
|
||||
{
|
||||
if (NotRun.Contains(run.Status) || org.EffectiveLeaderOf(run.AgentId) is not { } lead
|
||||
|| org.LedBy(lead.AgentId) is not { Inspection: { } settings } unit)
|
||||
continue;
|
||||
|
||||
var earlier = history.GetValueOrDefault(run.AgentId)?.Where(r => r.OccurredAt < run.OccurredAt && r.RunId != run.RunId).ToList() ?? [];
|
||||
if (Pick(run, earlier, settings) is not { } why)
|
||||
continue;
|
||||
|
||||
if (!usedToday.TryGetValue(unit.Id, out var used))
|
||||
used = usedToday[unit.Id] = await CountSinceAsync(unit.Id, today, ct);
|
||||
if (used >= Math.Max(0, settings.PerDay))
|
||||
continue;
|
||||
|
||||
if (await inspector.InspectAsync(run.RunId, ct) is not { } inspection)
|
||||
continue;
|
||||
if (await InsertAsync(unit.Id, lead.AgentId, run, why, inspection.Findings, now, ct) is { } check)
|
||||
{
|
||||
usedToday[unit.Id] = used + 1;
|
||||
created.Add(check);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var lead in created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase))
|
||||
await GiveTaskAsync(lead, ct);
|
||||
|
||||
if (created.Count > 0)
|
||||
logger.LogInformation("Spot checks: {Count} run(s) picked for {Leads}", created.Count,
|
||||
string.Join(", ", created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase)));
|
||||
return created;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One task per lead at a time, for all its open checks — every check costs the lead a look,
|
||||
/// but not a run each.
|
||||
/// </summary>
|
||||
private async Task GiveTaskAsync(string lead, CancellationToken ct)
|
||||
{
|
||||
var open = await PendingForAsync(lead, ct);
|
||||
var untasked = open.Where(c => c.TaskId is null).ToList();
|
||||
if (untasked.Count == 0 || board is null)
|
||||
return;
|
||||
|
||||
foreach (var taskId in open.Select(c => c.TaskId).OfType<string>().Distinct())
|
||||
{
|
||||
if (await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled or TaskItemStatus.Archived) })
|
||||
{
|
||||
await SetTaskAsync(untasked, taskId, ct);
|
||||
await board.AddCommentAsync(taskId, "system", $"{untasked.Count} more run(s) to check.", ct);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
var task = await board.CreateAsync(new TaskItem
|
||||
{
|
||||
Title = $"Spot check: {open.Count} run(s) of the agents you lead",
|
||||
Body = """
|
||||
Look at whether the agents you lead work properly. Not by asking them — by holding
|
||||
each result against its task and against what the engine recorded.
|
||||
|
||||
1. Escalation spot_checks lists the runs, with what the audit log shows.
|
||||
2. Escalation inspect_run (run) shows one run in full: task, calls, result.
|
||||
3. Judge: does the result answer the task, and do the calls back it up?
|
||||
4. Escalation check_done (check, verdict ok|finding, note) for each.
|
||||
|
||||
If something follows from it, Escalation propose_measure — a human decides:
|
||||
sharpen_task (unclear task — the most frequent case), change_model, adjust_rights,
|
||||
suspend, or security (suspected manipulation from outside: not a personnel matter).
|
||||
""",
|
||||
Assignee = "@" + lead,
|
||||
Priority = 3,
|
||||
Acceptance = "Every listed check is closed with a verdict."
|
||||
}, ct);
|
||||
await SetTaskAsync(open, task.Id, ct);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<SpotCheck>> PendingForAsync(string lead, CancellationToken ct)
|
||||
=> await QueryChecksAsync("Lead = @p AND Verdict IS NULL", lead, 50, ct);
|
||||
|
||||
public async Task<IReadOnlyList<SpotCheck>> ForUnitAsync(string unitId, int limit, CancellationToken ct)
|
||||
=> await QueryChecksAsync("UnitId = @p", unitId, limit, ct);
|
||||
|
||||
public async Task<SpotCheckOutcome> CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct)
|
||||
{
|
||||
var check = (await QueryChecksAsync("Id = @p", checkId.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault();
|
||||
if (check is null || !string.Equals(check.Lead, lead, StringComparison.OrdinalIgnoreCase))
|
||||
return new(false, $"No check {checkId} waits for you.");
|
||||
if (!check.Open)
|
||||
return new(false, $"Check {checkId} is closed already ({check.Verdict}).");
|
||||
if (finding && string.IsNullOrWhiteSpace(note))
|
||||
return new(false, "'note' is required for a finding — what did not fit, with the evidence.");
|
||||
|
||||
var verdict = finding ? "finding" : "ok";
|
||||
await storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "UPDATE SpotChecks SET Verdict = @v, Note = @n, ReviewedAt = @at WHERE Id = @id AND Verdict IS NULL";
|
||||
cmd.Parameters.AddWithValue("@v", verdict);
|
||||
cmd.Parameters.AddWithValue("@n", (note ?? "").Trim());
|
||||
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
|
||||
cmd.Parameters.AddWithValue("@id", checkId);
|
||||
return await cmd.ExecuteNonQueryAsync(ct);
|
||||
}, ct);
|
||||
|
||||
// The last check of a task closes the task.
|
||||
if (check.TaskId is { } taskId && board is not null
|
||||
&& (await PendingForAsync(lead, ct)).All(c => c.TaskId != taskId)
|
||||
&& await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) })
|
||||
await board.UpdateAsync(taskId, t => t with { Status = TaskItemStatus.Done }, ct);
|
||||
|
||||
return new(true, finding
|
||||
? $"Check {checkId} closed with a finding. If something should follow, propose_measure — a human decides."
|
||||
: $"Check {checkId} closed: in order.");
|
||||
}
|
||||
|
||||
public async Task<SpotCheckOutcome> ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct)
|
||||
{
|
||||
agentId = (agentId ?? "").Trim().TrimStart('@');
|
||||
if (!Leads(lead, agentId))
|
||||
return new(false, $"You do not lead '{agentId}' — propose measures only for the agents that answer to you.");
|
||||
if (string.IsNullOrWhiteSpace(reasoning))
|
||||
return new(false, "'reasoning' is required — what you found, with the evidence, and why this measure.");
|
||||
if (organisation().LedBy(lead) is not { } unit)
|
||||
return new(false, "Only a lead proposes measures.");
|
||||
|
||||
var open = await QueryMeasuresAsync("ProposedBy = @p AND Status = 'proposed'", lead, 200, ct);
|
||||
if (open.Any(m => m.Kind == kind && string.Equals(m.AgentId, agentId, StringComparison.OrdinalIgnoreCase)))
|
||||
return new(false, $"A {Label(kind)} for '{agentId}' waits for a human already.");
|
||||
|
||||
string? taskId = null;
|
||||
if (board is not null)
|
||||
taskId = (await board.CreateAsync(new TaskItem
|
||||
{
|
||||
Title = kind == MeasureKind.Security
|
||||
? $"Security suspicion: {agentId} (from its lead {lead})"
|
||||
: $"Proposed measure for {agentId}: {Label(kind)}",
|
||||
Body = (kind == MeasureKind.Security
|
||||
? "Not a personnel matter: the lead suspects the agent is steered from outside. A better model does not help here — consider pausing it and looking at its runs and the content it read.\n\n"
|
||||
: $"The lead {lead} proposes this after a spot check. Nothing has been changed — decide it in the organisation view.\n\n")
|
||||
+ $"Reasoning of {lead}:\n{reasoning.Trim()}"
|
||||
+ (checkId is { } c ? $"\n\nSpot check: {c}" : ""),
|
||||
Assignee = TaskAssignee.Human,
|
||||
Priority = kind == MeasureKind.Security ? 5 : 3
|
||||
}, ct)).Id;
|
||||
|
||||
var id = await storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT INTO SpotMeasures (CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, TaskId)
|
||||
VALUES (@check, @unit, @by, @agent, @kind, @reasoning, @at, 'proposed', @task)
|
||||
RETURNING Id
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@check", (object?)checkId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@unit", unit.Id);
|
||||
cmd.Parameters.AddWithValue("@by", lead);
|
||||
cmd.Parameters.AddWithValue("@agent", agentId);
|
||||
cmd.Parameters.AddWithValue("@kind", kind.ToString());
|
||||
cmd.Parameters.AddWithValue("@reasoning", reasoning.Trim());
|
||||
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
|
||||
cmd.Parameters.AddWithValue("@task", (object?)taskId ?? DBNull.Value);
|
||||
return Convert.ToInt64(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture);
|
||||
}, ct);
|
||||
|
||||
logger.LogInformation("Measure {Id} ({Kind}) for {Agent} proposed by {Lead}", id, kind, agentId, lead);
|
||||
return new(true, kind == MeasureKind.Security
|
||||
? $"Security suspicion {id} for '{agentId}' is with a human. Do not act on the agent yourself."
|
||||
: $"Measure {id} ({Label(kind)}) for '{agentId}' is with a human. Nothing changes until it is decided.");
|
||||
}
|
||||
|
||||
/// <summary>A human's decision on a proposed measure. Accepting records it — carrying it out stays with the human.</summary>
|
||||
public async Task<SpotCheckOutcome> DecideMeasureAsync(long id, bool accept, string note, string decidedBy, CancellationToken ct)
|
||||
{
|
||||
var measure = (await QueryMeasuresAsync("Id = @p", id.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault();
|
||||
if (measure is null)
|
||||
return new(false, $"There is no measure {id}.");
|
||||
if (measure.Status != "proposed")
|
||||
return new(false, $"Measure {id} is decided already ({measure.Status}).");
|
||||
|
||||
var status = accept ? "accepted" : "dismissed";
|
||||
await storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "UPDATE SpotMeasures SET Status = @s, DecidedBy = @by, DecidedAt = @at, Note = @n WHERE Id = @id AND Status = 'proposed'";
|
||||
cmd.Parameters.AddWithValue("@s", status);
|
||||
cmd.Parameters.AddWithValue("@by", decidedBy);
|
||||
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
|
||||
cmd.Parameters.AddWithValue("@n", (note ?? "").Trim());
|
||||
cmd.Parameters.AddWithValue("@id", id);
|
||||
return await cmd.ExecuteNonQueryAsync(ct);
|
||||
}, ct);
|
||||
|
||||
if (measure.TaskId is { } taskId && board is not null
|
||||
&& await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) })
|
||||
{
|
||||
await board.AddCommentAsync(taskId, decidedBy, $"Measure {status}." + (string.IsNullOrWhiteSpace(note) ? "" : " " + note.Trim()), ct);
|
||||
await board.UpdateAsync(taskId, t => t with { Status = accept ? TaskItemStatus.Done : TaskItemStatus.Canceled }, ct);
|
||||
}
|
||||
return new(true, $"Measure {id} {status}.");
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<Measure>> MeasuresByAsync(string lead, int days, CancellationToken ct)
|
||||
=> QueryMeasuresAsync("ProposedBy = @p AND ProposedAt >= @since", lead, 50, ct, _clock.GetUtcNow().UtcDateTime.AddDays(-Math.Max(1, days)));
|
||||
|
||||
public Task<IReadOnlyList<Measure>> MeasuresForUnitAsync(string unitId, int limit, CancellationToken ct)
|
||||
=> QueryMeasuresAsync("UnitId = @p", unitId, limit, ct);
|
||||
|
||||
public static string Label(MeasureKind kind) => kind switch
|
||||
{
|
||||
MeasureKind.SharpenTask => "sharpen_task",
|
||||
MeasureKind.ChangeModel => "change_model",
|
||||
MeasureKind.AdjustRights => "adjust_rights",
|
||||
MeasureKind.Suspend => "suspend",
|
||||
_ => "security"
|
||||
};
|
||||
|
||||
public static MeasureKind? ParseKind(string? text) => (text ?? "").Trim().ToLowerInvariant() switch
|
||||
{
|
||||
"sharpen_task" => MeasureKind.SharpenTask,
|
||||
"change_model" => MeasureKind.ChangeModel,
|
||||
"adjust_rights" => MeasureKind.AdjustRights,
|
||||
"suspend" => MeasureKind.Suspend,
|
||||
"security" => MeasureKind.Security,
|
||||
_ => null
|
||||
};
|
||||
|
||||
// ─── storage ───
|
||||
|
||||
private async Task<int> CountSinceAsync(string unitId, DateOnly localDay, CancellationToken ct)
|
||||
{
|
||||
var since = new DateTimeOffset(localDay.ToDateTime(TimeOnly.MinValue), _clock.LocalTimeZone.GetUtcOffset(localDay.ToDateTime(TimeOnly.MinValue))).UtcDateTime;
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM SpotChecks WHERE UnitId = @unit AND CreatedAt >= @since";
|
||||
cmd.Parameters.AddWithValue("@unit", unitId);
|
||||
cmd.Parameters.AddWithValue("@since", since.ToString("O"));
|
||||
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private Task<SpotCheck?> InsertAsync(string unitId, string lead, RunReceipt run, string why, IReadOnlyList<InspectionFinding> findings, DateTime now, CancellationToken ct)
|
||||
=> storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT OR IGNORE INTO SpotChecks (UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt)
|
||||
VALUES (@unit, @lead, @agent, @run, @why, @findings, @at)
|
||||
RETURNING Id
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@unit", unitId);
|
||||
cmd.Parameters.AddWithValue("@lead", lead);
|
||||
cmd.Parameters.AddWithValue("@agent", run.AgentId);
|
||||
cmd.Parameters.AddWithValue("@run", run.RunId);
|
||||
cmd.Parameters.AddWithValue("@why", why);
|
||||
cmd.Parameters.AddWithValue("@findings", JsonSerializer.Serialize(findings, Json));
|
||||
cmd.Parameters.AddWithValue("@at", now.ToString("O"));
|
||||
var id = await cmd.ExecuteScalarAsync(ct);
|
||||
return id is null or DBNull
|
||||
? null
|
||||
: new SpotCheck(Convert.ToInt64(id, CultureInfo.InvariantCulture), unitId, lead, run.AgentId, run.RunId, why, findings, now, null, null, null, null);
|
||||
}, ct);
|
||||
|
||||
private Task SetTaskAsync(IEnumerable<SpotCheck> checks, string taskId, CancellationToken ct)
|
||||
=> storage.WriteAsync(async conn =>
|
||||
{
|
||||
foreach (var check in checks)
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "UPDATE SpotChecks SET TaskId = @task WHERE Id = @id";
|
||||
cmd.Parameters.AddWithValue("@task", taskId);
|
||||
cmd.Parameters.AddWithValue("@id", check.Id);
|
||||
await cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
return 0;
|
||||
}, ct);
|
||||
|
||||
private async Task<IReadOnlyList<SpotCheck>> QueryChecksAsync(string where, string value, int limit, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = $"SELECT Id, UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt, Verdict, Note, ReviewedAt, TaskId FROM SpotChecks WHERE {where} ORDER BY Id DESC LIMIT @limit";
|
||||
cmd.Parameters.AddWithValue("@p", value);
|
||||
cmd.Parameters.AddWithValue("@limit", limit);
|
||||
var result = new List<SpotCheck>();
|
||||
await using var r = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await r.ReadAsync(ct))
|
||||
result.Add(new SpotCheck(r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3), r.GetString(4), r.GetString(5),
|
||||
JsonSerializer.Deserialize<List<InspectionFinding>>(r.GetString(6), Json) ?? [],
|
||||
Time(r, 7)!.Value, Text(r, 8), Text(r, 9), Time(r, 10), Text(r, 11)));
|
||||
return result;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<Measure>> QueryMeasuresAsync(string where, string value, int limit, CancellationToken ct, DateTime? since = null)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = $"SELECT Id, CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, DecidedBy, DecidedAt, Note, TaskId FROM SpotMeasures WHERE {where} ORDER BY Id DESC LIMIT @limit";
|
||||
cmd.Parameters.AddWithValue("@p", value);
|
||||
cmd.Parameters.AddWithValue("@limit", limit);
|
||||
if (since is { } s)
|
||||
cmd.Parameters.AddWithValue("@since", s.ToString("O"));
|
||||
var result = new List<Measure>();
|
||||
await using var r = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await r.ReadAsync(ct))
|
||||
result.Add(new Measure(r.GetInt64(0), r.IsDBNull(1) ? null : r.GetInt64(1), r.GetString(2), r.GetString(3), r.GetString(4),
|
||||
Enum.TryParse<MeasureKind>(r.GetString(5), out var k) ? k : MeasureKind.SharpenTask,
|
||||
r.GetString(6), Time(r, 7)!.Value, r.GetString(8), Text(r, 9), Time(r, 10), Text(r, 11), Text(r, 12)));
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string? Text(SqliteDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i);
|
||||
|
||||
private static DateTime? Time(SqliteDataReader r, int i)
|
||||
=> r.IsDBNull(i) ? null : DateTime.Parse(r.GetString(i), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
|
||||
|
||||
// ─── loop ───
|
||||
|
||||
public void Start() => _loop ??= RunAsync(_cts.Token);
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try { await Task.Delay(TimeSpan.FromMinutes(2), ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await SampleAsync(ct); }
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(ex, "Spot checks failed — next attempt in {Minutes} min", Interval.TotalMinutes);
|
||||
}
|
||||
|
||||
try { await Task.Delay(Interval, ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
if (_loop is not null)
|
||||
{
|
||||
try { await _loop; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -307,6 +307,41 @@ public sealed class SqliteStorage
|
||||
UNIQUE (Period, Scope, Kind)
|
||||
);
|
||||
|
||||
-- #693: runs picked for a lead to look at, and the measures a lead proposed.
|
||||
CREATE TABLE IF NOT EXISTS SpotChecks (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
UnitId TEXT NOT NULL,
|
||||
Lead TEXT NOT NULL,
|
||||
AgentId TEXT NOT NULL,
|
||||
RunId TEXT NOT NULL UNIQUE,
|
||||
Why TEXT NOT NULL,
|
||||
Findings TEXT NOT NULL,
|
||||
CreatedAt TEXT NOT NULL,
|
||||
Verdict TEXT NULL,
|
||||
Note TEXT NULL,
|
||||
ReviewedAt TEXT NULL,
|
||||
TaskId TEXT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS IX_SpotChecks_Lead ON SpotChecks (Lead, Verdict);
|
||||
CREATE INDEX IF NOT EXISTS IX_SpotChecks_Unit ON SpotChecks (UnitId, CreatedAt);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS SpotMeasures (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
CheckId INTEGER NULL,
|
||||
UnitId TEXT NOT NULL,
|
||||
ProposedBy TEXT NOT NULL,
|
||||
AgentId TEXT NOT NULL,
|
||||
Kind TEXT NOT NULL,
|
||||
Reasoning TEXT NOT NULL,
|
||||
ProposedAt TEXT NOT NULL,
|
||||
Status TEXT NOT NULL,
|
||||
DecidedBy TEXT NULL,
|
||||
DecidedAt TEXT NULL,
|
||||
Note TEXT NULL,
|
||||
TaskId TEXT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS StagedCalls (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
RunId TEXT NOT NULL,
|
||||
|
||||
@@ -55,6 +55,9 @@ public sealed record AgentToolContext(
|
||||
/// <summary>#698: The units' doctrines. <c>null</c> outside the engine.</summary>
|
||||
public Organisation.IDoctrineStore? Doctrines { get; init; }
|
||||
|
||||
/// <summary>#693: The spot checks of a lead. <c>null</c> outside the engine.</summary>
|
||||
public Inspection.ISpotChecks? SpotChecks { get; init; }
|
||||
|
||||
/// <summary>#654: Where code runs — never in the main process. Unavailable under Windows.</summary>
|
||||
public Sandbox.ISandboxRunner? Sandbox { get; init; }
|
||||
|
||||
|
||||
@@ -2637,6 +2637,28 @@ button.org-einheit .org-name {
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
/* #693: the lead's spot checks and the measures waiting for the human. */
|
||||
.org-stichproben h4 {
|
||||
margin: 14px 0 6px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.org-massnahme {
|
||||
border: 1px solid var(--warnung);
|
||||
border-left-width: 4px;
|
||||
padding: 8px 12px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.org-massnahme.sicherheit {
|
||||
border-color: var(--gefahr);
|
||||
}
|
||||
|
||||
.org-befunde {
|
||||
margin: 0;
|
||||
padding-left: 16px;
|
||||
}
|
||||
|
||||
tr.org-ausreisser td {
|
||||
background: rgba(181, 130, 42, 0.08);
|
||||
}
|
||||
|
||||
@@ -105,6 +105,39 @@
|
||||
"org.doctrinePlaceholder": "Jeder Lieferantenkontakt wird in der Datenbank erfasst, bevor er weitergegeben wird.",
|
||||
"org.doctrineFrequent": "In den letzten 7 Tagen {count}-mal geändert — eine Doktrin, die oft wechselt, ist Mikromanagement mit Tokenkosten.",
|
||||
"org.doctrineRemoved": "(entfernt)",
|
||||
"org.spotChecks": "Stichproben der Leitung",
|
||||
"org.spotChecksHint": "Wie oft die Leitung dieser Einheit in die Läufe der Agenten sieht, die sie führt — gegen den Auftrag und gegen das, was die Engine aufgezeichnet hat, nie durch Nachfragen beim Agenten. Dichter bei neuen Agenten, neuen Modellen und nach einem Fehlschlag. Jede Stichprobe kostet die Leitung einen Blick: die Obergrenze niedrig halten. Maßnahmen, die die Leitung vorschlägt, entscheidest du.",
|
||||
"org.spotChecksEnabled": "Stichproben an",
|
||||
"org.spotChecksRate": "Anteil der Läufe (%)",
|
||||
"org.spotChecksDense": "Anteil bei Neuem oder nach Fehlschlag (%)",
|
||||
"org.spotChecksNewDays": "Tage, die ein Agent als neu gilt",
|
||||
"org.spotChecksPerDay": "Höchstens Stichproben je Tag",
|
||||
"org.spotChecksRecent": "Letzte Stichproben",
|
||||
"org.spotChecksNone": "Noch kein Lauf geprüft.",
|
||||
"org.spotCheckRun": "Lauf",
|
||||
"org.spotCheckWhy": "Anlass",
|
||||
"org.spotCheckFindings": "Befund im Audit",
|
||||
"org.spotCheckVerdict": "Urteil der Leitung",
|
||||
"org.why.sample": "Stichprobe",
|
||||
"org.why.new_agent": "neuer Agent",
|
||||
"org.why.new_model": "neues Modell",
|
||||
"org.why.after_a_failure": "nach Fehlschlag",
|
||||
"org.verdict.ok": "in Ordnung",
|
||||
"org.verdict.finding": "Befund",
|
||||
"org.verdictOpen": "offen bei {lead}",
|
||||
"org.measuresOpen": "Maßnahmen, die auf dich warten",
|
||||
"org.measuresDecided": "Entschiedene Maßnahmen",
|
||||
"org.measureBy": "vorgeschlagen von {lead}",
|
||||
"org.measureNote": "Notiz (optional)",
|
||||
"org.measureAccept": "Annehmen",
|
||||
"org.measureDismiss": "Ablehnen",
|
||||
"org.measure.sharpen_task": "Auftrag schärfen",
|
||||
"org.measure.change_model": "Modell wechseln",
|
||||
"org.measure.adjust_rights": "Rechte anpassen",
|
||||
"org.measure.suspend": "Agent stilllegen",
|
||||
"org.measure.security": "Sicherheitsverdacht",
|
||||
"org.measureStatus.accepted": "angenommen",
|
||||
"org.measureStatus.dismissed": "abgelehnt",
|
||||
"org.ceilingInherited": "Von oben begrenzt auf: {tools}",
|
||||
"org.settings": "Sammelwerte",
|
||||
"org.settingsHint": "Werden nach einer Vorschau in die Agenten geschrieben. Die Einheit merkt sich den Wert nur als Vorgabe für neue Mitglieder — zur Laufzeit übertrumpft nichts einen Agenten.",
|
||||
|
||||
@@ -105,6 +105,39 @@
|
||||
"org.doctrinePlaceholder": "Every supplier contact is entered in the database before it is passed on.",
|
||||
"org.doctrineFrequent": "Changed {count} times in the last 7 days — a doctrine that changes often is micro-management with token costs.",
|
||||
"org.doctrineRemoved": "(removed)",
|
||||
"org.spotChecks": "Spot checks by the lead",
|
||||
"org.spotChecksHint": "How often the lead of this unit looks at the runs of the agents it leads — held against the task and against what the engine recorded, never by asking the agent. Denser for new agents, new models and after a failure. Every check costs the lead a look: keep the cap low. Measures the lead proposes are decided by you.",
|
||||
"org.spotChecksEnabled": "Spot checks on",
|
||||
"org.spotChecksRate": "Share of runs (%)",
|
||||
"org.spotChecksDense": "Share when new or after a failure (%)",
|
||||
"org.spotChecksNewDays": "Days an agent counts as new",
|
||||
"org.spotChecksPerDay": "Checks per day at most",
|
||||
"org.spotChecksRecent": "Recent checks",
|
||||
"org.spotChecksNone": "No run checked yet.",
|
||||
"org.spotCheckRun": "Run",
|
||||
"org.spotCheckWhy": "Why",
|
||||
"org.spotCheckFindings": "Found in the audit log",
|
||||
"org.spotCheckVerdict": "Verdict of the lead",
|
||||
"org.why.sample": "sample",
|
||||
"org.why.new_agent": "new agent",
|
||||
"org.why.new_model": "new model",
|
||||
"org.why.after_a_failure": "after a failure",
|
||||
"org.verdict.ok": "in order",
|
||||
"org.verdict.finding": "finding",
|
||||
"org.verdictOpen": "open with {lead}",
|
||||
"org.measuresOpen": "Measures waiting for you",
|
||||
"org.measuresDecided": "Decided measures",
|
||||
"org.measureBy": "proposed by {lead}",
|
||||
"org.measureNote": "Note (optional)",
|
||||
"org.measureAccept": "Accept",
|
||||
"org.measureDismiss": "Dismiss",
|
||||
"org.measure.sharpen_task": "Sharpen the task",
|
||||
"org.measure.change_model": "Change the model",
|
||||
"org.measure.adjust_rights": "Adjust the rights",
|
||||
"org.measure.suspend": "Pause the agent",
|
||||
"org.measure.security": "Security suspicion",
|
||||
"org.measureStatus.accepted": "accepted",
|
||||
"org.measureStatus.dismissed": "dismissed",
|
||||
"org.ceilingInherited": "Limited from above to: {tools}",
|
||||
"org.settings": "Unit-wide values",
|
||||
"org.settingsHint": "Written into the agents after a preview. The unit keeps the value only as a default for new members — nothing overrides an agent at run time.",
|
||||
|
||||
@@ -368,6 +368,7 @@ function unitPanel(unit) {
|
||||
cards.push(ceilingCard(unit));
|
||||
cards.push(approvalsCard(unit));
|
||||
cards.push(doctrineCard(unit));
|
||||
cards.push(inspectionCard(unit));
|
||||
cards.push(settingsCard(unit));
|
||||
|
||||
// Members.
|
||||
@@ -503,6 +504,101 @@ function doctrineCard(unit) {
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* #693: The lead's spot checks. How often it looks at the runs of the agents it leads, what
|
||||
* came of the last checks, and the measures it proposed — which only a human decides. Nothing
|
||||
* here changes an agent; accepting records the decision, carrying it out stays with you.
|
||||
*/
|
||||
function inspectionCard(unit) {
|
||||
const current = unit.inspection;
|
||||
const enabled = erstelleElement("input", { type: "checkbox" });
|
||||
enabled.checked = !!current;
|
||||
const percent = (v) => String(Math.round((v ?? 0) * 100));
|
||||
const rate = erstelleElement("input", { type: "number", min: "0", max: "100", step: "1", value: percent(current?.rate ?? 0.1) });
|
||||
const dense = erstelleElement("input", { type: "number", min: "0", max: "100", step: "1", value: percent(current?.denseRate ?? 0.5) });
|
||||
const newDays = erstelleElement("input", { type: "number", min: "0", max: "365", step: "1", value: String(current?.newDays ?? 7) });
|
||||
const perDay = erstelleElement("input", { type: "number", min: "0", max: "100", step: "1", value: String(current?.perDay ?? 5) });
|
||||
const results = erstelleElement("div", { klasse: "org-stichproben" });
|
||||
|
||||
(async () => {
|
||||
let data;
|
||||
try { data = await api(`/organisation/units/${encodeURIComponent(unit.id)}/spot-checks`); } catch { return; }
|
||||
|
||||
const proposed = data.measures.filter((m) => m.status === "proposed");
|
||||
if (proposed.length) {
|
||||
results.append(erstelleElement("h4", { text: t("org.measuresOpen") }));
|
||||
for (const m of proposed) {
|
||||
const note = erstelleElement("input", { type: "text", placeholder: t("org.measureNote") });
|
||||
results.append(erstelleElement("div", { klasse: `org-massnahme${m.kind === "security" ? " sicherheit" : ""}` },
|
||||
erstelleElement("div", {},
|
||||
erstellePlakette(t(`org.measure.${m.kind}`), m.kind === "security" ? "gefahr" : "warnung"),
|
||||
erstelleElement("strong", { text: ` ${m.agentId}` }),
|
||||
erstelleElement("span", { klasse: "gedaempft text-klein", text: ` · ${t("org.measureBy", { lead: m.proposedBy })} · ${formatiereDatumZeit(m.proposedAt)}` })),
|
||||
erstelleElement("p", { klasse: "text-klein", text: m.reasoning }),
|
||||
erstelleElement("div", { klasse: "gruppe-horizontal" }, note,
|
||||
erstelleElement("button", {
|
||||
type: "button", klasse: "primaer klein", text: t("org.measureAccept"),
|
||||
onclick: () => send(`/organisation/measures/${m.id}/decide`, "POST", { accept: true, note: note.value })
|
||||
}),
|
||||
erstelleElement("button", {
|
||||
type: "button", klasse: "klein", text: t("org.measureDismiss"),
|
||||
onclick: () => send(`/organisation/measures/${m.id}/decide`, "POST", { accept: false, note: note.value })
|
||||
}))));
|
||||
}
|
||||
}
|
||||
|
||||
results.append(erstelleElement("h4", { text: t("org.spotChecksRecent") }));
|
||||
if (!data.checks.length) {
|
||||
results.append(erstelleElement("p", { klasse: "gedaempft text-klein", text: t("org.spotChecksNone") }));
|
||||
} else {
|
||||
results.append(erstelleElement("table", { klasse: "daten-tabelle" },
|
||||
erstelleElement("thead", {}, erstelleElement("tr", {},
|
||||
...[t("org.agent"), t("org.spotCheckRun"), t("org.spotCheckWhy"), t("org.spotCheckFindings"), t("org.spotCheckVerdict")]
|
||||
.map((h) => erstelleElement("th", { text: h })))),
|
||||
erstelleElement("tbody", {}, ...data.checks.map((c) => erstelleElement("tr", {},
|
||||
erstelleElement("td", { text: c.agentId }),
|
||||
erstelleElement("td", {}, erstelleElement("a", { href: `#/verbrauch/runs/${encodeURIComponent(c.runId)}`, text: formatiereDatumZeit(c.createdAt) })),
|
||||
erstelleElement("td", { text: t(`org.why.${c.why.replace(/ /g, "_")}`) }),
|
||||
erstelleElement("td", {}, c.findings.length
|
||||
? erstelleElement("ul", { klasse: "org-befunde" }, ...c.findings.map((f) => erstelleElement("li", { klasse: "text-klein", text: f.text })))
|
||||
: erstelleElement("span", { klasse: "gedaempft", text: "—" })),
|
||||
erstelleElement("td", {}, c.verdict
|
||||
? erstellePlakette(t(`org.verdict.${c.verdict}`), c.verdict === "ok" ? "erfolg" : "warnung")
|
||||
: erstelleElement("span", { klasse: "gedaempft text-klein", text: t("org.verdictOpen", { lead: c.lead }) }),
|
||||
c.note ? erstelleElement("div", { klasse: "text-klein", text: c.note }) : ""))))));
|
||||
}
|
||||
|
||||
const decided = data.measures.filter((m) => m.status !== "proposed").slice(0, 10);
|
||||
if (decided.length) {
|
||||
results.append(erstelleElement("h4", { text: t("org.measuresDecided") }));
|
||||
results.append(erstelleElement("ul", { klasse: "org-liste" }, ...decided.map((m) => erstelleElement("li", { klasse: "text-klein",
|
||||
text: `${t(`org.measure.${m.kind}`)} · ${m.agentId} · ${t(`org.measureStatus.${m.status}`)}${m.note ? " — " + m.note : ""}` }))));
|
||||
}
|
||||
})();
|
||||
|
||||
return erstelleKarte(t("org.spotChecks"), [
|
||||
erstelleElement("p", { klasse: "gedaempft text-klein", text: t("org.spotChecksHint") }),
|
||||
erstelleElement("div", { klasse: "formular-raster" },
|
||||
erstelleElement("label", { text: t("org.spotChecksEnabled") }), enabled,
|
||||
erstelleElement("label", { text: t("org.spotChecksRate") }), rate,
|
||||
erstelleElement("label", { text: t("org.spotChecksDense") }), dense,
|
||||
erstelleElement("label", { text: t("org.spotChecksNewDays") }), newDays,
|
||||
erstelleElement("label", { text: t("org.spotChecksPerDay") }), perDay),
|
||||
erstelleElement("button", {
|
||||
type: "button", klasse: "klein", text: t("common.save"),
|
||||
onclick: () => send(`/organisation/units/${encodeURIComponent(unit.id)}/inspection`, "PUT", {
|
||||
enabled: enabled.checked,
|
||||
rate: Number(rate.value) / 100,
|
||||
denseRate: Number(dense.value) / 100,
|
||||
newDays: Number(newDays.value),
|
||||
perDay: Number(perDay.value),
|
||||
reason: reason()
|
||||
})
|
||||
}),
|
||||
results
|
||||
]);
|
||||
}
|
||||
|
||||
function settingsCard(unit) {
|
||||
const setting = erstelleElement("select", {},
|
||||
...SETTINGS.map((s) => erstelleElement("option", { value: s, text: t(`org.setting.${s}`) })));
|
||||
|
||||
@@ -3,6 +3,7 @@ using System.Text.Json;
|
||||
using System.Globalization;
|
||||
using ClawdDotNet.Core.Budget;
|
||||
using ClawdDotNet.Core.Cases;
|
||||
using ClawdDotNet.Core.Inspection;
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
using ClawdDotNet.Core.Staging;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
@@ -36,6 +37,10 @@ public sealed class EscalationTool : IAgentTool
|
||||
- doctrine_read: the doctrine of your unit — what holds there now.
|
||||
As a lead: doctrine_set (text, reason) sets your unit's doctrine — short, changed seldom; empty text removes it. Every version is kept.
|
||||
As a lead whose unit may approve: list_approvals shows the calls waiting for you; approve_call (call) or reject_call (call, reason). Never your own calls; if unsure, leave it to the human.
|
||||
As a lead, look at how the agents you lead work — never by asking them, always against what the engine recorded:
|
||||
spot_checks lists the runs picked for you, with what the audit log shows; inspect_run (run) shows one run in full;
|
||||
check_done (check, verdict ok|finding, note) closes a check. propose_measure (agent, measure, reasoning, check) puts a measure to a human:
|
||||
sharpen_task, change_model, adjust_rights, suspend — or security, if you suspect the agent is steered from outside.
|
||||
""";
|
||||
|
||||
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
|
||||
@@ -44,7 +49,7 @@ public sealed class EscalationTool : IAgentTool
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["find_decisions", "escalate", "list_cases", "decide", "handle", "forward", "open_conflict", "state_position", "request_budget", "list_approvals", "approve_call", "reject_call", "doctrine_read", "doctrine_set"]
|
||||
"enum": ["find_decisions", "escalate", "list_cases", "decide", "handle", "forward", "open_conflict", "state_position", "request_budget", "list_approvals", "approve_call", "reject_call", "doctrine_read", "doctrine_set", "spot_checks", "inspect_run", "check_done", "propose_measure"]
|
||||
},
|
||||
"amount_usd": { "type": "number", "description": "request_budget: the amount for the task and all its parts, in USD." },
|
||||
"subject": { "type": "string", "description": "What it is about, in a few words — like a title. escalate, open_conflict, find_decisions." },
|
||||
@@ -62,7 +67,12 @@ public sealed class EscalationTool : IAgentTool
|
||||
"position": { "type": "string", "description": "open_conflict, state_position: your side, with reasons." },
|
||||
"call": { "type": "integer", "description": "approve_call, reject_call: the approval number (#…)." },
|
||||
"reason": { "type": "string", "description": "reject_call: why — the agent must know what to change. doctrine_set: why the doctrine changes." },
|
||||
"text": { "type": "string", "description": "doctrine_set: what holds in your unit now. At most 2000 characters; empty removes it." }
|
||||
"text": { "type": "string", "description": "doctrine_set: what holds in your unit now. At most 2000 characters; empty removes it." },
|
||||
"run": { "type": "string", "description": "inspect_run: the run id." },
|
||||
"check": { "type": "integer", "description": "check_done, propose_measure: the spot check number." },
|
||||
"verdict": { "type": "string", "enum": ["ok", "finding"], "description": "check_done: ok, or finding (then 'note' with the evidence)." },
|
||||
"agent": { "type": "string", "description": "propose_measure: the agent it concerns." },
|
||||
"measure": { "type": "string", "enum": ["sharpen_task", "change_model", "adjust_rights", "suspend", "security"], "description": "propose_measure: what should follow." }
|
||||
},
|
||||
"required": ["action"]
|
||||
}
|
||||
@@ -90,6 +100,10 @@ public sealed class EscalationTool : IAgentTool
|
||||
if (Get(input, "action") is "doctrine_read" or "doctrine_set")
|
||||
return await DoctrineAsync(input, context, ct);
|
||||
|
||||
// #693: A lead looking at the runs of the agents it leads.
|
||||
if (Get(input, "action") is "spot_checks" or "inspect_run" or "check_done" or "propose_measure")
|
||||
return await SpotCheckAsync(input, context, ct);
|
||||
|
||||
// #687: A lead deciding the approvals its unit grants it.
|
||||
if (Get(input, "action") is "list_approvals" or "approve_call" or "reject_call")
|
||||
return await ApprovalAsync(input, context, ct);
|
||||
@@ -162,7 +176,7 @@ public sealed class EscalationTool : IAgentTool
|
||||
break;
|
||||
|
||||
default:
|
||||
return ToolResult.Fail("'action' must be one of find_decisions, escalate, list_cases, decide, handle, forward, open_conflict, state_position, request_budget, list_approvals, approve_call, reject_call.");
|
||||
return ToolResult.Fail("'action' must be one of find_decisions, escalate, list_cases, decide, handle, forward, open_conflict, state_position, request_budget, list_approvals, approve_call, reject_call, doctrine_read, doctrine_set, spot_checks, inspect_run, check_done, propose_measure.");
|
||||
}
|
||||
|
||||
return outcome.Ok ? ToolResult.Ok(outcome.Message) : ToolResult.Fail(outcome.Message);
|
||||
@@ -244,6 +258,105 @@ public sealed class EscalationTool : IAgentTool
|
||||
return decided.Ok ? ToolResult.Ok(decided.Message) : ToolResult.Fail(decided.Message);
|
||||
}
|
||||
|
||||
private static async Task<ToolResult> SpotCheckAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||||
{
|
||||
if (context.SpotChecks is not { } checks)
|
||||
return ToolResult.Fail("Spot checks are not available in this instance.");
|
||||
var me = context.AgentId;
|
||||
long? checkId = input.TryGetProperty("check", out var c) && c.ValueKind == JsonValueKind.Number && c.TryGetInt64(out var n) ? n : null;
|
||||
|
||||
switch (Get(input, "action"))
|
||||
{
|
||||
case "spot_checks":
|
||||
{
|
||||
var pending = await checks.PendingForAsync(me, ct);
|
||||
var measures = (await checks.MeasuresByAsync(me, 14, ct)).Take(5).ToList();
|
||||
var sb = new StringBuilder();
|
||||
if (pending.Count == 0)
|
||||
sb.AppendLine("No spot check waits for you.");
|
||||
else
|
||||
{
|
||||
sb.AppendLine($"{pending.Count} run(s) to check — the findings come from the audit log, not from the agent:");
|
||||
foreach (var check in pending)
|
||||
{
|
||||
sb.AppendLine($"- check {check.Id}: {check.AgentId}, run {check.RunId} ({check.Why}, {check.CreatedAt:yyyy-MM-dd HH:mm} UTC)");
|
||||
if (check.Findings.Count == 0)
|
||||
sb.AppendLine(" nothing mechanical — judge whether the result answers the task (inspect_run).");
|
||||
foreach (var f in check.Findings)
|
||||
sb.AppendLine($" {f.Kind}: {f.Text}");
|
||||
}
|
||||
}
|
||||
if (measures.Count > 0)
|
||||
{
|
||||
sb.AppendLine("Your recent proposals:");
|
||||
foreach (var m in measures)
|
||||
sb.AppendLine($"- {m.Id} {SpotChecks.Label(m.Kind)} for {m.AgentId}: {m.Status}" + (string.IsNullOrWhiteSpace(m.Note) ? "" : $" — {m.Note}"));
|
||||
}
|
||||
return ToolResult.Ok(sb.ToString().TrimEnd());
|
||||
}
|
||||
|
||||
case "inspect_run":
|
||||
{
|
||||
var runId = (Get(input, "run") ?? "").Trim();
|
||||
if (runId.Length == 0)
|
||||
return ToolResult.Fail("'run' is required — the run id.");
|
||||
if (await checks.InspectAsync(runId, ct) is not { } run)
|
||||
return ToolResult.Fail($"No finished run '{runId}'.");
|
||||
if (!checks.Leads(me, run.Receipt.AgentId))
|
||||
return ToolResult.Fail($"Run '{runId}' belongs to {run.Receipt.AgentId}, whom you do not lead.");
|
||||
return ToolResult.Ok(Describe(run));
|
||||
}
|
||||
|
||||
case "check_done":
|
||||
{
|
||||
if (checkId is not { } id)
|
||||
return ToolResult.Fail("'check' is required — the spot check number.");
|
||||
var verdict = Get(input, "verdict");
|
||||
if (verdict is not ("ok" or "finding"))
|
||||
return ToolResult.Fail("'verdict' must be ok or finding.");
|
||||
var closed = await checks.CloseAsync(id, me, verdict == "finding", Get(input, "note") ?? "", ct);
|
||||
return closed.Ok ? ToolResult.Ok(closed.Message) : ToolResult.Fail(closed.Message);
|
||||
}
|
||||
|
||||
default:
|
||||
{
|
||||
if (SpotChecks.ParseKind(Get(input, "measure")) is not { } kind)
|
||||
return ToolResult.Fail("'measure' must be one of sharpen_task, change_model, adjust_rights, suspend, security.");
|
||||
var proposed = await checks.ProposeAsync(me, Get(input, "agent") ?? "", kind, Get(input, "reasoning") ?? "", checkId, ct);
|
||||
return proposed.Ok ? ToolResult.Ok(proposed.Message) : ToolResult.Fail(proposed.Message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One run as the engine recorded it. The result text is the agent's — marked as such.</summary>
|
||||
private static string Describe(RunInspection run)
|
||||
{
|
||||
var r = run.Receipt;
|
||||
var sb = new StringBuilder();
|
||||
sb.AppendLine($"Run {r.RunId} of {r.AgentId} ({r.Model}), {r.OccurredAt:yyyy-MM-dd HH:mm} UTC: {r.Status}, {r.StepCount} step(s), " +
|
||||
(r.CostIsKnown ? $"{r.CostUsd.ToString("0.0000", CultureInfo.InvariantCulture)} USD" : "cost unknown"));
|
||||
if (run.Task is { } task)
|
||||
{
|
||||
sb.AppendLine($"Task {task.Id}: {task.Title}");
|
||||
if (!string.IsNullOrWhiteSpace(task.Acceptance))
|
||||
sb.AppendLine($"Done when: {task.Acceptance}");
|
||||
}
|
||||
sb.AppendLine($"Calls recorded by the engine ({run.Calls.Count}):");
|
||||
foreach (var call in run.Calls.Take(40))
|
||||
sb.AppendLine($"- {call.At:HH:mm:ss} {call.Tool}{(call.Operation is { } op ? "." + op : "")}: {call.Status}" +
|
||||
(string.IsNullOrWhiteSpace(call.Summary) ? "" : $" — {Cap(call.Summary, 160)}"));
|
||||
if (run.Calls.Count > 40)
|
||||
sb.AppendLine($"- … and {run.Calls.Count - 40} more");
|
||||
sb.AppendLine(run.Findings.Count == 0 ? "Mechanical findings: none." : "Mechanical findings:");
|
||||
foreach (var f in run.Findings)
|
||||
sb.AppendLine($"- {f.Kind}: {f.Text}");
|
||||
sb.AppendLine("Result as the agent wrote it (its claim, not evidence):");
|
||||
sb.Append(string.IsNullOrWhiteSpace(run.Result) ? "(none)" : run.Result);
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static string Cap(string text, int max) => text.Length <= max ? text : text[..max] + "…";
|
||||
|
||||
private static async Task<ToolResult> DoctrineAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||||
{
|
||||
if (context.Doctrines is not { } doctrines || context.Organisation is not { HasUnits: true } org)
|
||||
|
||||
@@ -20,7 +20,9 @@ public sealed record UnitDto(
|
||||
string Path,
|
||||
int Depth,
|
||||
// #687: what the lead may approve instead of the human; null: nothing.
|
||||
UnitApprovalsDto? Approvals = null);
|
||||
UnitApprovalsDto? Approvals = null,
|
||||
// #693: the lead's spot checks; null: none.
|
||||
UnitInspectionDto? Inspection = null);
|
||||
|
||||
/// <summary>#698: One version of a unit's doctrine.</summary>
|
||||
public sealed record DoctrineDto(int Version, string Text, string SetBy, DateTime SetAt, string? RunId, string Reason);
|
||||
@@ -31,6 +33,27 @@ public sealed record UnitApprovalsDto(IReadOnlyList<string> Operations, int PerD
|
||||
|
||||
public sealed record ApprovalsInputDto(IReadOnlyList<string>? Operations, int? PerDay, double? Hours, string Reason = "");
|
||||
|
||||
/// <summary>#693: How often the lead looks at the runs of the agents it leads.</summary>
|
||||
public sealed record UnitInspectionDto(double Rate, double DenseRate, int NewDays, int PerDay);
|
||||
|
||||
public sealed record InspectionInputDto(bool Enabled, double? Rate, double? DenseRate, int? NewDays, int? PerDay, string Reason = "");
|
||||
|
||||
public sealed record FindingDto(string Kind, string Text);
|
||||
|
||||
/// <summary>#693: A run picked for the lead, with what the audit log showed and the lead's verdict.</summary>
|
||||
public sealed record SpotCheckDto(
|
||||
long Id, string Lead, string AgentId, string RunId, string Why, IReadOnlyList<FindingDto> Findings,
|
||||
DateTime CreatedAt, string? Verdict, string? Note, DateTime? ReviewedAt);
|
||||
|
||||
/// <summary>#693: A measure a lead proposed; a human decides it.</summary>
|
||||
public sealed record MeasureDto(
|
||||
long Id, long? CheckId, string AgentId, string ProposedBy, string Kind, string Reasoning,
|
||||
DateTime ProposedAt, string Status, string? DecidedBy, DateTime? DecidedAt, string? Note);
|
||||
|
||||
public sealed record SpotCheckOverviewDto(IReadOnlyList<SpotCheckDto> Checks, IReadOnlyList<MeasureDto> Measures);
|
||||
|
||||
public sealed record MeasureDecisionDto(bool Accept, string Note = "");
|
||||
|
||||
public sealed record UnitDefaultsDto(string? Model, decimal? DailyCostUsd, long? DailyTokens, string? WhenExhausted);
|
||||
|
||||
/// <summary>An agent as the organisation page shows it: where it is, whom it answers to.</summary>
|
||||
|
||||
@@ -28,7 +28,8 @@ internal static class OrganisationEndpoints
|
||||
chart.ToolCeiling(u.Id)?.Order(StringComparer.OrdinalIgnoreCase).ToList(),
|
||||
new UnitDefaultsDto(u.Defaults.Model, u.Defaults.DailyCostUsd, u.Defaults.DailyTokens, u.Defaults.WhenExhausted),
|
||||
chart.PathOf(u.Id), chart.Depth(u.Id),
|
||||
u.Approvals is { } ap ? new UnitApprovalsDto(ap.Operations, ap.PerDay, ap.Hours) : null))
|
||||
u.Approvals is { } ap ? new UnitApprovalsDto(ap.Operations, ap.PerDay, ap.Hours) : null,
|
||||
u.Inspection is { } ins ? new UnitInspectionDto(ins.Rate, ins.DenseRate, ins.NewDays, ins.PerDay) : null))
|
||||
.ToList();
|
||||
|
||||
var agents = host.Instance.Agents
|
||||
@@ -103,6 +104,32 @@ internal static class OrganisationEndpoints
|
||||
return Results.Ok(new OutcomeDto(true, set.IsEmpty ? $"Doctrine of '{unit.Name}' removed." : $"Doctrine of '{unit.Name}' set (version {set.Version})."));
|
||||
});
|
||||
|
||||
// #693: The lead's spot checks — how often, what came of them, and the measures it proposed.
|
||||
org.MapPut("/units/{unitId}/inspection", (AppHost host, string unitId, InspectionInputDto input) =>
|
||||
host.Organisation.SetInspection(unitId, input.Enabled, input.Rate, input.DenseRate, input.NewDays, input.PerDay, input.Reason).ToResponse());
|
||||
|
||||
org.MapGet("/units/{unitId}/spot-checks", async (AppHost host, string unitId, CancellationToken ct) =>
|
||||
{
|
||||
if (host.SpotChecks is not { } checks)
|
||||
return Results.Ok(new SpotCheckOverviewDto([], []));
|
||||
var list = await checks.ForUnitAsync(unitId, 30, ct);
|
||||
var measures = await checks.MeasuresForUnitAsync(unitId, 30, ct);
|
||||
return Results.Ok(new SpotCheckOverviewDto(
|
||||
list.Select(c => new SpotCheckDto(c.Id, c.Lead, c.AgentId, c.RunId, c.Why,
|
||||
c.Findings.Select(f => new FindingDto(f.Kind.ToString(), f.Text)).ToList(),
|
||||
c.CreatedAt, c.Verdict, c.Note, c.ReviewedAt)).ToList(),
|
||||
measures.Select(m => new MeasureDto(m.Id, m.CheckId, m.AgentId, m.ProposedBy,
|
||||
Core.Inspection.SpotChecks.Label(m.Kind), m.Reasoning, m.ProposedAt, m.Status, m.DecidedBy, m.DecidedAt, m.Note)).ToList()));
|
||||
});
|
||||
|
||||
org.MapPost("/measures/{measureId:long}/decide", async (AppHost host, long measureId, MeasureDecisionDto input, CancellationToken ct) =>
|
||||
{
|
||||
if (host.SpotChecks is not { } checks)
|
||||
return Results.BadRequest(new OutcomeDto(false, "Spot checks are not available in this instance."));
|
||||
var decided = await checks.DecideMeasureAsync(measureId, input.Accept, input.Note, "human", ct);
|
||||
return decided.Ok ? Results.Ok(new OutcomeDto(true, decided.Message)) : Results.BadRequest(new OutcomeDto(false, decided.Message));
|
||||
});
|
||||
|
||||
// #687: What the lead may approve instead of the human.
|
||||
org.MapPut("/units/{unitId}/approvals", (AppHost host, string unitId, ApprovalsInputDto input) =>
|
||||
host.Organisation.SetApprovals(unitId, input.Operations, input.PerDay, input.Hours, input.Reason).ToResponse());
|
||||
|
||||
Reference in New Issue
Block a user