using System.Globalization; using System.Security.Cryptography; using System.Text; using System.Text.Json; using ClawdDotNet.Core.Audit; using ClawdDotNet.Core.Config; using ClawdDotNet.Core.Organisation; using ClawdDotNet.Core.Storage; using ClawdDotNet.Core.Tasks; using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging; namespace ClawdDotNet.Core.Inspection; /// A run picked for the lead to look at, with what the mechanical check found in it. public sealed record SpotCheck( long Id, string UnitId, string Lead, string AgentId, string RunId, string Why, IReadOnlyList Findings, DateTime CreatedAt, string? Verdict, string? Note, DateTime? ReviewedAt, string? TaskId) { public bool Open => Verdict is null; } /// /// #693: What follows from a finding — graded, not a switch. The first four are personnel /// matters; is not one: an agent that may be steered from outside is /// not helped by a better model. /// public enum MeasureKind { /// The task was unclear — the most frequent case (#663). SharpenTask, /// The model is overtaxed: suitability probe (#681), then another model. ChangeModel, /// Rights too wide or too narrow (#641). AdjustRights, /// Repeated failures despite everything: pause the agent, hand its tasks on. Suspend, /// Suspected manipulation from outside — for the security side, not for personnel. Security } public sealed record Measure( long Id, long? CheckId, string UnitId, string ProposedBy, string AgentId, MeasureKind Kind, string Reasoning, DateTime ProposedAt, string Status, string? DecidedBy, DateTime? DecidedAt, string? Note, string? TaskId); public sealed record SpotCheckOutcome(bool Ok, string Message); /// #693: The lead's side of the spot checks, as the Escalation tool sees it. public interface ISpotChecks { /// Whether is the one answers to. bool Leads(string lead, string agentId); Task InspectAsync(string runId, CancellationToken ct); Task> PendingForAsync(string lead, CancellationToken ct); Task CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct); Task ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct); Task> MeasuresByAsync(string lead, int days, CancellationToken ct); } /// /// #693: A lead does not only delegate — it looks. This picks runs of the agents a lead leads, /// holds each against the audit () and gives the lead one task to /// look at them: the mechanical part finds what does not fit, the lead judges whether the /// result answers the task. /// /// Which runs. A share per unit (), denser for a new /// agent, a new model or the run after a failure — and capped per day, because every check /// costs the lead's run. Picked by the run id, not by chance: the same run gets the same /// answer on every pass. /// /// What follows. The lead proposes a measure with its reasons; a human decides. /// A model that pauses or reconfigures another changes the organisation without anyone /// deciding it — nothing here carries a measure out. /// public sealed class SpotChecks( SqliteStorage storage, IAuditRepository audit, RunInspector inspector, Func organisation, TaskboardService? board, ILogger logger, TimeProvider? clock = null) : ISpotChecks, IAsyncDisposable { public static readonly TimeSpan Interval = TimeSpan.FromMinutes(15); /// How far back a pass looks for an agent's earlier runs. public const int HistoryDays = 30; /// Statuses where no run took place — nothing to check. private static readonly HashSet NotRun = new(StringComparer.OrdinalIgnoreCase) { "BudgetExceeded", "TargetBusy", "Cancelled" }; private static readonly JsonSerializerOptions Json = new() { Converters = { new System.Text.Json.Serialization.JsonStringEnumConverter() } }; private readonly TimeProvider _clock = clock ?? TimeProvider.System; private readonly CancellationTokenSource _cts = new(); private DateTime _seenUntil = DateTime.MinValue; private Task? _loop; public bool Leads(string lead, string agentId) => !string.Equals(lead, agentId, StringComparison.OrdinalIgnoreCase) && string.Equals(organisation().EffectiveLeaderOf(agentId)?.AgentId, lead, StringComparison.OrdinalIgnoreCase); public Task InspectAsync(string runId, CancellationToken ct) => inspector.InspectAsync(runId, ct); /// /// Why this run is checked — or null if it is not. are the /// agent's runs before this one, oldest first. /// public static string? Pick(RunReceipt run, IReadOnlyList earlier, UnitInspection settings) { string? dense = null; if (earlier.Count == 0 || earlier[0].OccurredAt >= run.OccurredAt.AddDays(-Math.Max(0, settings.NewDays))) dense = "new agent"; else if (!earlier.Any(r => string.Equals(r.Model, run.Model, StringComparison.OrdinalIgnoreCase))) dense = "new model"; else if (earlier.LastOrDefault(r => !NotRun.Contains(r.Status)) is { } previous && !string.Equals(previous.Status, "Completed", StringComparison.OrdinalIgnoreCase)) dense = "after a failure"; var rate = Math.Clamp(dense is null ? settings.Rate : settings.DenseRate, 0, 1); return Fraction(run.RunId) < rate ? dense ?? "sample" : null; } /// A number in [0, 1) from the run id — the same for the same run, spread like chance. internal static double Fraction(string runId) { var hash = SHA256.HashData(Encoding.UTF8.GetBytes(runId)); return BitConverter.ToUInt32(hash, 0) / 4294967296.0; } /// One pass: picks among the runs since the last pass. Returns the checks it created. public async Task> SampleAsync(CancellationToken ct) { var org = organisation(); if (!org.HasUnits || org.Units.All(u => u.Inspection is null)) return []; var now = _clock.GetUtcNow().UtcDateTime; var from = _seenUntil == DateTime.MinValue ? now.AddDays(-1) : _seenUntil; var fresh = await audit.ListReceiptsAsync(from, now, 5_000, ct); if (fresh.Count == 0) return []; _seenUntil = fresh[^1].OccurredAt.AddTicks(1); var history = (await audit.ListReceiptsAsync(now.AddDays(-HistoryDays), now, 100_000, ct)) .GroupBy(r => r.AgentId, StringComparer.OrdinalIgnoreCase) .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.OrdinalIgnoreCase); var today = DateOnly.FromDateTime(_clock.GetLocalNow().DateTime); var usedToday = new Dictionary(StringComparer.OrdinalIgnoreCase); var created = new List(); foreach (var run in fresh) { if (NotRun.Contains(run.Status) || org.EffectiveLeaderOf(run.AgentId) is not { } lead || org.LedBy(lead.AgentId) is not { Inspection: { } settings } unit) continue; var earlier = history.GetValueOrDefault(run.AgentId)?.Where(r => r.OccurredAt < run.OccurredAt && r.RunId != run.RunId).ToList() ?? []; if (Pick(run, earlier, settings) is not { } why) continue; if (!usedToday.TryGetValue(unit.Id, out var used)) used = usedToday[unit.Id] = await CountSinceAsync(unit.Id, today, ct); if (used >= Math.Max(0, settings.PerDay)) continue; if (await inspector.InspectAsync(run.RunId, ct) is not { } inspection) continue; if (await InsertAsync(unit.Id, lead.AgentId, run, why, inspection.Findings, now, ct) is { } check) { usedToday[unit.Id] = used + 1; created.Add(check); } } foreach (var lead in created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase)) await GiveTaskAsync(lead, ct); if (created.Count > 0) logger.LogInformation("Spot checks: {Count} run(s) picked for {Leads}", created.Count, string.Join(", ", created.Select(c => c.Lead).Distinct(StringComparer.OrdinalIgnoreCase))); return created; } /// /// One task per lead at a time, for all its open checks — every check costs the lead a look, /// but not a run each. /// private async Task GiveTaskAsync(string lead, CancellationToken ct) { var open = await PendingForAsync(lead, ct); var untasked = open.Where(c => c.TaskId is null).ToList(); if (untasked.Count == 0 || board is null) return; foreach (var taskId in open.Select(c => c.TaskId).OfType().Distinct()) { if (await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled or TaskItemStatus.Archived) }) { await SetTaskAsync(untasked, taskId, ct); await board.AddCommentAsync(taskId, "system", $"{untasked.Count} more run(s) to check.", ct); return; } } var task = await board.CreateAsync(new TaskItem { Title = $"Spot check: {open.Count} run(s) of the agents you lead", Body = """ Look at whether the agents you lead work properly. Not by asking them — by holding each result against its task and against what the engine recorded. 1. Escalation spot_checks lists the runs, with what the audit log shows. 2. Escalation inspect_run (run) shows one run in full: task, calls, result. 3. Judge: does the result answer the task, and do the calls back it up? 4. Escalation check_done (check, verdict ok|finding, note) for each. If something follows from it, Escalation propose_measure — a human decides: sharpen_task (unclear task — the most frequent case), change_model, adjust_rights, suspend, or security (suspected manipulation from outside: not a personnel matter). """, Assignee = "@" + lead, Priority = 3, Acceptance = "Every listed check is closed with a verdict." }, ct); await SetTaskAsync(open, task.Id, ct); } public async Task> PendingForAsync(string lead, CancellationToken ct) => await QueryChecksAsync("Lead = @p AND Verdict IS NULL", lead, 50, ct); public async Task> ForUnitAsync(string unitId, int limit, CancellationToken ct) => await QueryChecksAsync("UnitId = @p", unitId, limit, ct); public async Task CloseAsync(long checkId, string lead, bool finding, string note, CancellationToken ct) { var check = (await QueryChecksAsync("Id = @p", checkId.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault(); if (check is null || !string.Equals(check.Lead, lead, StringComparison.OrdinalIgnoreCase)) return new(false, $"No check {checkId} waits for you."); if (!check.Open) return new(false, $"Check {checkId} is closed already ({check.Verdict})."); if (finding && string.IsNullOrWhiteSpace(note)) return new(false, "'note' is required for a finding — what did not fit, with the evidence."); var verdict = finding ? "finding" : "ok"; await storage.WriteAsync(async conn => { using var cmd = conn.CreateCommand(); cmd.CommandText = "UPDATE SpotChecks SET Verdict = @v, Note = @n, ReviewedAt = @at WHERE Id = @id AND Verdict IS NULL"; cmd.Parameters.AddWithValue("@v", verdict); cmd.Parameters.AddWithValue("@n", (note ?? "").Trim()); cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O")); cmd.Parameters.AddWithValue("@id", checkId); return await cmd.ExecuteNonQueryAsync(ct); }, ct); // The last check of a task closes the task. if (check.TaskId is { } taskId && board is not null && (await PendingForAsync(lead, ct)).All(c => c.TaskId != taskId) && await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) }) await board.UpdateAsync(taskId, t => t with { Status = TaskItemStatus.Done }, ct); return new(true, finding ? $"Check {checkId} closed with a finding. If something should follow, propose_measure — a human decides." : $"Check {checkId} closed: in order."); } public async Task ProposeAsync(string lead, string agentId, MeasureKind kind, string reasoning, long? checkId, CancellationToken ct) { agentId = (agentId ?? "").Trim().TrimStart('@'); if (!Leads(lead, agentId)) return new(false, $"You do not lead '{agentId}' — propose measures only for the agents that answer to you."); if (string.IsNullOrWhiteSpace(reasoning)) return new(false, "'reasoning' is required — what you found, with the evidence, and why this measure."); if (organisation().LedBy(lead) is not { } unit) return new(false, "Only a lead proposes measures."); var open = await QueryMeasuresAsync("ProposedBy = @p AND Status = 'proposed'", lead, 200, ct); if (open.Any(m => m.Kind == kind && string.Equals(m.AgentId, agentId, StringComparison.OrdinalIgnoreCase))) return new(false, $"A {Label(kind)} for '{agentId}' waits for a human already."); string? taskId = null; if (board is not null) taskId = (await board.CreateAsync(new TaskItem { Title = kind == MeasureKind.Security ? $"Security suspicion: {agentId} (from its lead {lead})" : $"Proposed measure for {agentId}: {Label(kind)}", Body = (kind == MeasureKind.Security ? "Not a personnel matter: the lead suspects the agent is steered from outside. A better model does not help here — consider pausing it and looking at its runs and the content it read.\n\n" : $"The lead {lead} proposes this after a spot check. Nothing has been changed — decide it in the organisation view.\n\n") + $"Reasoning of {lead}:\n{reasoning.Trim()}" + (checkId is { } c ? $"\n\nSpot check: {c}" : ""), Assignee = TaskAssignee.Human, Priority = kind == MeasureKind.Security ? 5 : 3 }, ct)).Id; var id = await storage.WriteAsync(async conn => { using var cmd = conn.CreateCommand(); cmd.CommandText = """ INSERT INTO SpotMeasures (CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, TaskId) VALUES (@check, @unit, @by, @agent, @kind, @reasoning, @at, 'proposed', @task) RETURNING Id """; cmd.Parameters.AddWithValue("@check", (object?)checkId ?? DBNull.Value); cmd.Parameters.AddWithValue("@unit", unit.Id); cmd.Parameters.AddWithValue("@by", lead); cmd.Parameters.AddWithValue("@agent", agentId); cmd.Parameters.AddWithValue("@kind", kind.ToString()); cmd.Parameters.AddWithValue("@reasoning", reasoning.Trim()); cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O")); cmd.Parameters.AddWithValue("@task", (object?)taskId ?? DBNull.Value); return Convert.ToInt64(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); }, ct); logger.LogInformation("Measure {Id} ({Kind}) for {Agent} proposed by {Lead}", id, kind, agentId, lead); return new(true, kind == MeasureKind.Security ? $"Security suspicion {id} for '{agentId}' is with a human. Do not act on the agent yourself." : $"Measure {id} ({Label(kind)}) for '{agentId}' is with a human. Nothing changes until it is decided."); } /// A human's decision on a proposed measure. Accepting records it — carrying it out stays with the human. public async Task DecideMeasureAsync(long id, bool accept, string note, string decidedBy, CancellationToken ct) { var measure = (await QueryMeasuresAsync("Id = @p", id.ToString(CultureInfo.InvariantCulture), 1, ct)).FirstOrDefault(); if (measure is null) return new(false, $"There is no measure {id}."); if (measure.Status != "proposed") return new(false, $"Measure {id} is decided already ({measure.Status})."); var status = accept ? "accepted" : "dismissed"; await storage.WriteAsync(async conn => { using var cmd = conn.CreateCommand(); cmd.CommandText = "UPDATE SpotMeasures SET Status = @s, DecidedBy = @by, DecidedAt = @at, Note = @n WHERE Id = @id AND Status = 'proposed'"; cmd.Parameters.AddWithValue("@s", status); cmd.Parameters.AddWithValue("@by", decidedBy); cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O")); cmd.Parameters.AddWithValue("@n", (note ?? "").Trim()); cmd.Parameters.AddWithValue("@id", id); return await cmd.ExecuteNonQueryAsync(ct); }, ct); if (measure.TaskId is { } taskId && board is not null && await board.GetAsync(taskId, ct) is { Status: not (TaskItemStatus.Done or TaskItemStatus.Canceled) }) { await board.AddCommentAsync(taskId, decidedBy, $"Measure {status}." + (string.IsNullOrWhiteSpace(note) ? "" : " " + note.Trim()), ct); await board.UpdateAsync(taskId, t => t with { Status = accept ? TaskItemStatus.Done : TaskItemStatus.Canceled }, ct); } return new(true, $"Measure {id} {status}."); } public Task> MeasuresByAsync(string lead, int days, CancellationToken ct) => QueryMeasuresAsync("ProposedBy = @p AND ProposedAt >= @since", lead, 50, ct, _clock.GetUtcNow().UtcDateTime.AddDays(-Math.Max(1, days))); public Task> MeasuresForUnitAsync(string unitId, int limit, CancellationToken ct) => QueryMeasuresAsync("UnitId = @p", unitId, limit, ct); public static string Label(MeasureKind kind) => kind switch { MeasureKind.SharpenTask => "sharpen_task", MeasureKind.ChangeModel => "change_model", MeasureKind.AdjustRights => "adjust_rights", MeasureKind.Suspend => "suspend", _ => "security" }; public static MeasureKind? ParseKind(string? text) => (text ?? "").Trim().ToLowerInvariant() switch { "sharpen_task" => MeasureKind.SharpenTask, "change_model" => MeasureKind.ChangeModel, "adjust_rights" => MeasureKind.AdjustRights, "suspend" => MeasureKind.Suspend, "security" => MeasureKind.Security, _ => null }; // ─── storage ─── private async Task CountSinceAsync(string unitId, DateOnly localDay, CancellationToken ct) { var since = new DateTimeOffset(localDay.ToDateTime(TimeOnly.MinValue), _clock.LocalTimeZone.GetUtcOffset(localDay.ToDateTime(TimeOnly.MinValue))).UtcDateTime; await using var conn = await storage.OpenConnectionAsync(ct); using var cmd = conn.CreateCommand(); cmd.CommandText = "SELECT COUNT(*) FROM SpotChecks WHERE UnitId = @unit AND CreatedAt >= @since"; cmd.Parameters.AddWithValue("@unit", unitId); cmd.Parameters.AddWithValue("@since", since.ToString("O")); return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); } private Task InsertAsync(string unitId, string lead, RunReceipt run, string why, IReadOnlyList findings, DateTime now, CancellationToken ct) => storage.WriteAsync(async conn => { using var cmd = conn.CreateCommand(); cmd.CommandText = """ INSERT OR IGNORE INTO SpotChecks (UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt) VALUES (@unit, @lead, @agent, @run, @why, @findings, @at) RETURNING Id """; cmd.Parameters.AddWithValue("@unit", unitId); cmd.Parameters.AddWithValue("@lead", lead); cmd.Parameters.AddWithValue("@agent", run.AgentId); cmd.Parameters.AddWithValue("@run", run.RunId); cmd.Parameters.AddWithValue("@why", why); cmd.Parameters.AddWithValue("@findings", JsonSerializer.Serialize(findings, Json)); cmd.Parameters.AddWithValue("@at", now.ToString("O")); var id = await cmd.ExecuteScalarAsync(ct); return id is null or DBNull ? null : new SpotCheck(Convert.ToInt64(id, CultureInfo.InvariantCulture), unitId, lead, run.AgentId, run.RunId, why, findings, now, null, null, null, null); }, ct); private Task SetTaskAsync(IEnumerable checks, string taskId, CancellationToken ct) => storage.WriteAsync(async conn => { foreach (var check in checks) { using var cmd = conn.CreateCommand(); cmd.CommandText = "UPDATE SpotChecks SET TaskId = @task WHERE Id = @id"; cmd.Parameters.AddWithValue("@task", taskId); cmd.Parameters.AddWithValue("@id", check.Id); await cmd.ExecuteNonQueryAsync(ct); } return 0; }, ct); private async Task> QueryChecksAsync(string where, string value, int limit, CancellationToken ct) { await using var conn = await storage.OpenConnectionAsync(ct); using var cmd = conn.CreateCommand(); cmd.CommandText = $"SELECT Id, UnitId, Lead, AgentId, RunId, Why, Findings, CreatedAt, Verdict, Note, ReviewedAt, TaskId FROM SpotChecks WHERE {where} ORDER BY Id DESC LIMIT @limit"; cmd.Parameters.AddWithValue("@p", value); cmd.Parameters.AddWithValue("@limit", limit); var result = new List(); await using var r = await cmd.ExecuteReaderAsync(ct); while (await r.ReadAsync(ct)) result.Add(new SpotCheck(r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3), r.GetString(4), r.GetString(5), JsonSerializer.Deserialize>(r.GetString(6), Json) ?? [], Time(r, 7)!.Value, Text(r, 8), Text(r, 9), Time(r, 10), Text(r, 11))); return result; } private async Task> QueryMeasuresAsync(string where, string value, int limit, CancellationToken ct, DateTime? since = null) { await using var conn = await storage.OpenConnectionAsync(ct); using var cmd = conn.CreateCommand(); cmd.CommandText = $"SELECT Id, CheckId, UnitId, ProposedBy, AgentId, Kind, Reasoning, ProposedAt, Status, DecidedBy, DecidedAt, Note, TaskId FROM SpotMeasures WHERE {where} ORDER BY Id DESC LIMIT @limit"; cmd.Parameters.AddWithValue("@p", value); cmd.Parameters.AddWithValue("@limit", limit); if (since is { } s) cmd.Parameters.AddWithValue("@since", s.ToString("O")); var result = new List(); await using var r = await cmd.ExecuteReaderAsync(ct); while (await r.ReadAsync(ct)) result.Add(new Measure(r.GetInt64(0), r.IsDBNull(1) ? null : r.GetInt64(1), r.GetString(2), r.GetString(3), r.GetString(4), Enum.TryParse(r.GetString(5), out var k) ? k : MeasureKind.SharpenTask, r.GetString(6), Time(r, 7)!.Value, r.GetString(8), Text(r, 9), Time(r, 10), Text(r, 11), Text(r, 12))); return result; } private static string? Text(SqliteDataReader r, int i) => r.IsDBNull(i) ? null : r.GetString(i); private static DateTime? Time(SqliteDataReader r, int i) => r.IsDBNull(i) ? null : DateTime.Parse(r.GetString(i), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); // ─── loop ─── public void Start() => _loop ??= RunAsync(_cts.Token); private async Task RunAsync(CancellationToken ct) { try { await Task.Delay(TimeSpan.FromMinutes(2), ct); } catch (OperationCanceledException) { return; } while (!ct.IsCancellationRequested) { try { await SampleAsync(ct); } catch (Exception ex) when (ex is not OperationCanceledException) { logger.LogWarning(ex, "Spot checks failed — next attempt in {Minutes} min", Interval.TotalMinutes); } try { await Task.Delay(Interval, ct); } catch (OperationCanceledException) { return; } } } public async ValueTask DisposeAsync() { await _cts.CancelAsync(); if (_loop is not null) { try { await _loop; } catch (OperationCanceledException) { } } _cts.Dispose(); } }