using Microsoft.Extensions.Logging; namespace ClawdDotNet.Core.Tasks; /// Wie ein Dispatch ausging. public enum TaskLauf { /// Der Lauf schloss regulär ab. Erledigt, /// Der Lauf fand statt und scheiterte. Gescheitert, /// /// Der Lauf fand gar nicht statt — Instanz im Herunterfahren (#643). /// Eine einmalige Aufgabe bleibt dann fällig, statt als gelaufen liegenzubleiben. /// NichtGelaufen, /// /// #716: The run did not take place because the budget is exhausted. Unlike a /// shutdown, retrying the same day is pointless — the scanner defers to the next day /// and applies the agent's . /// KeinBudget } /// /// Führt eine fällige Aufgabe aus. Kapselt, wie ein Assignee zu einem Lauf wird — der /// Scanner selbst kennt die Engine nicht und bleibt so ohne sie testbar. /// public interface ITaskDispatcher { /// Führt die Aufgabe aus und meldet, wie der Lauf ausging. Task DispatchAsync(TaskItem task, CancellationToken ct); /// /// #717: Runs the task with an instruction from the code ahead of it. Without an /// implementation the instruction is dropped and the task runs as usual. /// Task DispatchAsync(TaskItem task, string? instruction, CancellationToken ct) => DispatchAsync(task, ct); } /// /// #717: Whether a task is too large to simply start — its estimate is a large share of the /// agent's daily budget. Then its run plans instead: split it, or ask for a special budget. /// public interface ILargeTaskCheck { /// The instruction for the planning run, or null if the task can simply run. Task PlanningInstructionAsync(TaskItem task, CancellationToken ct); /// Whether the task's tree has a special budget — then, after planning, it runs. Task IsFundedAsync(TaskItem task, CancellationToken ct); } /// /// Entscheidet, ob ein marktabhängiger Termin () /// jetzt laufen darf. Platzhalter, bis C1 (Marktkalender) den echten Kalender liefert. /// public interface IMarketCalendar { bool IsOpen(DateTime nowUtc); } /// /// #716: Decides whether a late run — carried-over work or a catch-up — may start now. /// Without one, late runs only wait for the regular ones of the same tick. /// public interface ILateRunGate { Task HasRoomAsync(TaskItem lateTask, IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct); } /// #717: Whether fill work may start now. public enum FillVerdict { /// There is room — it starts. Run, /// Too early in the day; the forecast is still a guess. NotYet, /// The rest of the day needs the budget. NoRoom } /// /// #717: Decides whether fill work — one-time work with a deadline instead of an appointment — /// may use the budget the day would otherwise leave unused. Without one, fill work runs like /// late work: after everything else of the tick. /// public interface IFillGate { Task CheckAsync(TaskItem fillTask, IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct); } /// /// #716: The day ahead — where the budget will not reach the end of today's schedule. /// public interface IBudgetOutlook { Task> ShortfallsAsync(IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct); } /// /// #716: An agent whose budget runs out before today's schedule does — from when on, and /// how many planned runs will be refused. is empty for the instance /// limit. /// public sealed record BudgetShortfall(string AgentId, DateTime ExhaustsAtUtc, int RunsRefused, int RunsPlanned); /// Bis C1 kommt: der Markt gilt als immer offen. public sealed class AlwaysOpenMarketCalendar : IMarketCalendar { public bool IsOpen(DateTime nowUtc) => true; } /// /// Der Taktgeber des Taskboards (A1). Ein einziger Takt prüft, was fällig ist — /// keine langen Delays (B6). Persistiert wird nur der Last-Fired-Marker; ein /// fehlgeschlagener Lauf bleibt der einzige Versuch für diesen Termin (kein Retry-Sturm). /// /// Der heikle Kern ist streng gegen drei Invarianten gebaut, die die Repository-Tests /// und die Scanner-Tests absichern: /// 1. Nie zwei Läufe auf denselben Termin — das atomare . /// 2. Kein Dispatch bei offenem Blocker — blockierte Aufgaben sind nicht claimbar. /// 3. Doppelter Takt = ein Lauf — konkurrierende Claims um denselben Occurrence-Key. /// public sealed class TaskScanner : IAsyncDisposable { private readonly ITaskRepository _repo; private readonly ITaskDispatcher _dispatcher; private readonly IMarketCalendar _market; private readonly TimeProvider _clock; private readonly ILogger _logger; private readonly TimeSpan _tick; private readonly TimeSpan _lease; /// Aufgaben, deren unauflösbare Zeitzone bereits gemeldet wurde — der Takt /// läuft jede Minute, die Meldung soll nicht mitlaufen. private readonly HashSet _warnedTimeZones = []; /// Aufgaben, deren ungültiger Cron-Ausdruck bereits gemeldet wurde. private readonly HashSet _warnedCron = []; /// Wiederkehrende @agent-Aufgaben, deren wachsender Kontext bereits gemeldet wurde (#598). private readonly HashSet _warnedContext = []; /// /// #643: Einmalige Aufgaben, deren Lauf nicht stattfand, und ab wann sie wieder dran sind. /// Ohne diese Pause würde eine wegen Budget abgewiesene Aufgabe jede Minute erneut /// anlaufen. Nur im Speicher — nach einem Neustart darf sie einmal sofort versuchen. /// private readonly System.Collections.Concurrent.ConcurrentDictionary _zurueckgestellt = new(); /// Pause, bevor eine nicht gelaufene einmalige Aufgabe erneut versucht wird. public static readonly TimeSpan Wiedervorlage = TimeSpan.FromMinutes(30); /// #682: Default for . public const int DefaultMaxParallelRuns = 4; private readonly int _maxParallelRuns; /// /// Runs started by the tick loop that have not finished yet. The loop no longer waits /// for them (#682), so disposal has to. /// private readonly HashSet _inFlight = []; /// How many due tasks had to wait at the last tick — logged only on change. private int _lastWaiting; /// #716: What the budget refused, per agent, for the day summary. private readonly Func _missPolicy; private readonly Dictionary _missesToday = new(StringComparer.OrdinalIgnoreCase); private DateOnly? _missDay; private readonly ILateRunGate? _lateRunGate; private readonly IBudgetOutlook? _outlook; private readonly ITaskTreeWatch? _treeWatch; /// #717: Whether fill work may use today's unused budget. private readonly IFillGate? _fillGate; /// #717: Asks a human about fill work that found no room before its deadline. private readonly Func? _askAboutFill; private readonly Func _fillWarnBefore; /// #717: Whether a task is too large to simply start. private readonly ILargeTaskCheck? _largeCheck; /// #717: Leaves a note on a task — why the scanner holds it. private readonly Func? _noteOnTask; /// #717: Fill work already reported as past its deadline — once per task. private readonly HashSet _warnedOverdueFill = []; /// How many fill tasks waited for room at the last tick. private int _lastWaitingFill; /// How often the day ahead is looked at — a forecast, not a watchdog. public static readonly TimeSpan OutlookInterval = TimeSpan.FromMinutes(30); private DateTime _lastOutlookAt = DateTime.MinValue; /// Agents already warned about today, with the day — once per agent and day. private readonly Dictionary _shortfallWarned = new(StringComparer.OrdinalIgnoreCase); /// How many late runs waited for budget headroom at the last tick. private int _lastWaitingForHeadroom; private readonly CancellationTokenSource _cts = new(); private Task? _loop; public TaskScanner( ITaskRepository repo, ITaskDispatcher dispatcher, ILoggerFactory loggerFactory, IMarketCalendar? market = null, TimeProvider? clock = null, TimeSpan? tick = null, TimeSpan? lease = null, int maxParallelRuns = DefaultMaxParallelRuns, Func? budgetMissPolicy = null, ILateRunGate? lateRunGate = null, IBudgetOutlook? budgetOutlook = null, ITaskTreeWatch? treeWatch = null, IFillGate? fillGate = null, Func? askAboutFill = null, Func? fillWarnBefore = null, ILargeTaskCheck? largeTaskCheck = null, Func? noteOnTask = null) { _largeCheck = largeTaskCheck; _noteOnTask = noteOnTask; _treeWatch = treeWatch; _fillGate = fillGate; _askAboutFill = askAboutFill; _fillWarnBefore = fillWarnBefore ?? (() => TimeSpan.FromHours(24)); _repo = repo; _dispatcher = dispatcher; _market = market ?? new AlwaysOpenMarketCalendar(); _clock = clock ?? TimeProvider.System; _logger = loggerFactory.CreateLogger("ClawdDotNet.Core.Tasks.Scanner"); _tick = tick ?? TimeSpan.FromSeconds(60); _lease = lease ?? TimeSpan.FromMinutes(15); _maxParallelRuns = Math.Max(1, maxParallelRuns); _missPolicy = budgetMissPolicy ?? (_ => BudgetMissPolicy.Expire); _lateRunGate = lateRunGate; _outlook = budgetOutlook; } /// How many scheduled runs may be in flight at once (#682). public int MaxParallelRuns => _maxParallelRuns; public void Start() { _loop ??= RunLoopAsync(_cts.Token); } /// Läuft die Taktschleife gerade? public bool IsRunning => _loop is { IsCompleted: false }; /// /// Die Schleife lief und ist beendet — anders als „noch nicht gestartet". Der /// Unterschied zählt für die Zustandsmeldung an den Watchdog: Ein Scanner, der noch /// auf die Startabgleichung wartet, ist in Ordnung; einer, dessen Schleife /// ausgestiegen ist, bedeutet, dass keine Aufgabe mehr läuft. /// public bool HasStopped => _loop is { IsCompleted: true }; // ─── Geordnetes Ende (#528) ─── private int _openRuns; private int _accepting = 1; /// /// Wie viele Aufgaben gerade tatsächlich laufen — also beansprucht und noch nicht /// abgeschlossen. /// public int OpenCount => Volatile.Read(ref _openRuns); /// Beansprucht der Scanner noch neue Aufgaben? public bool IsAcceptingWork => Volatile.Read(ref _accepting) == 1; /// /// Schließt die Tür: kein neuer Takt, keine neue Beanspruchung. /// /// Warum das nicht dasselbe ist wie . Dispose /// bricht den laufenden Takt über sein Token ab — mitten in ProcessAsync, also /// nachdem eine Aufgabe beansprucht wurde und bevor CompleteClaimAsync sie /// wieder freigibt. Sie steht danach auf running und bleibt dort bis zum /// Ablauf des Lease. Wer geordnet enden will, schließt erst hier die Tür, wartet auf /// gleich null und ruft Dispose erst dann. /// public void StopAccepting() => Volatile.Write(ref _accepting, 0); private async Task RunLoopAsync(CancellationToken ct) { // PeriodicTimer über den TimeProvider — im Test steuerbar, im Betrieb driftfrei. using var timer = new PeriodicTimer(_tick, _clock); while (await timer.WaitForNextTickAsync(ct)) { if (!IsAcceptingWork) continue; // Die Tuer ist zu. Der Takt laeuft leer, bis Dispose ihn beendet. try { // #682: Start, don't wait. Before, the tick awaited every run it started, // so one ten-minute run held back every other schedule for ten minutes. // Now a finished run frees its slot and the next tick fills it. await LaunchDueAsync(ct); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogError(ex, "Ein Scanner-Takt ist gescheitert"); } } } /// /// Ein Durchlauf: fällige Aufgaben beanspruchen und ausführen. Gibt die Zahl der /// tatsächlich angestoßenen Läufe zurück und kehrt erst zurück, wenn sie fertig sind. /// Öffentlich, damit Tests einen Takt deterministisch auslösen können — die /// Taktschleife selbst wartet nicht auf die Läufe (#682). /// public async Task ScanOnceAsync(CancellationToken ct) { var launched = await LaunchDueAsync(ct); await Task.WhenAll(launched); return launched.Count; } /// /// Claims due tasks while a slot is free and starts them without waiting. /// /// Claim only what can start now (#682). A claim sets the task to /// running with a lease. Claiming everything and queueing it in memory would /// let a waiting task outlive its lease and be claimed a second time. So tasks beyond /// the free slots stay todo and are picked up by a later tick — a cron occurrence /// is not lost by that, searches back to the /// last run. /// /// Order: priority first, then the oldest due occurrence — when slots are /// short, what matters and what has waited longest goes first. /// private async Task> LaunchDueAsync(CancellationToken ct) { if (!IsAcceptingWork) return []; var now = _clock.GetUtcNow().UtcDateTime; var leaseCutoff = now - _lease; // #682: ungekappt. Vorher ListAsync mit Limit 500 — darüber fielen Termine lautlos aus. var candidates = await _repo.ListRunnableAsync(ct); SummarizeMissesIfNewDay(now); await WarnAboutShortfallsAsync(candidates, now, ct); // #714: resting task trees are looked at — the watch decides itself how often. if (_treeWatch is not null) await _treeWatch.CheckAsync(now, ct); var due = new List<(TaskItem Task, string Occurrence, RunKind Kind)>(); foreach (var task in candidates) { if (task.Status != TaskItemStatus.Todo) continue; // backlog (Halte-Status), blockiert, laufend, erledigt bleiben außen vor // Ein Mensch ist kein Modell-Lauf — solche Aufgaben rührt der Scanner nicht an. if (TaskAssignee.KindOf(task.Assignee) == TaskAssigneeKind.Human) continue; // Eine Aufgabe mit unauflösbarer Zeitzone feuert nie. Das einmal melden, // sonst sucht man den Fehler bei der Aufgabe statt beim System. if (TaskSchedule.UnresolvableTimeZone(task) is { } badZone && _warnedTimeZones.Add(task.Id)) { _logger.LogWarning( "Aufgabe {TaskId} ({Title}) hat die Zeitzone '{TimeZone}', die auf diesem " + "System nicht auflösbar ist — sie wird nicht ausgeführt. Zeitzone in " + "IANA-Schreibweise eintragen (z.B. 'Europe/Berlin') und sicherstellen, " + "dass tzdata und ICU vorhanden sind.", task.Id, task.Title, badZone); } // Dasselbe für einen Cron-Ausdruck, den der Parser nicht versteht (#597). if (TaskSchedule.InvalidCron(task) is { } cronError && _warnedCron.Add(task.Id)) { _logger.LogWarning( "Aufgabe {TaskId} ({Title}) hat den Termin cron \"{Cron}\", der nicht lesbar ist " + "— sie wird nicht ausgeführt. {Fehler}", task.Id, task.Title, task.When!.Value, cronError); } // #598: Laeuft, aber teurer mit jedem Lauf - einmal sagen, nicht bei jedem Takt. if (TaskHinweise.WachsenderKontext(task) is { } hinweis && _warnedContext.Add(task.Id)) { _logger.LogWarning("Aufgabe {TaskId} ({Title}): {Hinweis}", task.Id, task.Title, hinweis); } if (_zurueckgestellt.TryGetValue(task.Id, out var ab) && now < ab) continue; // #716: Refused for lack of budget — not before the next day. For a recurring // task this only holds back the catch-up; its regular occurrences keep coming. var deferred = task.DeferredUntil is { } until && now < until; if (deferred && !task.IsRecurring) continue; if (task.OnlyWhenMarketOpen && !_market.IsOpen(now)) continue; var occ = TaskSchedule.DueOccurrence(task, now); // #717: Fill work waits for room — until its deadline. Past it, the deadline was // the promise, and it runs like any other due work. if (task.IsFillWork && occ is not null) { await AskAboutFillIfDueAsync(task, now, ct); if (now < task.FillBy!.Value) { due.Add((task, occ, RunKind.Fill)); continue; } if (_warnedOverdueFill.Add(task.Id)) _logger.LogWarning( "Fill work {TaskId} ({Title}) reached its deadline {Deadline:yyyy-MM-dd HH:mm} UTC without " + "finding room — it now runs as regular work", task.Id, task.Title, task.FillBy); } if (occ is not null) { // A regular occurrence also covers a pending catch-up: one run, not two. due.Add((task, occ, task.IsCarriedOver ? RunKind.Late : RunKind.Regular)); continue; } if (task.CatchUpPending && !deferred) due.Add((task, Iso(now), RunKind.Late)); } // #716: The new day has right of way — carried-over and catch-up runs come after // everything regularly due, and fill work (#717) after those: late work was due, // fill work never was. Within each group: priority, then the oldest occurrence — // for fill work the nearest deadline first. Occurrence keys are ISO-8601 UTC ("O"), // so ordinal order is time order. due.Sort((a, b) => { if (a.Kind != b.Kind) return a.Kind.CompareTo(b.Kind); if (a.Kind == RunKind.Fill && Nullable.Compare(a.Task.FillBy, b.Task.FillBy) is var byDeadline and not 0) return byDeadline; var byPriority = b.Task.Priority.CompareTo(a.Task.Priority); return byPriority != 0 ? byPriority : string.CompareOrdinal(a.Occurrence, b.Occurrence); }); var launched = new List(); var waiting = 0; var waitingForHeadroom = 0; var waitingFill = 0; // #717: Agents with regular work that had to wait this tick — their fill work waits too. var agentsWithWaitingWork = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (var (task, occ, kind) in due) { // OpenCount also counts runs started by hand (RunTaskNowAsync) — they take a // slot, even though they did not wait for one. if (OpenCount >= _maxParallelRuns) { waiting++; if (kind != RunKind.Fill) agentsWithWaitingWork.Add(TaskAssignee.AgentId(task.Assignee)); continue; } // #716: Late work only uses what the rest of the day does not need. if (kind == RunKind.Late && _lateRunGate is not null && !await _lateRunGate.HasRoomAsync(task, candidates, now, ct)) { waitingForHeadroom++; agentsWithWaitingWork.Add(TaskAssignee.AgentId(task.Assignee)); continue; } // #717: Fill work only when nothing of the agent's own waits, and only in room // the day would otherwise leave unused. That is decided here, not by the model. if (kind == RunKind.Fill) { if (agentsWithWaitingWork.Contains(TaskAssignee.AgentId(task.Assignee)) || (_fillGate is not null && await _fillGate.CheckAsync(task, candidates, now, ct) != FillVerdict.Run)) { waitingFill++; continue; } } var token = Guid.NewGuid().ToString("N"); if (await _repo.TryClaimAsync(task.Id, occ, token, now, leaseCutoff, ct)) launched.Add(Track(RunLoggedAsync(task, token, ct))); } if (waitingFill != _lastWaitingFill) { if (waitingFill > 0) _logger.LogInformation( "{Waiting} fill task(s) waiting for unused budget — they run when the rest of the day leaves room", waitingFill); _lastWaitingFill = waitingFill; } // A cap that hides its effect is worse than none (the lesson of #673): say when // due work had to wait, but only when that changes — the tick runs every minute. if (waiting != _lastWaiting) { if (waiting > 0) _logger.LogInformation( "{Waiting} due task(s) waiting for a free slot — {Max} run(s) at a time (maxParallelTaskRuns)", waiting, _maxParallelRuns); else _logger.LogInformation("No due tasks waiting for a slot anymore"); _lastWaiting = waiting; } if (waitingForHeadroom != _lastWaitingForHeadroom) { if (waitingForHeadroom > 0) _logger.LogInformation( "{Waiting} carried-over or catch-up run(s) waiting for budget headroom — " + "the rest of today's schedule comes first", waitingForHeadroom); _lastWaitingForHeadroom = waitingForHeadroom; } return launched; } /// /// #717: Fill work that has not found room by deadline - warnBefore is taken to a /// human once: raise the budget or move the deadline. Still waiting at that point means /// it has not run — a run that took place would have finished it. /// private async Task AskAboutFillIfDueAsync(TaskItem task, DateTime now, CancellationToken ct) { if (_askAboutFill is null || task.FillAlertedAt is not null || task.FillBy is not { } deadline || now < deadline - _fillWarnBefore()) return; // Marked first: asking twice is worse than asking once and failing to log it. if (!await _repo.MarkFillAlertedAsync(task.Id, now, ct)) return; try { await _askAboutFill(task, ct); _logger.LogWarning( "Fill work {TaskId} ({Title}) has found no room and is due by {Deadline:yyyy-MM-dd HH:mm} UTC — " + "a human was asked to raise the budget or move the deadline", task.Id, task.Title, deadline); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogError(ex, "Could not ask a human about fill work {TaskId}", task.Id); } } /// /// Runs a claimed task and logs what escapes it. The tick loop does not await its runs, /// so an exception here would otherwise go unobserved. /// private async Task RunLoggedAsync(TaskItem task, string token, CancellationToken ct) { try { await ProcessAsync(task, token, ct); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { // Shutdown. The claim is released by the lease or the next start's reconciliation. } catch (Exception ex) { _logger.LogError(ex, "Run of task {TaskId} failed outside the dispatch", task.Id); } } private Task Track(Task run) { lock (_inFlight) _inFlight.Add(run); _ = run.ContinueWith(t => { lock (_inFlight) _inFlight.Remove(t); }, TaskScheduler.Default); return run; } /// /// Führt einen Task sofort aus (manueller „Jetzt ausführen"-Knopf), unabhängig vom /// Termin. Beansprucht ihn atomar wie ein Takt — läuft er schon oder ist er kein /// todo, gibt die Methode false zurück, statt ihn doppelt anzustoßen. /// public async Task RunTaskNowAsync(string taskId, CancellationToken ct) { if (!IsAcceptingWork) return false; var task = await _repo.GetAsync(taskId, ct); if (task is null) return false; var now = _clock.GetUtcNow().UtcDateTime; var token = Guid.NewGuid().ToString("N"); if (!await _repo.TryClaimAsync(task.Id, now.ToString("O"), token, now, now - _lease, ct)) return false; await ProcessAsync(task, token, ct); return true; } private async Task ProcessAsync(TaskItem task, string token, CancellationToken ct) { // Ab hier ist die Aufgabe beansprucht und steht auf `running`. Wer das // Herunterfahren abwartet, wartet genau auf diesen Zaehler - denn erst // CompleteClaimAsync weiter unten gibt sie wieder frei. Interlocked.Increment(ref _openRuns); try { await ProcessCoreAsync(task, token, ct); } finally { Interlocked.Decrement(ref _openRuns); } } private async Task ProcessCoreAsync(TaskItem task, string token, CancellationToken ct) { // #717: A large task is not simply started — its run plans first. var planning = await PlanningInstructionAsync(task, ct); TaskLauf lauf; try { lauf = planning is null ? await _dispatcher.DispatchAsync(task, ct) : await _dispatcher.DispatchAsync(task, planning, ct); } catch (OperationCanceledException) when (ct.IsCancellationRequested) { throw; } catch (Exception ex) { _logger.LogError(ex, "Dispatch für Aufgabe {TaskId} ist gescheitert", task.Id); lauf = TaskLauf.Gescheitert; } var now = _clock.GetUtcNow().UtcDateTime; var ok = lauf == TaskLauf.Erledigt; if (lauf == TaskLauf.KeinBudget) { await HandleBudgetMissAsync(task, token, now, ct); return; } // #643: Fand der Lauf einer einmaligen Aufgabe gar nicht statt, gilt ihr Termin als // nicht verbraucht. Vorher blieb sie mit gesetztem Marker für immer auf todo liegen. if (lauf == TaskLauf.NichtGelaufen && !task.IsRecurring) { if (await _repo.ReleaseClaimAsync(task.Id, token, task.LastOccurrence, now, ct)) { _zurueckgestellt[task.Id] = now + Wiedervorlage; _logger.LogInformation( "Aufgabe {TaskId} lief nicht (Budget/Herunterfahren) — erneut fällig ab {Ab:HH:mm} UTC", task.Id, now + Wiedervorlage); } return; } _zurueckgestellt.TryRemove(task.Id, out _); if (planning is not null) { await SettlePlanningAsync(task, token, lauf, now, ct); return; } // Endstatus: // - Fehlschlag → zurück auf todo (der Marker steht schon beim Claim, derselbe Termin // wird nicht wiederholt). // - Wiederkehrend (Cron/every/tool_job) → zurück auf todo, damit der nächste Termin // feuern kann. Sonst liefe ein Cron-Task nur ein einziges Mal. // - Einmalig → done bzw. in_review (A2). var final = !ok || task.IsRecurring ? TaskItemStatus.Todo : task.RequireApproval ? TaskItemStatus.InReview : TaskItemStatus.Done; // #689/#715: A status the run set itself is the run's word and is kept. An agent that // escalates, or sets its task to blocked or canceled to report up, would otherwise see // it overwritten with "done" the moment its run ends — and the report would be lost. if (await _repo.GetAsync(task.Id, ct) is { Status: TaskItemStatus.Blocked or TaskItemStatus.Canceled } during) final = during.Status; await _repo.CompleteClaimAsync(task.Id, token, final, now, ct); // #716: A run took place — earlier budget misses are settled. if (lauf != TaskLauf.NichtGelaufen && (task.BudgetMissCount > 0 || task.CatchUpPending || task.DeferredUntil is not null)) await _repo.ClearBudgetMissAsync(task.Id, now, ct); if (ok && final == TaskItemStatus.Done) { await UnblockDependentsAsync(task.Id, now, ct); await ResumeLargeParentAsync(task, now, ct); } } /// /// #717: The planning instruction for a large one-time task that has not been planned yet. /// A failing check costs the check, not the run. /// private async Task PlanningInstructionAsync(TaskItem task, CancellationToken ct) { if (_largeCheck is null || task.IsRecurring || task.LargePlannedAt is not null || task.Type == TaskItemType.ToolJob) return null; try { return await _largeCheck.PlanningInstructionAsync(task, ct); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Large-task check failed for {TaskId} — it runs as usual", task.Id); return null; } } /// /// #717: After the planning run of a large task. Split into parts → it waits for them. /// A special budget approved meanwhile → it runs. Otherwise it waits, blocked, for the /// human's decision on the budget — never silently: the task says why. /// /// The occurrence is given back first: the planning run did not do the work, and a /// one-time task whose marker is used up would never run again. /// private async Task SettlePlanningAsync(TaskItem task, string token, TaskLauf lauf, DateTime now, CancellationToken ct) { if (!await _repo.ReleaseClaimAsync(task.Id, token, task.LastOccurrence, now, ct)) return; if (lauf != TaskLauf.Erledigt) return; // the planning did not take place or failed — the next due tick plans again await _repo.MarkLargePlannedAsync(task.Id, now, ct); var parts = (await _repo.ListChildrenAsync(task.Id, ct)).Count(IsOpen); string? note = null; if (parts > 0) { await _repo.SetStatusAsync(task.Id, TaskItemStatus.Blocked, now, ct); note = $"Large task: split into {parts} part(s). It runs again once they are done."; } else if (_largeCheck is not null && await _largeCheck.IsFundedAsync(task, ct)) { _logger.LogInformation("Large task {TaskId} has a special budget — it runs on it", task.Id); } else { await _repo.SetStatusAsync(task.Id, TaskItemStatus.Blocked, now, ct); note = "Large task: not started. It waits for a special budget to be approved (staging), " + "or for a human to split it or set it back to todo."; } if (note is not null) { _logger.LogInformation("Task {TaskId}: {Note}", task.Id, note); if (_noteOnTask is not null) { try { await _noteOnTask(task.Id, note, ct); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Could not note on task {TaskId}", task.Id); } } } } /// /// #717: A part of a split large task is done. When it was the last open one, the large /// task is taken up again — to bring the parts together. /// private async Task ResumeLargeParentAsync(TaskItem part, DateTime now, CancellationToken ct) { if (string.IsNullOrWhiteSpace(part.ParentId) || await _repo.GetAsync(part.ParentId, ct) is not { Status: TaskItemStatus.Blocked, LargePlannedAt: not null } parent) return; if ((await _repo.ListChildrenAsync(parent.Id, ct)).Any(IsOpen)) return; await _repo.SetStatusAsync(parent.Id, TaskItemStatus.Todo, now, ct); _logger.LogInformation("Large task {TaskId}: all parts done — it runs again to bring them together", parent.Id); } private static bool IsOpen(TaskItem task) => task.Status is not (TaskItemStatus.Done or TaskItemStatus.Canceled or TaskItemStatus.Archived); /// /// #716: The budget refused the run. A one-time task is carried over to the next day /// with its occurrence intact; a recurring one follows the agent's policy — the /// occurrence lapses, and under one /// catch-up run is scheduled for the next day. /// private async Task HandleBudgetMissAsync(TaskItem task, string token, DateTime now, CancellationToken ct) { var nextDay = StartOfNextLocalDay(now); var agent = TaskAssignee.AgentId(task.Assignee); var carried = false; var catchUp = false; if (!task.IsRecurring) { carried = await _repo.CarryOverAsync(task.Id, token, task.LastOccurrence, nextDay, now, ct); } else { catchUp = _missPolicy(task) == BudgetMissPolicy.CatchUpOnce; await _repo.RecordBudgetMissAsync(task.Id, token, catchUp ? nextDay : null, now, ct); } lock (_missesToday) { var entry = _missesToday.GetValueOrDefault(agent); _missesToday[agent] = (entry.Misses + 1, entry.CarriedOver + (carried ? 1 : 0), entry.CatchUps + (catchUp && !task.CatchUpPending ? 1 : 0)); } _logger.LogInformation( "Task {TaskId} refused for lack of budget — {Outcome}", task.Id, carried ? $"carried over, due again from {nextDay:yyyy-MM-dd HH:mm} UTC" : catchUp ? "occurrence lapsed, one catch-up run scheduled for the next day" : "occurrence lapsed (policy: expire)"); } /// /// #716: Prevent rather than catch up. Every half hour the day ahead is checked; an /// agent whose budget will not last until midnight is named once a day — with the time /// it runs out and how many planned runs that costs — while there is still time to /// raise the budget or decide what can go. /// private async Task WarnAboutShortfallsAsync(IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct) { if (_outlook is null || nowUtc - _lastOutlookAt < OutlookInterval) return; _lastOutlookAt = nowUtc; IReadOnlyList shortfalls; try { shortfalls = await _outlook.ShortfallsAsync(runnable, nowUtc, ct); } catch (Exception ex) when (ex is not OperationCanceledException) { _logger.LogWarning(ex, "Budget outlook failed — no forecast this time"); return; } var today = DateOnly.FromDateTime(TimeZoneInfo.ConvertTimeFromUtc(nowUtc, _clock.LocalTimeZone)); foreach (var shortfall in shortfalls) { if (_shortfallWarned.TryGetValue(shortfall.AgentId, out var warnedOn) && warnedOn == today) continue; _shortfallWarned[shortfall.AgentId] = today; var local = TimeZoneInfo.ConvertTimeFromUtc(shortfall.ExhaustsAtUtc, _clock.LocalTimeZone); _logger.LogWarning( "Budget forecast: the daily budget of {Who} will run out around {Time:HH:mm} — " + "{Refused} of {Planned} runs planned for the rest of today would be refused. " + "Raise the budget or decide what can wait.", shortfall.AgentId.Length == 0 ? "the instance" : $"agent '{shortfall.AgentId}'", local, shortfall.RunsRefused, shortfall.RunsPlanned); } } /// /// #716: On the first tick of a new local day, one line per agent about what the /// budget refused the day before. The details are on the tasks themselves. /// private void SummarizeMissesIfNewDay(DateTime nowUtc) { var today = DateOnly.FromDateTime(TimeZoneInfo.ConvertTimeFromUtc(nowUtc, _clock.LocalTimeZone)); List> summary; DateOnly? day; lock (_missesToday) { if (_missDay is null) { _missDay = today; return; } if (_missDay == today) return; day = _missDay; summary = [.. _missesToday]; _missesToday.Clear(); _missDay = today; } foreach (var (agent, counts) in summary) { _logger.LogWarning( "Budget summary {Day:yyyy-MM-dd} for agent {AgentId}: {Misses} run(s) refused for lack of " + "budget, {Carried} task(s) carried over, {CatchUps} catch-up run(s) scheduled", day, agent.Length == 0 ? "-" : agent, counts.Misses, counts.CarriedOver, counts.CatchUps); } } /// /// Local midnight after , in UTC. The budget day follows the /// machine's local date, like . /// private DateTime StartOfNextLocalDay(DateTime nowUtc) { var zone = _clock.LocalTimeZone; var local = TimeZoneInfo.ConvertTimeFromUtc(nowUtc, zone); var midnight = DateTime.SpecifyKind(local.Date.AddDays(1), DateTimeKind.Unspecified); // A midnight skipped by a DST change does not exist; take the next valid minute. while (zone.IsInvalidTime(midnight)) midnight = midnight.AddMinutes(1); return TimeZoneInfo.ConvertTimeToUtc(midnight, zone); } private static string Iso(DateTime utc) => utc.ToUniversalTime().ToString("O"); /// The order in which due work is started: what is due, what is late, what fills. private enum RunKind { Regular, Late, Fill } /// /// Auto-Dispatch: Wird eine Aufgabe fertig, werden die auf sie wartenden Aufgaben /// freigegeben, sobald alle ihre Blocker erledigt sind. /// private async Task UnblockDependentsAsync(string completedId, DateTime now, CancellationToken ct) { var dependents = await _repo.ListBlockedByAsync(completedId, ct); foreach (var dependent in dependents) { if (dependent.Status != TaskItemStatus.Blocked) continue; var allDone = true; foreach (var blockerId in dependent.BlockedBy) { var blocker = await _repo.GetAsync(blockerId, ct); if (blocker is null || blocker.Status != TaskItemStatus.Done) { allDone = false; break; } } if (allDone) { await _repo.SetStatusAsync(dependent.Id, TaskItemStatus.Todo, now, ct); _logger.LogInformation( "Aufgabe {TaskId} freigegeben — alle Blocker erledigt", dependent.Id); } } } public async ValueTask DisposeAsync() { await _cts.CancelAsync(); if (_loop is not null) { try { await _loop; } catch (OperationCanceledException) { } } // #682: The loop no longer waits for its runs, so wait for them here — otherwise // a run could still write to the repository after the scanner is gone. Task[] inFlight; lock (_inFlight) inFlight = [.. _inFlight]; await Task.WhenAll(inFlight); _cts.Dispose(); } }