From 5f3a85595b6f410daaad1e74941b5fe5103280ae Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 24 Sep 2026 12:03:15 +0200 Subject: [PATCH 1/4] Budgetende, Stufe 1: Ausfall vermerken, Regel je Agent, mitnehmen statt anrennen (#716) Bisher teilte sich eine Budgetablehnung den Ausgang NichtGelaufen mit dem Herunterfahren. Einmalige Aufgaben wurden den Rest des Tages alle 30 Minuten erneut abgelehnt (jedes Mal ein Beleg) und liefen nach Mitternacht still zu Lasten des neuen Tages. Wiederkehrende verfielen still. - TaskLauf.KeinBudget als eigener Ausgang; der Dispatcher bildet BudgetExceeded darauf ab. - Vier Felder an der Aufgabe: BudgetMissCount, LastBudgetMissAt, DeferredUntil, CatchUpPending (Spalten werden fuer bestehende Datenbanken nachgeruestet). - Einmalige Aufgaben werden mitgenommen: Der Termin bleibt erhalten, faellig ist die Aufgabe wieder ab der naechsten lokalen Mitternacht, nicht vorher. - Wiederkehrende Aufgaben folgen der Regel des Agenten (budget.whenExhausted): expire vermerkt den Ausfall und laesst den Termin verfallen; catchUpOnce fasst alle verpassten Termine zu einem Nachholauf am naechsten Tag zusammen. Ein regulaer faelliger Termin deckt das Nachholen mit ab, es bleibt bei einem Lauf. - Der neue Tag hat Vorfahrt: Mitgenommenes und Nachholaeufe kommen nach allem regulaer Faelligen, unabhaengig von der Prioritaet. - Beim ersten Takt eines neuen Tages eine Zusammenfassung je Agent im Log. - Sichtbar: Die Aufgabenkarte der WebUI zeigt Ausfaelle, Mitnahme und Nachholen; die Regel ist im Desktop je Agent einstellbar. Nebenbei sind "Next:"/"Last:" auf der Karte ins Sprachpaket gewandert. Stufen 2 bis 4 (Reservierung, Spielraum statt pauschalem Anteil, Vorbeugen) folgen. Tests: BudgetMissTests (9 neu). 1.238 gruen. Co-Authored-By: Claude Opus 5.5 --- src/ClawdDotNet.App/AppHost.cs | 10 +- src/ClawdDotNet.Core/Config/AgentConfig.cs | 15 + src/ClawdDotNet.Core/Storage/SqliteStorage.cs | 6 + .../Tasks/EngineTaskDispatcher.cs | 3 +- src/ClawdDotNet.Core/Tasks/ITaskRepository.cs | 22 ++ .../Tasks/SqliteTaskRepository.cs | 71 ++++- src/ClawdDotNet.Core/Tasks/TaskModels.cs | 39 +++ src/ClawdDotNet.Core/Tasks/TaskScanner.cs | 151 +++++++++- .../ViewModels/AgentsPageViewModel.cs | 10 + .../Views/AgentsPageView.axaml | 9 + src/ClawdDotNet.Desktop/i18n/de.json | 2 + src/ClawdDotNet.Desktop/i18n/en.json | 2 + .../wwwroot/i18n/de.json | 6 + .../wwwroot/i18n/en.json | 6 + .../wwwroot/views/tasks.js | 17 +- src/ClawdDotNet.Web/Contracts/Dtos.cs | 9 +- src/ClawdDotNet.Web/Contracts/Mapping.cs | 4 + .../Tasks/BudgetMissTests.cs | 257 ++++++++++++++++++ 18 files changed, 624 insertions(+), 15 deletions(-) create mode 100644 tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs diff --git a/src/ClawdDotNet.App/AppHost.cs b/src/ClawdDotNet.App/AppHost.cs index 1acb05b..675cf46 100644 --- a/src/ClawdDotNet.App/AppHost.cs +++ b/src/ClawdDotNet.App/AppHost.cs @@ -726,7 +726,15 @@ public sealed class AppHost : IAsyncDisposable engine, () => instance.Agents, instance.InstanceId, tools, stateStore, loggerFactory); var scanner = new TaskScanner(taskRepository, dispatcher, loggerFactory, - maxParallelRuns: instance.MaxParallelTaskRuns); + maxParallelRuns: instance.MaxParallelTaskRuns, + budgetMissPolicy: task => + { + // #716: The agent's rule. Bare '@new' names no agent — then the default applies. + var agentId = TaskAssignee.AgentId(task.Assignee); + return instance.Agents + .FirstOrDefault(a => string.Equals(a.AgentId, agentId, StringComparison.OrdinalIgnoreCase)) + ?.Budget.MissPolicy ?? BudgetMissPolicy.Expire; + }); // Reconciliation nicht blockierend: Der Start soll nicht auf das Dateisystem warten. _ = Task.Run(async () => diff --git a/src/ClawdDotNet.Core/Config/AgentConfig.cs b/src/ClawdDotNet.Core/Config/AgentConfig.cs index c851282..1bbbd92 100644 --- a/src/ClawdDotNet.Core/Config/AgentConfig.cs +++ b/src/ClawdDotNet.Core/Config/AgentConfig.cs @@ -183,6 +183,21 @@ public sealed class AgentBudget [JsonPropertyName("dailyTokens")] public long DailyTokens { get; set; } + /// + /// #716: What happens to a run of a recurring task when the budget is exhausted — + /// expire (default) or catchUpOnce. One-time tasks are always carried + /// over to the next day. Kept as text so an unknown value falls back instead of + /// failing the whole configuration. + /// + [JsonPropertyName("whenExhausted")] + public string WhenExhausted { get; set; } = "expire"; + + [JsonIgnore] + public Tasks.BudgetMissPolicy MissPolicy + => string.Equals(WhenExhausted?.Trim(), "catchUpOnce", StringComparison.OrdinalIgnoreCase) + ? Tasks.BudgetMissPolicy.CatchUpOnce + : Tasks.BudgetMissPolicy.Expire; + [JsonIgnore] public bool HasAnyLimit => DailyCostUsd > 0 || DailyTokens > 0; } diff --git a/src/ClawdDotNet.Core/Storage/SqliteStorage.cs b/src/ClawdDotNet.Core/Storage/SqliteStorage.cs index e54e4cb..712da89 100644 --- a/src/ClawdDotNet.Core/Storage/SqliteStorage.cs +++ b/src/ClawdDotNet.Core/Storage/SqliteStorage.cs @@ -339,6 +339,12 @@ public sealed class SqliteStorage // #554: Paart den "angefangen"-Eintrag mit seinem Abschluss. SpalteErgaenzen(connection, "AuditLog", "CallId", "TEXT NOT NULL DEFAULT ''"); + // #716: What happened when the budget refused a run. + SpalteErgaenzen(connection, "Tasks", "BudgetMissCount", "INTEGER NOT NULL DEFAULT 0"); + SpalteErgaenzen(connection, "Tasks", "LastBudgetMissAt", "TEXT NULL"); + SpalteErgaenzen(connection, "Tasks", "DeferredUntil", "TEXT NULL"); + SpalteErgaenzen(connection, "Tasks", "CatchUpPending", "INTEGER NOT NULL DEFAULT 0"); + // Findet offene Aufrufe beim Start, ohne die ganze Tabelle zu lesen. using var index = connection.CreateCommand(); index.CommandText = """ diff --git a/src/ClawdDotNet.Core/Tasks/EngineTaskDispatcher.cs b/src/ClawdDotNet.Core/Tasks/EngineTaskDispatcher.cs index 3164069..90bf206 100644 --- a/src/ClawdDotNet.Core/Tasks/EngineTaskDispatcher.cs +++ b/src/ClawdDotNet.Core/Tasks/EngineTaskDispatcher.cs @@ -98,7 +98,8 @@ public sealed class EngineTaskDispatcher : ITaskDispatcher return result.Status switch { AgentRunStatus.Completed => TaskLauf.Erledigt, - AgentRunStatus.BudgetExceeded or AgentRunStatus.ShuttingDown => TaskLauf.NichtGelaufen, + AgentRunStatus.BudgetExceeded => TaskLauf.KeinBudget, + AgentRunStatus.ShuttingDown => TaskLauf.NichtGelaufen, _ => TaskLauf.Gescheitert }; } diff --git a/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs b/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs index 559fc2b..a8cadc2 100644 --- a/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs +++ b/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs @@ -65,6 +65,28 @@ public interface ITaskRepository /// Ändert den Status direkt (Tool-Aktionen, Auto-Dispatch, Eskalation). Task SetStatusAsync(string id, TaskItemStatus status, DateTime now, CancellationToken ct); + /// + /// #716: The budget refused a run of a one-time task. Releases the claim, restores the + /// occurrence (the task is still due) and defers it to . + /// + Task CarryOverAsync( + string id, string claimToken, string? previousOccurrence, DateTime deferredUntil, + DateTime now, CancellationToken ct); + + /// + /// #716: The budget refused a run of a recurring task. Releases the claim back to + /// todo; the occurrence stays consumed. With a + /// single catch-up run is scheduled for then — an already pending one is kept. + /// + Task RecordBudgetMissAsync( + string id, string claimToken, DateTime? catchUpUntil, DateTime now, CancellationToken ct); + + /// + /// #716: A run took place — clears the miss counter, a pending catch-up and the + /// deferral. stays as history. + /// + Task ClearBudgetMissAsync(string id, DateTime now, CancellationToken ct); + /// /// Reconciliation beim Start: verwaiste Claims (älter als ) /// lösen und die betroffenen Aufgaben von zurück diff --git a/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs b/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs index d4ee2cd..dba39e2 100644 --- a/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs +++ b/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs @@ -193,6 +193,68 @@ public sealed class SqliteTaskRepository : ITaskRepository return await cmd.ExecuteNonQueryAsync(ct) > 0; }, ct); + public Task CarryOverAsync( + string id, string claimToken, string? previousOccurrence, DateTime deferredUntil, + DateTime now, CancellationToken ct) + => _storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + UPDATE Tasks + SET Status = 'todo', ClaimToken = NULL, ClaimedAt = NULL, + LastOccurrence = @occ, UpdatedAt = @now, + BudgetMissCount = BudgetMissCount + 1, LastBudgetMissAt = @now, + DeferredUntil = @deferred + WHERE Id = @id AND ClaimToken = @token + """; + cmd.Parameters.AddWithValue("@occ", (object?)previousOccurrence ?? DBNull.Value); + cmd.Parameters.AddWithValue("@now", Format(now)); + cmd.Parameters.AddWithValue("@deferred", Format(deferredUntil)); + cmd.Parameters.AddWithValue("@id", id); + cmd.Parameters.AddWithValue("@token", claimToken); + + return await cmd.ExecuteNonQueryAsync(ct) > 0; + }, ct); + + public Task RecordBudgetMissAsync( + string id, string claimToken, DateTime? catchUpUntil, DateTime now, CancellationToken ct) + => _storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + // A pending catch-up keeps its date: several misses still mean one catch-up run. + cmd.CommandText = """ + UPDATE Tasks + SET Status = 'todo', ClaimToken = NULL, ClaimedAt = NULL, UpdatedAt = @now, + BudgetMissCount = BudgetMissCount + 1, LastBudgetMissAt = @now, + CatchUpPending = CASE WHEN @catchUp = 1 THEN 1 ELSE CatchUpPending END, + DeferredUntil = CASE WHEN @catchUp = 1 AND CatchUpPending = 0 THEN @deferred + ELSE DeferredUntil END + WHERE Id = @id AND ClaimToken = @token + """; + cmd.Parameters.AddWithValue("@now", Format(now)); + cmd.Parameters.AddWithValue("@catchUp", catchUpUntil is null ? 0 : 1); + cmd.Parameters.AddWithValue("@deferred", catchUpUntil is { } until ? Format(until) : DBNull.Value); + cmd.Parameters.AddWithValue("@id", id); + cmd.Parameters.AddWithValue("@token", claimToken); + + return await cmd.ExecuteNonQueryAsync(ct) > 0; + }, ct); + + public Task ClearBudgetMissAsync(string id, DateTime now, CancellationToken ct) + => _storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + UPDATE Tasks + SET BudgetMissCount = 0, CatchUpPending = 0, DeferredUntil = NULL, UpdatedAt = @now + WHERE Id = @id + """; + cmd.Parameters.AddWithValue("@now", Format(now)); + cmd.Parameters.AddWithValue("@id", id); + + return await cmd.ExecuteNonQueryAsync(ct) > 0; + }, ct); + public Task ReleaseStaleClaimsAsync(DateTime leaseCutoff, DateTime now, CancellationToken ct) => _storage.WriteAsync(async conn => { @@ -252,7 +314,8 @@ public sealed class SqliteTaskRepository : ITaskRepository private const string Columns = "Id, Title, Status, Type, Priority, Assignee, WhenKind, WhenValue, WhenTz, " + "RequireApproval, Acceptance, BlockedBy, OnlyWhenMarketOpen, Body, FileName, " + - "ToolName, JobTypeId, LastOccurrence, ClaimToken, ClaimedAt, CreatedAt, UpdatedAt"; + "ToolName, JobTypeId, LastOccurrence, ClaimToken, ClaimedAt, CreatedAt, UpdatedAt, " + + "BudgetMissCount, LastBudgetMissAt, DeferredUntil, CatchUpPending"; private static void BindDefinition(SqliteCommand cmd, TaskItem task) { @@ -318,7 +381,11 @@ public sealed class SqliteTaskRepository : ITaskRepository ClaimToken = r.IsDBNull(18) ? null : r.GetString(18), ClaimedAt = r.IsDBNull(19) ? null : Parse(r.GetString(19)), CreatedAt = Parse(r.GetString(20)), - UpdatedAt = Parse(r.GetString(21)) + UpdatedAt = Parse(r.GetString(21)), + BudgetMissCount = r.GetInt32(22), + LastBudgetMissAt = r.IsDBNull(23) ? null : Parse(r.GetString(23)), + DeferredUntil = r.IsDBNull(24) ? null : Parse(r.GetString(24)), + CatchUpPending = r.GetInt32(25) != 0 }; } diff --git a/src/ClawdDotNet.Core/Tasks/TaskModels.cs b/src/ClawdDotNet.Core/Tasks/TaskModels.cs index 31e390b..9f75f22 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskModels.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskModels.cs @@ -205,6 +205,32 @@ public sealed record TaskItem public DateTime CreatedAt { get; init; } public DateTime UpdatedAt { get; init; } + // ─── Budget misses (#716) ─── + + /// + /// How many times in a row a run of this task was refused because the budget was + /// exhausted. Reset by the next run that actually takes place. + /// + public int BudgetMissCount { get; init; } + + /// When the last budget refusal happened (UTC). Kept after a reset, as history. + public DateTime? LastBudgetMissAt { get; init; } + + /// + /// Not before this point (UTC). Set after a budget refusal to the start of the next + /// local day: retrying within the same day only produces further refusals. + /// + public DateTime? DeferredUntil { get; init; } + + /// + /// A recurring task under missed at least + /// one occurrence; all of them are caught up by a single run. + /// + public bool CatchUpPending { get; init; } + + /// A one-time task that was refused for lack of budget and carried over. + public bool IsCarriedOver => !IsRecurring && BudgetMissCount > 0; + /// /// Wiederkehrend? Solche Tasks kehren nach dem Feuern auf todo zurück statt auf /// done — sonst würde ein Cron-Task nur ein einziges Mal laufen. Ein @@ -215,6 +241,19 @@ public sealed record TaskItem || When is { Kind: TaskWhenKind.Cron or TaskWhenKind.Every }; } +/// +/// #716: What happens to a run of a recurring task that the budget refused. One-time +/// tasks are always carried over — they are real work that must not disappear. +/// +public enum BudgetMissPolicy +{ + /// The occurrence lapses and is recorded; the next regular one comes anyway. + Expire, + + /// All missed occurrences are caught up by one run on the next day. + CatchUpOnce +} + /// Suchkriterien für . public sealed record TaskQuery { diff --git a/src/ClawdDotNet.Core/Tasks/TaskScanner.cs b/src/ClawdDotNet.Core/Tasks/TaskScanner.cs index 288d29b..c1571c6 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskScanner.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskScanner.cs @@ -12,10 +12,17 @@ public enum TaskLauf Gescheitert, /// - /// Der Lauf fand gar nicht statt — Budget erschöpft, Instanz im Herunterfahren (#643). + /// Der Lauf fand gar nicht statt — Instanz im Herunterfahren (#643). /// Eine einmalige Aufgabe bleibt dann fällig, statt als gelaufen liegenzubleiben. /// - NichtGelaufen + NichtGelaufen, + + /// + /// #716: The run did not take place because the budget is exhausted. Unlike a + /// shutdown, retrying the same day is pointless — the scanner defers to the next day + /// and applies the agent's . + /// + KeinBudget } /// @@ -98,6 +105,11 @@ public sealed class TaskScanner : IAsyncDisposable /// How many due tasks had to wait at the last tick — logged only on change. private int _lastWaiting; + /// #716: What the budget refused, per agent, for the day summary. + private readonly Func _missPolicy; + private readonly Dictionary _missesToday = new(StringComparer.OrdinalIgnoreCase); + private DateOnly? _missDay; + private readonly CancellationTokenSource _cts = new(); private Task? _loop; @@ -109,7 +121,8 @@ public sealed class TaskScanner : IAsyncDisposable TimeProvider? clock = null, TimeSpan? tick = null, TimeSpan? lease = null, - int maxParallelRuns = DefaultMaxParallelRuns) + int maxParallelRuns = DefaultMaxParallelRuns, + Func? budgetMissPolicy = null) { _repo = repo; _dispatcher = dispatcher; @@ -119,6 +132,7 @@ public sealed class TaskScanner : IAsyncDisposable _tick = tick ?? TimeSpan.FromSeconds(60); _lease = lease ?? TimeSpan.FromMinutes(15); _maxParallelRuns = Math.Max(1, maxParallelRuns); + _missPolicy = budgetMissPolicy ?? (_ => BudgetMissPolicy.Expire); } /// How many scheduled runs may be in flight at once (#682). @@ -226,7 +240,9 @@ public sealed class TaskScanner : IAsyncDisposable // #682: ungekappt. Vorher ListAsync mit Limit 500 — darüber fielen Termine lautlos aus. var candidates = await _repo.ListRunnableAsync(ct); - var due = new List<(TaskItem Task, string Occurrence)>(); + SummarizeMissesIfNewDay(now); + + var due = new List<(TaskItem Task, string Occurrence, bool Late)>(); foreach (var task in candidates) { if (task.Status != TaskItemStatus.Todo) @@ -269,26 +285,41 @@ public sealed class TaskScanner : IAsyncDisposable if (_zurueckgestellt.TryGetValue(task.Id, out var ab) && now < ab) continue; - var occ = TaskSchedule.DueOccurrence(task, now); - if (occ is null) + // #716: Refused for lack of budget — not before the next day. For a recurring + // task this only holds back the catch-up; its regular occurrences keep coming. + var deferred = task.DeferredUntil is { } until && now < until; + if (deferred && !task.IsRecurring) continue; if (task.OnlyWhenMarketOpen && !_market.IsOpen(now)) continue; - due.Add((task, occ)); + var occ = TaskSchedule.DueOccurrence(task, now); + if (occ is not null) + { + // A regular occurrence also covers a pending catch-up: one run, not two. + due.Add((task, occ, Late: task.IsCarriedOver)); + continue; + } + + if (task.CatchUpPending && !deferred) + due.Add((task, Iso(now), Late: true)); } + // #716: The new day has right of way — carried-over and catch-up runs come after + // everything regularly due. Within each group: priority, then the oldest occurrence. // Occurrence keys are ISO-8601 UTC ("O"), so ordinal order is time order. due.Sort((a, b) => { + if (a.Late != b.Late) + return a.Late ? 1 : -1; var byPriority = b.Task.Priority.CompareTo(a.Task.Priority); return byPriority != 0 ? byPriority : string.CompareOrdinal(a.Occurrence, b.Occurrence); }); var launched = new List(); var waiting = 0; - foreach (var (task, occ) in due) + foreach (var (task, occ, _) in due) { // OpenCount also counts runs started by hand (RunTaskNowAsync) — they take a // slot, even though they did not wait for one. @@ -413,6 +444,12 @@ public sealed class TaskScanner : IAsyncDisposable var now = _clock.GetUtcNow().UtcDateTime; var ok = lauf == TaskLauf.Erledigt; + if (lauf == TaskLauf.KeinBudget) + { + await HandleBudgetMissAsync(task, token, now, ct); + return; + } + // #643: Fand der Lauf einer einmaligen Aufgabe gar nicht statt, gilt ihr Termin als // nicht verbraucht. Vorher blieb sie mit gesetztem Marker für immer auf todo liegen. if (lauf == TaskLauf.NichtGelaufen && !task.IsRecurring) @@ -443,10 +480,108 @@ public sealed class TaskScanner : IAsyncDisposable await _repo.CompleteClaimAsync(task.Id, token, final, now, ct); + // #716: A run took place — earlier budget misses are settled. + if (lauf != TaskLauf.NichtGelaufen + && (task.BudgetMissCount > 0 || task.CatchUpPending || task.DeferredUntil is not null)) + await _repo.ClearBudgetMissAsync(task.Id, now, ct); + if (ok && final == TaskItemStatus.Done) await UnblockDependentsAsync(task.Id, now, ct); } + /// + /// #716: The budget refused the run. A one-time task is carried over to the next day + /// with its occurrence intact; a recurring one follows the agent's policy — the + /// occurrence lapses, and under one + /// catch-up run is scheduled for the next day. + /// + private async Task HandleBudgetMissAsync(TaskItem task, string token, DateTime now, CancellationToken ct) + { + var nextDay = StartOfNextLocalDay(now); + var agent = TaskAssignee.AgentId(task.Assignee); + var carried = false; + var catchUp = false; + + if (!task.IsRecurring) + { + carried = await _repo.CarryOverAsync(task.Id, token, task.LastOccurrence, nextDay, now, ct); + } + else + { + catchUp = _missPolicy(task) == BudgetMissPolicy.CatchUpOnce; + await _repo.RecordBudgetMissAsync(task.Id, token, catchUp ? nextDay : null, now, ct); + } + + lock (_missesToday) + { + var entry = _missesToday.GetValueOrDefault(agent); + _missesToday[agent] = (entry.Misses + 1, + entry.CarriedOver + (carried ? 1 : 0), + entry.CatchUps + (catchUp && !task.CatchUpPending ? 1 : 0)); + } + + _logger.LogInformation( + "Task {TaskId} refused for lack of budget — {Outcome}", + task.Id, + carried ? $"carried over, due again from {nextDay:yyyy-MM-dd HH:mm} UTC" + : catchUp ? "occurrence lapsed, one catch-up run scheduled for the next day" + : "occurrence lapsed (policy: expire)"); + } + + /// + /// #716: On the first tick of a new local day, one line per agent about what the + /// budget refused the day before. The details are on the tasks themselves. + /// + private void SummarizeMissesIfNewDay(DateTime nowUtc) + { + var today = DateOnly.FromDateTime(TimeZoneInfo.ConvertTimeFromUtc(nowUtc, _clock.LocalTimeZone)); + + List> summary; + DateOnly? day; + lock (_missesToday) + { + if (_missDay is null) + { + _missDay = today; + return; + } + if (_missDay == today) + return; + + day = _missDay; + summary = [.. _missesToday]; + _missesToday.Clear(); + _missDay = today; + } + + foreach (var (agent, counts) in summary) + { + _logger.LogWarning( + "Budget summary {Day:yyyy-MM-dd} for agent {AgentId}: {Misses} run(s) refused for lack of " + + "budget, {Carried} task(s) carried over, {CatchUps} catch-up run(s) scheduled", + day, agent.Length == 0 ? "-" : agent, counts.Misses, counts.CarriedOver, counts.CatchUps); + } + } + + /// + /// Local midnight after , in UTC. The budget day follows the + /// machine's local date, like . + /// + private DateTime StartOfNextLocalDay(DateTime nowUtc) + { + var zone = _clock.LocalTimeZone; + var local = TimeZoneInfo.ConvertTimeFromUtc(nowUtc, zone); + var midnight = DateTime.SpecifyKind(local.Date.AddDays(1), DateTimeKind.Unspecified); + + // A midnight skipped by a DST change does not exist; take the next valid minute. + while (zone.IsInvalidTime(midnight)) + midnight = midnight.AddMinutes(1); + + return TimeZoneInfo.ConvertTimeToUtc(midnight, zone); + } + + private static string Iso(DateTime utc) => utc.ToUniversalTime().ToString("O"); + /// /// Auto-Dispatch: Wird eine Aufgabe fertig, werden die auf sie wartenden Aufgaben /// freigegeben, sobald alle ihre Blocker erledigt sind. diff --git a/src/ClawdDotNet.Desktop/ViewModels/AgentsPageViewModel.cs b/src/ClawdDotNet.Desktop/ViewModels/AgentsPageViewModel.cs index e513bb8..ebbde54 100644 --- a/src/ClawdDotNet.Desktop/ViewModels/AgentsPageViewModel.cs +++ b/src/ClawdDotNet.Desktop/ViewModels/AgentsPageViewModel.cs @@ -1,3 +1,4 @@ +using ClawdDotNet.Core.Tasks; using System.Collections.ObjectModel; using Avalonia.Threading; using ClawdDotNet.App; @@ -77,6 +78,13 @@ public sealed partial class AgentItemViewModel : ViewModelBase [ObservableProperty] private long _dailyTokens; + /// #716: What happens to a recurring task's run when the budget is exhausted. + [ObservableProperty] + private string _whenExhausted = "expire"; + + /// The values understands. + public IReadOnlyList WhenExhaustedOptions { get; } = ["expire", "catchUpOnce"]; + public ObservableCollection AvailableTools { get; } = []; /// @@ -101,6 +109,7 @@ public sealed partial class AgentItemViewModel : ViewModelBase _timeoutSeconds = config.LoopGuard.TimeoutSeconds; _dailyCostUsd = config.Budget.DailyCostUsd; _dailyTokens = config.Budget.DailyTokens; + _whenExhausted = config.Budget.MissPolicy == BudgetMissPolicy.CatchUpOnce ? "catchUpOnce" : "expire"; foreach (var tool in KnownTools) { @@ -137,6 +146,7 @@ public sealed partial class AgentItemViewModel : ViewModelBase Config.LoopGuard.TimeoutSeconds = TimeoutSeconds; Config.Budget.DailyCostUsd = DailyCostUsd; Config.Budget.DailyTokens = DailyTokens; + Config.Budget.WhenExhausted = WhenExhausted; } } diff --git a/src/ClawdDotNet.Desktop/Views/AgentsPageView.axaml b/src/ClawdDotNet.Desktop/Views/AgentsPageView.axaml index 019da6e..2dd8cca 100644 --- a/src/ClawdDotNet.Desktop/Views/AgentsPageView.axaml +++ b/src/ClawdDotNet.Desktop/Views/AgentsPageView.axaml @@ -208,6 +208,15 @@ + + + + + + diff --git a/src/ClawdDotNet.Desktop/i18n/de.json b/src/ClawdDotNet.Desktop/i18n/de.json index 10e6a7c..ca9f13f 100644 --- a/src/ClawdDotNet.Desktop/i18n/de.json +++ b/src/ClawdDotNet.Desktop/i18n/de.json @@ -64,6 +64,8 @@ "desktop.agents.maxToolResultChars": "Max. Tool-Ergebnis (Zeichen)", "desktop.agents.dailyCost": "Tagesbudget ($ USD)", "desktop.agents.dailyTokens": "Tagesbudget (Tokens)", + "desktop.agents.whenExhausted": "Wenn das Budget erschöpft ist (wiederkehrende Aufgaben)", + "desktop.agents.whenExhaustedHint": "expire: Der verpasste Lauf verfällt und wird an der Aufgabe vermerkt. catchUpOnce: Alle verpassten Läufe werden am nächsten Tag durch einen Lauf nachgeholt, nach der regulären Arbeit des Tages. Einmalige Aufgaben werden immer auf den nächsten Tag mitgenommen.", "desktop.agents.allowedToolsFor": "ZUGELASSENE WERKZEUGE FÜR {0}", "desktop.agents.thTool": "WERKZEUG", "desktop.agents.thDescription": "BESCHREIBUNG", diff --git a/src/ClawdDotNet.Desktop/i18n/en.json b/src/ClawdDotNet.Desktop/i18n/en.json index a24da00..6590e4f 100644 --- a/src/ClawdDotNet.Desktop/i18n/en.json +++ b/src/ClawdDotNet.Desktop/i18n/en.json @@ -64,6 +64,8 @@ "desktop.agents.maxToolResultChars": "Max Tool Result (chars)", "desktop.agents.dailyCost": "Daily Budget ($ USD)", "desktop.agents.dailyTokens": "Daily Budget (Tokens)", + "desktop.agents.whenExhausted": "When the budget is exhausted (recurring tasks)", + "desktop.agents.whenExhaustedHint": "expire: the missed run lapses and is recorded on the task. catchUpOnce: all missed runs are caught up by one run the next day, after the day's regular work. One-time tasks are always carried over to the next day.", "desktop.agents.allowedToolsFor": "ALLOWED TOOLS FOR {0}", "desktop.agents.thTool": "TOOL", "desktop.agents.thDescription": "DESCRIPTION", diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json index 681e8dc..1755eb9 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json +++ b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json @@ -192,6 +192,12 @@ "tasks.saved": "Aufgabe gespeichert.", "tasks.deleted": "Aufgabe gelöscht.", "tasks.runTriggered": "Lauf ausgelöst.", + "tasks.nextRun": "Nächster", + "tasks.lastRun": "Letzter", + "tasks.budgetMissed": "Budget erschöpft: {n} Lauf/Läufe abgelehnt", + "tasks.budgetMissedLast": "zuletzt {time}", + "tasks.budgetCarriedOver": "mitgenommen, wieder fällig ab {time}", + "tasks.budgetCatchUp": "ein Nachholauf ab {time}", "services.title": "Dienste", "services.newService": "Neuer Dienst", diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json index e5044c2..220b99a 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json +++ b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json @@ -192,6 +192,12 @@ "tasks.saved": "Task saved.", "tasks.deleted": "Task deleted.", "tasks.runTriggered": "Run triggered.", + "tasks.nextRun": "Next", + "tasks.lastRun": "Last", + "tasks.budgetMissed": "Budget exhausted: {n} run(s) refused", + "tasks.budgetMissedLast": "last at {time}", + "tasks.budgetCarriedOver": "carried over, due again from {time}", + "tasks.budgetCatchUp": "one catch-up run from {time}", "services.title": "Services", "services.newService": "New Service", diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/views/tasks.js b/src/ClawdDotNet.Manager.Web/wwwroot/views/tasks.js index 17df6df..2d44bb0 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/views/tasks.js +++ b/src/ClawdDotNet.Manager.Web/wwwroot/views/tasks.js @@ -163,11 +163,11 @@ function erstelleAufgabenKarte(task) { zeilen.push( erstelleElement("p", { klasse: "gedaempft text-klein", - text: `Next: ${task.nextRunUtc ? formatiereDatumZeit(task.nextRunUtc) : "—"}` + text: `${t("tasks.nextRun")}: ${task.nextRunUtc ? formatiereDatumZeit(task.nextRunUtc) : "—"}` }) ); - const letzter = erstelleElement("p", { klasse: "gedaempft text-klein" }, "Last: "); + const letzter = erstelleElement("p", { klasse: "gedaempft text-klein" }, `${t("tasks.lastRun")}: `); if (task.lastRun) { letzter.append( `${formatiereDatumZeit(task.lastRun.occurredAt)} `, @@ -184,6 +184,19 @@ function erstelleAufgabenKarte(task) { zeilen.push(erstelleElement("p", { klasse: "text-klein", text: `⚠ ${task.hinweis}` })); } + // #716: What the budget refused, and what happens next. + const ausfall = task.budgetMiss; + if (ausfall) { + const teile = [t("tasks.budgetMissed", { n: ausfall.count })]; + if (ausfall.lastAt) teile.push(t("tasks.budgetMissedLast", { time: formatiereDatumZeit(ausfall.lastAt) })); + if (ausfall.carriedOver && ausfall.deferredUntil) { + teile.push(t("tasks.budgetCarriedOver", { time: formatiereDatumZeit(ausfall.deferredUntil) })); + } else if (ausfall.catchUpPending) { + teile.push(t("tasks.budgetCatchUp", { time: ausfall.deferredUntil ? formatiereDatumZeit(ausfall.deferredUntil) : "—" })); + } + zeilen.push(erstelleElement("p", { klasse: "text-klein", text: `⚠ ${teile.join(" · ")}` })); + } + const aktionen = erstelleElement("div", { klasse: "gruppe-horizontal", stil: "margin-top: 8px; justify-content: flex-end;" diff --git a/src/ClawdDotNet.Web/Contracts/Dtos.cs b/src/ClawdDotNet.Web/Contracts/Dtos.cs index 42a42fc..477a21a 100644 --- a/src/ClawdDotNet.Web/Contracts/Dtos.cs +++ b/src/ClawdDotNet.Web/Contracts/Dtos.cs @@ -116,7 +116,14 @@ public sealed record TaskDto( TaskWhenDto? When = null, DateTime? NextRunUtc = null, TaskLastRunDto? LastRun = null, - string? Hinweis = null); + string? Hinweis = null, + TaskBudgetMissDto? BudgetMiss = null); + +/// +/// #716: Runs of this task the budget refused — only present while something is open +/// (misses since the last run that took place, a pending catch-up or a deferral). +/// +public sealed record TaskBudgetMissDto(int Count, DateTime? LastAt, DateTime? DeferredUntil, bool CatchUpPending, bool CarriedOver); /// Der letzte Lauf einer Aufgabe (#594) - aus dem Beleg, samt Kosten. public sealed record TaskLastRunDto(DateTime OccurredAt, string Status, decimal CostUsd, bool CostIsKnown, string ResultRef); diff --git a/src/ClawdDotNet.Web/Contracts/Mapping.cs b/src/ClawdDotNet.Web/Contracts/Mapping.cs index 5244ba1..5d837ae 100644 --- a/src/ClawdDotNet.Web/Contracts/Mapping.cs +++ b/src/ClawdDotNet.Web/Contracts/Mapping.cs @@ -129,6 +129,10 @@ internal static class Mapping task.UpdatedAt, task.When is { } when ? new TaskWhenDto(when.Kind.ToString().ToLowerInvariant(), when.Value, when.TimeZone) + : null, + BudgetMiss: task.BudgetMissCount > 0 || task.CatchUpPending || task.DeferredUntil is not null + ? new TaskBudgetMissDto(task.BudgetMissCount, task.LastBudgetMissAt, task.DeferredUntil, + task.CatchUpPending, task.IsCarriedOver) : null); /// diff --git a/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs b/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs new file mode 100644 index 0000000..c3cb850 --- /dev/null +++ b/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs @@ -0,0 +1,257 @@ +using ClawdDotNet.Core.Storage; +using ClawdDotNet.Core.Tasks; +using ClawdDotNet.Core.Tests.Infrastructure; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace ClawdDotNet.Core.Tests.Tasks; + +/// +/// #716: What the scanner does with a run the budget refused. +/// +/// Before, a one-time task was retried every 30 minutes for the rest of the day — each +/// attempt another refusal and another receipt — and then ran after midnight at the +/// expense of the new day, without anyone having decided that. A recurring task's +/// occurrence lapsed silently. +/// +public sealed class BudgetMissTests : IDisposable +{ + private readonly string _directory; + private readonly SqliteStorage _storage; + private readonly SqliteTaskRepository _repo; + + // 2027: a cron schedule never looks back before the task's creation date. + private readonly FakeTimeProvider _clock = new(new DateTimeOffset(2027, 3, 10, 12, 0, 0, TimeSpan.Zero)); + + public BudgetMissTests() + { + _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N")); + _storage = new SqliteStorage(Path.Combine(_directory, "state.db")); + _repo = new SqliteTaskRepository(_storage); + } + + public void Dispose() + { + Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools(); + try { Directory.Delete(_directory, recursive: true); } + catch { /* cleanup is secondary */ } + } + + private TaskScanner Scanner(ITaskDispatcher dispatcher, BudgetMissPolicy policy = BudgetMissPolicy.Expire) + => new(_repo, dispatcher, NullLoggerFactory.Instance, clock: _clock, budgetMissPolicy: _ => policy); + + private Task SeedOneTime(string id, int priority = 3) + => _repo.UpsertAsync(new TaskItem { Id = id, Title = id, Assignee = "@worker", Priority = priority }, default); + + private Task SeedHourly(string id) + => _repo.UpsertAsync(new TaskItem + { + Id = id, Title = id, Assignee = "@worker", + When = new TaskWhen { Kind = TaskWhenKind.Cron, Value = "0 * * * *", TimeZone = "UTC" } + }, default); + + private async Task Get(string id) => (await _repo.GetAsync(id, default))!; + + /// The next local midnight, in UTC — the budget day follows the local date. + private DateTime NextLocalMidnightUtc() + { + var zone = _clock.LocalTimeZone; + var local = TimeZoneInfo.ConvertTimeFromUtc(_clock.GetUtcNow().UtcDateTime, zone); + return TimeZoneInfo.ConvertTimeToUtc(DateTime.SpecifyKind(local.Date.AddDays(1), DateTimeKind.Unspecified), zone); + } + + private void MoveTo(DateTime utc) => _clock.SetUtcNow(new DateTimeOffset(utc, TimeSpan.Zero)); + + // ═══════════════════ one-time tasks: carried over ═══════════════════ + + [Fact] + public async Task A_refused_one_time_task_is_carried_over_to_the_next_day() + { + await SeedOneTime("t-report"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher); + var midnight = NextLocalMidnightUtc(); + + await scanner.ScanOnceAsync(default); + + var task = await Get("t-report"); + task.Status.ShouldBe(TaskItemStatus.Todo, "it is still work to be done"); + task.BudgetMissCount.ShouldBe(1); + task.LastBudgetMissAt.ShouldNotBeNull(); + task.DeferredUntil.ShouldBe(midnight); + task.IsCarriedOver.ShouldBeTrue(); + } + + [Fact] + public async Task A_carried_over_task_is_not_retried_the_same_day() + { + await SeedOneTime("t-report"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher); + await scanner.ScanOnceAsync(default); + + // Before #716: again after 30 minutes, and again, until midnight. + MoveTo(NextLocalMidnightUtc().AddMinutes(-1)); + (await scanner.ScanOnceAsync(default)).ShouldBe(0); + + dispatcher.Calls.ShouldBe(1, "one refusal, not one every half hour"); + } + + [Fact] + public async Task On_the_next_day_it_runs_and_the_miss_is_settled() + { + await SeedOneTime("t-report"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher); + await scanner.ScanOnceAsync(default); + + MoveTo(NextLocalMidnightUtc().AddMinutes(1)); + dispatcher.HasBudget = true; + (await scanner.ScanOnceAsync(default)).ShouldBe(1); + + var task = await Get("t-report"); + task.Status.ShouldBe(TaskItemStatus.Done); + task.BudgetMissCount.ShouldBe(0); + task.DeferredUntil.ShouldBeNull(); + task.LastBudgetMissAt.ShouldNotBeNull("kept as history"); + } + + [Fact] + public async Task The_new_day_has_right_of_way_over_carried_over_work() + { + await SeedOneTime("t-yesterday", priority: 5); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher); + await scanner.ScanOnceAsync(default); + + MoveTo(NextLocalMidnightUtc().AddMinutes(1)); + await SeedOneTime("t-today", priority: 1); + dispatcher.HasBudget = true; + dispatcher.Started.Clear(); + await scanner.ScanOnceAsync(default); + + dispatcher.Started.ShouldBe(["t-today", "t-yesterday"], + "carried-over work comes after the day's own, even with a higher priority"); + } + + // ═══════════════════ recurring tasks: the agent's policy ═══════════════════ + + [Fact] + public async Task Under_expire_a_refused_occurrence_lapses_and_is_recorded() + { + await SeedHourly("t-hourly"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher, BudgetMissPolicy.Expire); + + await scanner.ScanOnceAsync(default); + + var task = await Get("t-hourly"); + task.Status.ShouldBe(TaskItemStatus.Todo); + task.BudgetMissCount.ShouldBe(1); + task.CatchUpPending.ShouldBeFalse(); + + // Nothing is caught up the next day beyond the regular occurrences. + MoveTo(NextLocalMidnightUtc().AddMinutes(30)); + dispatcher.HasBudget = true; + dispatcher.Started.Clear(); + (await scanner.ScanOnceAsync(default)).ShouldBe(1, "only the regular occurrence of this hour"); + } + + [Fact] + public async Task Under_catch_up_once_several_misses_leave_one_pending_catch_up() + { + await SeedHourly("t-hourly"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher, BudgetMissPolicy.CatchUpOnce); + var midnight = NextLocalMidnightUtc(); + + // Three hours in a row refused. + for (var hour = 0; hour < 3; hour++) + { + await scanner.ScanOnceAsync(default); + _clock.Advance(TimeSpan.FromHours(1)); + } + + var task = await Get("t-hourly"); + task.BudgetMissCount.ShouldBe(3); + task.CatchUpPending.ShouldBeTrue(); + task.DeferredUntil.ShouldBe(midnight, "set by the first miss, not pushed on by the later ones"); + } + + [Fact] + public async Task The_catch_up_runs_exactly_once_on_the_next_day() + { + // A daily report at 22:30: its next regular occurrence is 24 hours away, so here + // the catch-up is the only run — the case catchUpOnce exists for. + MoveTo(new DateTime(2027, 3, 10, 22, 30, 0, DateTimeKind.Utc)); + await _repo.UpsertAsync(new TaskItem + { + Id = "t-daily", Title = "t-daily", Assignee = "@worker", + When = new TaskWhen { Kind = TaskWhenKind.Cron, Value = "30 22 * * *", TimeZone = "UTC" } + }, default); + + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher, BudgetMissPolicy.CatchUpOnce); + (await scanner.ScanOnceAsync(default)).ShouldBe(1, "the regular 22:30 run, refused"); + + MoveTo(NextLocalMidnightUtc().AddMinutes(1)); + dispatcher.HasBudget = true; + dispatcher.Started.Clear(); + + (await scanner.ScanOnceAsync(default)).ShouldBe(1, "the catch-up"); + _clock.Advance(TimeSpan.FromMinutes(1)); + (await scanner.ScanOnceAsync(default)).ShouldBe(0, "once, not on every tick"); + + dispatcher.Started.ShouldBe(["t-daily"]); + var task = await Get("t-daily"); + task.CatchUpPending.ShouldBeFalse(); + task.BudgetMissCount.ShouldBe(0); + task.Status.ShouldBe(TaskItemStatus.Todo, "still recurring"); + } + + [Fact] + public async Task A_pending_catch_up_does_not_run_on_the_same_day() + { + await SeedHourly("t-hourly"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var scanner = Scanner(dispatcher, BudgetMissPolicy.CatchUpOnce); + await scanner.ScanOnceAsync(default); + + // Budget raised by the human in the evening: the regular occurrence runs, and it + // covers the pending catch-up — one run, not two. + dispatcher.HasBudget = true; + dispatcher.Started.Clear(); + _clock.Advance(TimeSpan.FromHours(1)); + (await scanner.ScanOnceAsync(default)).ShouldBe(1); + (await Get("t-hourly")).CatchUpPending.ShouldBeFalse(); + } + + [Fact] + public async Task A_shutdown_is_not_a_budget_miss() + { + await SeedOneTime("t-report"); + var dispatcher = new BudgetDispatcher { Result = TaskLauf.NichtGelaufen }; + await Scanner(dispatcher).ScanOnceAsync(default); + + var task = await Get("t-report"); + task.BudgetMissCount.ShouldBe(0); + task.DeferredUntil.ShouldBeNull("a shutdown keeps the old 30-minute retry"); + } + + // ═══════════════════ test double ═══════════════════ + + private sealed class BudgetDispatcher : ITaskDispatcher + { + public bool HasBudget { get; set; } = true; + public TaskLauf? Result { get; set; } + public int Calls { get; private set; } + public List Started { get; } = []; + + public Task DispatchAsync(TaskItem task, CancellationToken ct) + { + Calls++; + Started.Add(task.Id); + return Task.FromResult(Result ?? (HasBudget ? TaskLauf.Erledigt : TaskLauf.KeinBudget)); + } + } +} From 8a83a67fb466c79d55a4c91fe82b703211c31746 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 24 Sep 2026 12:06:07 +0200 Subject: [PATCH 2/4] Budgetende, Stufe 2: laufende Laeufe reservieren ihr Budget (#716) Der offene Rest aus #682: Jeder Lauf pruefte das Budget einzeln gegen den bisher erfassten Verbrauch. Laufende Laeufe waren unsichtbar, gleichzeitig startende passierten die Pruefung gemeinsam und ueberzogen gemeinsam. - BudgetGuard.ReserveAsync prueft und reserviert unter einer Sperre. Die Reservierung ist der Mittelwert der letzten 20 Laeufe des Agenten (Laeufe ohne Verbrauch zaehlen nicht mit) und gilt, bis der Verbrauch des Laufs erfasst ist. - Gegen die Grenze zaehlt: verbraucht + in laufenden Laeufen reserviert. Die eigene Schaetzung eines Laufs zaehlt bewusst nicht mit; ein einzelner Lauf wird weiter nur abgelehnt, wenn das Budget schon verbraucht ist. Sonst fiele eine kleine Aufgabe wegen eines grossen Durchschnitts aus. - Gilt fuer Kosten- und Tokengrenze, je Agent und je Instanz. - Die Ablehnungsmeldung nennt den reservierten Anteil, sonst wirkte "0,60 von 1,00 USD verbraucht" als Ablehnungsgrund falsch. - IUsageRepository.GetRecentForAgentAsync, mit leerer Vorgabe im Interface, damit bestehende Attrappen unberuehrt bleiben. Tests: BudgetReservationTests (8 neu), darunter zehn gleichzeitige Starts, von denen genau vier passieren. 1.246 gruen. Co-Authored-By: Claude Opus 5.5 --- .../Accounting/SqliteUsageRepository.cs | 23 +++ .../Accounting/UsageModels.cs | 8 + src/ClawdDotNet.Core/Budget/BudgetGuard.cs | 149 ++++++++++++++- src/ClawdDotNet.Core/Engine/AgentEngine.cs | 21 ++- .../Budget/BudgetReservationTests.cs | 173 ++++++++++++++++++ 5 files changed, 359 insertions(+), 15 deletions(-) create mode 100644 tests/ClawdDotNet.Core.Tests/Budget/BudgetReservationTests.cs diff --git a/src/ClawdDotNet.Core/Accounting/SqliteUsageRepository.cs b/src/ClawdDotNet.Core/Accounting/SqliteUsageRepository.cs index a63364d..40df9f4 100644 --- a/src/ClawdDotNet.Core/Accounting/SqliteUsageRepository.cs +++ b/src/ClawdDotNet.Core/Accounting/SqliteUsageRepository.cs @@ -125,6 +125,29 @@ public sealed class SqliteUsageRepository : IUsageRepository return results; } + public async Task> GetRecentForAgentAsync(string agentId, int limit, CancellationToken ct) + { + await using var conn = await _storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + SELECT Id, AgentId, Model, PromptTokens, CompletionTokens, CachedTokens, + CostUsd, CostIsKnown, Status, StepCount, DurationMs, OccurredAt + FROM RunUsage + WHERE AgentId = @agent COLLATE NOCASE + ORDER BY OccurredAt DESC + LIMIT @limit + """; + cmd.Parameters.AddWithValue("@agent", agentId); + cmd.Parameters.AddWithValue("@limit", Math.Clamp(limit, 1, 1000)); + + var results = new List(); + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + results.Add(Read(reader)); + + return results; + } + public Task PurgeBeforeAsync(DateOnly date, CancellationToken ct) => _storage.WriteAsync(async conn => { diff --git a/src/ClawdDotNet.Core/Accounting/UsageModels.cs b/src/ClawdDotNet.Core/Accounting/UsageModels.cs index b60a975..f66d403 100644 --- a/src/ClawdDotNet.Core/Accounting/UsageModels.cs +++ b/src/ClawdDotNet.Core/Accounting/UsageModels.cs @@ -53,6 +53,14 @@ public interface IUsageRepository Task> GetRecentAsync(int limit, CancellationToken ct); + /// + /// #716: The most recent runs of one agent — the basis for estimating what a run of + /// it will cost. Without an implementation there is no estimate, and a reservation + /// counts nothing. + /// + Task> GetRecentForAgentAsync(string agentId, int limit, CancellationToken ct) + => Task.FromResult>([]); + /// Entfernt Einträge älter als das angegebene Datum. Gibt die Anzahl zurück. Task PurgeBeforeAsync(DateOnly date, CancellationToken ct); } diff --git a/src/ClawdDotNet.Core/Budget/BudgetGuard.cs b/src/ClawdDotNet.Core/Budget/BudgetGuard.cs index 2ecac6b..6419b60 100644 --- a/src/ClawdDotNet.Core/Budget/BudgetGuard.cs +++ b/src/ClawdDotNet.Core/Budget/BudgetGuard.cs @@ -13,6 +13,13 @@ public enum BudgetLimit InstanceTokens } +/// #716: What a run is expected to use — the agent's average over recent runs. +public readonly record struct RunEstimate(decimal CostUsd, long Tokens) +{ + public static RunEstimate operator +(RunEstimate a, RunEstimate b) => new(a.CostUsd + b.CostUsd, a.Tokens + b.Tokens); + public static RunEstimate operator -(RunEstimate a, RunEstimate b) => new(a.CostUsd - b.CostUsd, a.Tokens - b.Tokens); +} + public sealed record BudgetStatus(bool IsAllowed, BudgetLimit Limit, string? Reason) { public static BudgetStatus Allowed { get; } = new(true, BudgetLimit.None, null); @@ -34,6 +41,19 @@ public sealed class BudgetGuard private readonly IUsageRepository _usage; private readonly Func _now; + /// How many recent runs of an agent make up its per-run estimate (#716). + public const int EstimateSample = 20; + + /// + /// #716: Check and reserve happen under this lock. Without it, two runs starting at + /// the same moment would both read the same "used" figure and both pass. + /// + private readonly SemaphoreSlim _reserveGate = new(1, 1); + + private readonly Lock _reservedLock = new(); + private readonly Dictionary _reservedByAgent = new(StringComparer.OrdinalIgnoreCase); + private RunEstimate _reservedInstance; + public BudgetGuard(IUsageRepository usage, Func? now = null) { _usage = usage; @@ -43,8 +63,112 @@ public sealed class BudgetGuard /// /// Prüft, ob der Agent heute noch laufen darf. /// - public async Task CheckAsync( + public Task CheckAsync( AgentConfig agent, InstanceBudget instanceBudget, CancellationToken ct) + => CheckCoreAsync(agent, instanceBudget, default, default, ct); + + /// + /// #716: Checks like and, if the run may start, reserves its + /// estimated usage until the returned handle is disposed — i.e. until the run's usage + /// is recorded. + /// + /// What changes. Before, each run was checked on its own against what was + /// already recorded. Runs in flight were invisible, so several starting together all + /// passed and exceeded the budget together (#682 capped how many; this closes it). + /// Now the runs in flight count with their estimate. + /// + /// What does not change. A single run is still refused only when the + /// budget is already used up — its own estimate is not added. Otherwise a small task + /// would be refused because the agent's average run is large. + /// + public async Task<(BudgetStatus Status, IDisposable? Reservation)> ReserveAsync( + AgentConfig agent, InstanceBudget instanceBudget, CancellationToken ct) + { + if (!agent.Budget.HasAnyLimit && !instanceBudget.HasAnyLimit) + return (BudgetStatus.Allowed, null); + + var estimate = await EstimateRunAsync(agent.AgentId, ct); + + await _reserveGate.WaitAsync(ct); + try + { + RunEstimate agentReserved, instanceReserved; + lock (_reservedLock) + { + agentReserved = _reservedByAgent.GetValueOrDefault(agent.AgentId); + instanceReserved = _reservedInstance; + } + + var status = await CheckCoreAsync(agent, instanceBudget, agentReserved, instanceReserved, ct); + if (!status.IsAllowed) + return (status, null); + + lock (_reservedLock) + { + _reservedByAgent[agent.AgentId] = _reservedByAgent.GetValueOrDefault(agent.AgentId) + estimate; + _reservedInstance += estimate; + } + + return (status, new Reservation(this, agent.AgentId, estimate)); + } + finally + { + _reserveGate.Release(); + } + } + + /// What is currently reserved by runs in flight — for the agent, or with an + /// empty id for the whole instance. + public RunEstimate Reserved(string agentId) + { + lock (_reservedLock) + return agentId.Length == 0 ? _reservedInstance : _reservedByAgent.GetValueOrDefault(agentId); + } + + /// + /// The average of the agent's recent runs. Runs without usage (refusals, aborted + /// before the first call) do not count — they would drag the estimate towards zero. + /// + public async Task EstimateRunAsync(string agentId, CancellationToken ct) + { + var recent = await _usage.GetRecentForAgentAsync(agentId, EstimateSample, ct); + var runs = recent.Where(r => r.TotalTokens > 0).ToList(); + if (runs.Count == 0) + return default; + + return new RunEstimate( + runs.Average(r => r.CostUsd), + (long)Math.Ceiling(runs.Average(r => (double)r.TotalTokens))); + } + + private void Release(string agentId, RunEstimate estimate) + { + lock (_reservedLock) + { + var left = _reservedByAgent.GetValueOrDefault(agentId) - estimate; + if (left.CostUsd <= 0 && left.Tokens <= 0) + _reservedByAgent.Remove(agentId); + else + _reservedByAgent[agentId] = left; + + _reservedInstance -= estimate; + } + } + + private sealed class Reservation(BudgetGuard guard, string agentId, RunEstimate estimate) : IDisposable + { + private int _disposed; + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + guard.Release(agentId, estimate); + } + } + + private async Task CheckCoreAsync( + AgentConfig agent, InstanceBudget instanceBudget, + RunEstimate agentReserved, RunEstimate instanceReserved, CancellationToken ct) { var today = DateOnly.FromDateTime(_now()); @@ -54,19 +178,21 @@ public sealed class BudgetGuard { var used = await _usage.GetDailyAsync(today, agent.AgentId, ct); - if (Exceeds(agentBudget.DailyCostUsd, used.CostUsd)) + if (Exceeds(agentBudget.DailyCostUsd, used.CostUsd + agentReserved.CostUsd)) { return new BudgetStatus(false, BudgetLimit.AgentCost, $"Tagesbudget des Agenten erschöpft: {used.CostUsd:F4} von maximal " + $"{agentBudget.DailyCostUsd:F2} USD verbraucht." + + ReservedHint(agentReserved.CostUsd) + UnpricedHint(used)); } - if (Exceeds(agentBudget.DailyTokens, used.TotalTokens)) + if (Exceeds(agentBudget.DailyTokens, used.TotalTokens + agentReserved.Tokens)) { return new BudgetStatus(false, BudgetLimit.AgentTokens, $"Token-Tageslimit des Agenten erschöpft: {used.TotalTokens:N0} von maximal " + - $"{agentBudget.DailyTokens:N0}."); + $"{agentBudget.DailyTokens:N0}." + + ReservedTokensHint(agentReserved.Tokens)); } } @@ -75,19 +201,21 @@ public sealed class BudgetGuard { var used = await _usage.GetDailyAsync(today, agentId: "", ct); - if (Exceeds(instanceBudget.DailyCostUsd, used.CostUsd)) + if (Exceeds(instanceBudget.DailyCostUsd, used.CostUsd + instanceReserved.CostUsd)) { return new BudgetStatus(false, BudgetLimit.InstanceCost, $"Tagesbudget der Instanz erschöpft: {used.CostUsd:F4} von maximal " + $"{instanceBudget.DailyCostUsd:F2} USD verbraucht." + + ReservedHint(instanceReserved.CostUsd) + UnpricedHint(used)); } - if (Exceeds(instanceBudget.DailyTokens, used.TotalTokens)) + if (Exceeds(instanceBudget.DailyTokens, used.TotalTokens + instanceReserved.Tokens)) { return new BudgetStatus(false, BudgetLimit.InstanceTokens, $"Token-Tageslimit der Instanz erschöpft: {used.TotalTokens:N0} von maximal " + - $"{instanceBudget.DailyTokens:N0}."); + $"{instanceBudget.DailyTokens:N0}." + + ReservedTokensHint(instanceReserved.Tokens)); } } @@ -109,6 +237,13 @@ public sealed class BudgetGuard private static bool Exceeds(long limit, long used) => limit > 0 && used >= limit; + /// #716: Without this, a refusal at "3.20 of 5.00 USD used" would look wrong. + private static string ReservedHint(decimal reserved) + => reserved > 0 ? $" Weitere {reserved:F4} USD sind für laufende Läufe reserviert." : ""; + + private static string ReservedTokensHint(long reserved) + => reserved > 0 ? $" Weitere {reserved:N0} Tokens sind für laufende Läufe reserviert." : ""; + /// /// Ohne Preisangaben ist die verbrauchte Summe unvollständig — sonst wirkte ein /// niedriger Wert wie ein noch offener Spielraum. diff --git a/src/ClawdDotNet.Core/Engine/AgentEngine.cs b/src/ClawdDotNet.Core/Engine/AgentEngine.cs index b38a0c3..cc8389b 100644 --- a/src/ClawdDotNet.Core/Engine/AgentEngine.cs +++ b/src/ClawdDotNet.Core/Engine/AgentEngine.cs @@ -163,8 +163,11 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut return zu; // Vor der ersten Anfrage prüfen — ein erschöpftes Budget soll gar nichts kosten. - if (await CheckBudgetAsync(agentConfig, source ?? AuditSource.Direct, taskId, externalCt) is { } denied) + // #716: Die Reservierung gilt, bis der Verbrauch erfasst ist (Ende der Methode). + var (denied, reservation) = await CheckBudgetAsync(agentConfig, source ?? AuditSource.Direct, taskId, externalCt); + if (denied is not null) return denied; + using var budgetReservation = reservation; using var lauf = TrackRun(); @@ -444,8 +447,10 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut if (RefuseWhileShuttingDown(agentConfig) is { } zu) return zu; - if (await CheckBudgetAsync(agentConfig, source, taskId, externalCt) is { } denied) + var (denied, reservation) = await CheckBudgetAsync(agentConfig, source, taskId, externalCt); + if (denied is not null) return denied; + using var budgetReservation = reservation; using var lauf = TrackRun(); @@ -1638,17 +1643,17 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut /// /// Prüft das Tagesbudget. Gibt ein Ergebnis zurück, wenn der Lauf nicht stattfinden - /// darf — sonst null. + /// darf — sonst null und, bei gesetzter Grenze, die Reservierung des Laufs (#716). /// - private async Task CheckBudgetAsync( + private async Task<(AgentRunResult? Denied, IDisposable? Reservation)> CheckBudgetAsync( AgentConfig agentConfig, string? source, string? taskId, CancellationToken ct) { if (_budgetGuard is null) - return null; + return (null, null); - var status = await _budgetGuard.CheckAsync(agentConfig, InstanceBudget, ct); + var (status, reservation) = await _budgetGuard.ReserveAsync(agentConfig, InstanceBudget, ct); if (status.IsAllowed) - return null; + return (null, reservation); // #608: Ist ein Budget erschoepft, lehnt es jeden weiteren Lauf des Tages ab. Einmal je // Agent, Grenze und Tag als Warnung genuegt - danach steht jede Ablehnung als Beleg da. @@ -1676,7 +1681,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut await RecordReceiptAsync(Guid.NewGuid().ToString("N"), agentConfig, result, source, taskId); OnRunCompleted?.Invoke(agentConfig.Model, result); - return result; + return (result, null); } /// Welche Budgetgrenze je Agent zuletzt an welchem Tag als Warnung gemeldet wurde. diff --git a/tests/ClawdDotNet.Core.Tests/Budget/BudgetReservationTests.cs b/tests/ClawdDotNet.Core.Tests/Budget/BudgetReservationTests.cs new file mode 100644 index 0000000..69418ea --- /dev/null +++ b/tests/ClawdDotNet.Core.Tests/Budget/BudgetReservationTests.cs @@ -0,0 +1,173 @@ +using ClawdDotNet.Core.Accounting; +using ClawdDotNet.Core.Budget; +using ClawdDotNet.Core.Config; +using Shouldly; + +namespace ClawdDotNet.Core.Tests.Budget; + +/// +/// #716 (the rest of #682): Runs in flight count against the budget with their estimate. +/// +/// Before, every run was checked on its own against what was already recorded, so runs +/// starting together all saw the same figure and all passed. #682 capped how many could +/// start at once; the reservation closes the gap. +/// +public sealed class BudgetReservationTests +{ + private static AgentConfig Agent(decimal dailyCost = 1.00m, long dailyTokens = 0) + => new() { AgentId = "worker", Budget = new AgentBudget { DailyCostUsd = dailyCost, DailyTokens = dailyTokens } }; + + /// Recent runs averaging 0.30 USD and 3,000 tokens. + private static List History(decimal cost = 0.30m, int tokens = 3_000) + => [.. Enumerable.Range(0, 5).Select(_ => new RunUsage + { + AgentId = "worker", PromptTokens = tokens, CostUsd = cost, CostIsKnown = true, Status = "Completed" + })]; + + [Fact] + public async Task Runs_in_flight_count_against_the_budget() + { + var guard = new BudgetGuard(new FakeUsage(usedCost: 0.60m, History())); + + var (first, a) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + var (second, b) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + var (third, c) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + + first.IsAllowed.ShouldBeTrue("0.60 used, nothing in flight"); + second.IsAllowed.ShouldBeTrue("0.60 used + 0.30 in flight"); + third.IsAllowed.ShouldBeFalse("0.60 used + 0.60 in flight — before #716 this one passed too"); + third.Reason.ShouldNotBeNull().ShouldContain("reserviert"); + c.ShouldBeNull(); + + a!.Dispose(); + b!.Dispose(); + } + + [Fact] + public async Task A_finished_run_frees_its_reservation() + { + var guard = new BudgetGuard(new FakeUsage(usedCost: 0.60m, History())); + + var (_, a) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + var (_, b) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + a!.Dispose(); + a.Dispose(); // twice is harmless + + var (again, c) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + again.IsAllowed.ShouldBeTrue(); + + b!.Dispose(); + c!.Dispose(); + guard.Reserved("worker").ShouldBe(default(RunEstimate)); + guard.Reserved("").ShouldBe(default(RunEstimate)); + } + + [Fact] + public async Task A_single_run_is_not_refused_because_of_its_own_estimate() + { + // 0.95 of 1.00 used, an average run costs 0.30: a small task may still fit, and + // refusing it on the average would be guessing. + var guard = new BudgetGuard(new FakeUsage(usedCost: 0.95m, History())); + + var (status, reservation) = await guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default); + + status.IsAllowed.ShouldBeTrue(); + reservation!.Dispose(); + } + + [Fact] + public async Task Parallel_starts_cannot_all_pass_together() + { + var guard = new BudgetGuard(new FakeUsage(usedCost: 0m, History())); + + var results = await Task.WhenAll(Enumerable.Range(0, 10) + .Select(_ => guard.ReserveAsync(Agent(), InstanceBudget.Unlimited, default))); + + // 0.00, 0.30, 0.60, 0.90 in flight pass; from 1.20 on the budget is spoken for. + results.Count(r => r.Status.IsAllowed).ShouldBe(4); + + foreach (var (_, reservation) in results) + reservation?.Dispose(); + } + + [Fact] + public async Task The_token_limit_counts_reservations_too() + { + var guard = new BudgetGuard(new FakeUsage(usedCost: 0m, History(cost: 0m, tokens: 4_000))); + var agent = Agent(dailyCost: 0, dailyTokens: 10_000); + + var (a, ra) = await guard.ReserveAsync(agent, InstanceBudget.Unlimited, default); + var (b, rb) = await guard.ReserveAsync(agent, InstanceBudget.Unlimited, default); + var (c, _) = await guard.ReserveAsync(agent, InstanceBudget.Unlimited, default); + var (d, _) = await guard.ReserveAsync(agent, InstanceBudget.Unlimited, default); + + a.IsAllowed.ShouldBeTrue(); + b.IsAllowed.ShouldBeTrue(); + c.IsAllowed.ShouldBeTrue("8,000 of 10,000 reserved"); + d.IsAllowed.ShouldBeFalse("12,000 reserved"); + d.Limit.ShouldBe(BudgetLimit.AgentTokens); + + ra!.Dispose(); + rb!.Dispose(); + } + + [Fact] + public async Task The_instance_limit_sees_the_runs_of_all_agents() + { + var guard = new BudgetGuard(new FakeUsage(usedCost: 0m, History())); + var instance = new InstanceBudget { DailyCostUsd = 0.50m }; + var unlimitedAgent = Agent(dailyCost: 0); + + var (a, _) = await guard.ReserveAsync(unlimitedAgent, instance, default); + var (b, _) = await guard.ReserveAsync(unlimitedAgent, instance, default); + var (c, _) = await guard.ReserveAsync(unlimitedAgent, instance, default); + + a.IsAllowed.ShouldBeTrue(); + b.IsAllowed.ShouldBeTrue("0.30 in flight"); + c.IsAllowed.ShouldBeFalse("0.60 in flight over an instance limit of 0.50"); + c.Limit.ShouldBe(BudgetLimit.InstanceCost); + } + + [Fact] + public async Task Without_any_limit_nothing_is_reserved() + { + var guard = new BudgetGuard(new FakeUsage(usedCost: 0m, History())); + + var (status, reservation) = await guard.ReserveAsync(Agent(dailyCost: 0), InstanceBudget.Unlimited, default); + + status.IsAllowed.ShouldBeTrue(); + reservation.ShouldBeNull(); + } + + [Fact] + public async Task Runs_without_usage_do_not_drag_the_estimate_down() + { + var history = History(); + history.Add(new RunUsage { AgentId = "worker", Status = "BudgetExceeded" }); // 0 tokens + var guard = new BudgetGuard(new FakeUsage(usedCost: 0m, history)); + + var estimate = await guard.EstimateRunAsync("worker", default); + + estimate.CostUsd.ShouldBe(0.30m); + estimate.Tokens.ShouldBe(3_000); + } + + private sealed class FakeUsage(decimal usedCost, IReadOnlyList recent) : IUsageRepository + { + public Task RecordAsync(RunUsage usage, CancellationToken ct) => Task.CompletedTask; + + public async Task GetDailyAsync(DateOnly date, string agentId, CancellationToken ct) + { + await Task.Yield(); // like the database: the caller does not get the answer synchronously + return new DailyUsage(date, 0, 0, usedCost, true, 0); + } + + public Task> GetRecentAsync(int limit, CancellationToken ct) + => Task.FromResult(recent); + + public Task> GetRecentForAgentAsync(string agentId, int limit, CancellationToken ct) + => Task.FromResult(recent); + + public Task PurgeBeforeAsync(DateOnly date, CancellationToken ct) => Task.FromResult(0); + } +} From 9cf9a2cfc2b5c67a5bcb418c89082a9ee3604ae0 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 24 Sep 2026 12:10:40 +0200 Subject: [PATCH 3/4] Budgetende, Stufe 3: Nachholen nur im Spielraum des Tages, keine Kappung bei 500 (#716) - TaskSchedule.OccurrencesBetween zaehlt, wie oft eine Aufgabe bis zu einem Zeitpunkt noch laeuft, einschliesslich eines jetzt faelligen Laufs. - BudgetHeadroom (ILateRunGate): Ein mitgenommener oder nachzuholender Lauf startet nur, wenn er in den Spielraum passt: Grenze minus Verbrauch minus Reservierung minus Bedarf der restlichen geplanten Laeufe bis Mitternacht minus 10 % Reserve. Das gilt je Agent und je Instanz, fuer Kosten und Tokens. Anderes Spaetwerk zaehlt nicht als Bedarf, sonst blockierten sich mitgenommene Aufgaben gegenseitig. - Der Scanner fragt den Spielraum nur fuer Spaetwerk und protokolliert, wenn es warten muss. Die Engine gibt ihren BudgetGuard heraus, damit Scanner und Engine dieselben Reservierungen sehen. - Ersetzt den pauschalen 25-%-Anteil aus dem Konzept (dort nachgezogen): Ein fester Anteil ist an vollen Tagen zu viel, an ruhigen zu wenig. Drei weitere stille Kappungen bei 500 beseitigt (ListAllAsync, ungekappt): Kostenprognose (Aufgaben ueber 500 fehlten), Waisenabgleich beim Import und task_find aus #674, wo Dubletten unter aelteren Aufgaben mit niedriger Prioritaet unsichtbar wurden. Tests: BudgetHeadroomTests (9), Scanner-Pforte (1), Dublette hinter 500 Aufgaben (1). 1.257 gruen. Co-Authored-By: Claude Opus 5.5 --- docs/Organisation-Konzept.md | 11 +- src/ClawdDotNet.App/AppHost.cs | 7 +- .../Services/TaskAdministration.cs | 3 +- src/ClawdDotNet.Core/Budget/BudgetGuard.cs | 4 + src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs | 144 +++++++++++++++ src/ClawdDotNet.Core/Engine/AgentEngine.cs | 6 + src/ClawdDotNet.Core/Tasks/ITaskRepository.cs | 7 + .../Tasks/SqliteTaskRepository.cs | 16 ++ src/ClawdDotNet.Core/Tasks/TaskScanner.cs | 39 +++- src/ClawdDotNet.Core/Tasks/TaskSchedule.cs | 64 +++++++ .../Tasks/TaskboardService.cs | 10 +- .../TaskboardTool.cs | 4 +- .../Budget/BudgetHeadroomTests.cs | 166 ++++++++++++++++++ .../Tasks/BudgetMissTests.cs | 36 ++++ .../Taskboard/DublettenTests.cs | 18 ++ 15 files changed, 524 insertions(+), 11 deletions(-) create mode 100644 src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs create mode 100644 tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs diff --git a/docs/Organisation-Konzept.md b/docs/Organisation-Konzept.md index 63fad21..e3c9ce7 100644 --- a/docs/Organisation-Konzept.md +++ b/docs/Organisation-Konzept.md @@ -954,9 +954,14 @@ scheitert: | `carry_over` | wird mitgenommen | einmalige Aufgaben: echte Arbeit, die nicht verschwinden darf | Dazu gilt: **Der neue Tag hat Vorfahrt.** Nachholläufe kommen nach den regulär fälligen, -unterhalb jeder Priorität, und dürfen höchstens einen Anteil des neuen Tagesbudgets -verbrauchen (Vorgabe etwa 25 %). Ohne diese Regel schiebt ein einziger überzogener Tag -seinen Ausfall Tag für Tag vor sich her, statt ihn aufzulösen. Eine einmalige Aufgabe, die +unterhalb jeder Priorität, und sie nutzen nur den **Spielraum**, den der Rest des Tages +nicht braucht: Grenze minus Verbrauch, minus die Reservierung laufender Läufe, minus der +Bedarf der noch geplanten Läufe bis Mitternacht, minus 10 % Reserve. Ohne diese Regel +schiebt ein einziger überzogener Tag seinen Ausfall Tag für Tag vor sich her, statt ihn +aufzulösen. *Beim Bau am 2026-09-24 geändert:* Das Konzept sah hier zunächst einen +pauschalen Anteil von 25 % vor. Ein fester Anteil ist an einem vollen Tag zu viel und an +einem ruhigen unnötig wenig; der Spielraum folgt dem tatsächlichen Plan, und dieselbe +Rechnung trägt später die Füllarbeit. Eine einmalige Aufgabe, die nach einigen Tagen immer noch nicht drankam, geht an den Menschen: Was nie läuft, darf nicht unbemerkt liegen bleiben. diff --git a/src/ClawdDotNet.App/AppHost.cs b/src/ClawdDotNet.App/AppHost.cs index 675cf46..3268f28 100644 --- a/src/ClawdDotNet.App/AppHost.cs +++ b/src/ClawdDotNet.App/AppHost.cs @@ -3,6 +3,7 @@ using ClawdDotNet.App.Settings; using ClawdDotNet.Core.Accounting; using ClawdDotNet.Core.Api; using ClawdDotNet.Core.Audit; +using ClawdDotNet.Core.Budget; using ClawdDotNet.Core.Config; using ClawdDotNet.Core.Engine; using ClawdDotNet.Core.Logging; @@ -734,7 +735,11 @@ public sealed class AppHost : IAsyncDisposable return instance.Agents .FirstOrDefault(a => string.Equals(a.AgentId, agentId, StringComparison.OrdinalIgnoreCase)) ?.Budget.MissPolicy ?? BudgetMissPolicy.Expire; - }); + }, + // #716: Late work waits for headroom the rest of the day does not need. + lateRunGate: engine.BudgetGuard is { } guard + ? new BudgetHeadroom(guard, () => instance.Agents, () => engine.InstanceBudget) + : null); // Reconciliation nicht blockierend: Der Start soll nicht auf das Dateisystem warten. _ = Task.Run(async () => diff --git a/src/ClawdDotNet.App/Services/TaskAdministration.cs b/src/ClawdDotNet.App/Services/TaskAdministration.cs index cf390e0..2a71fd5 100644 --- a/src/ClawdDotNet.App/Services/TaskAdministration.cs +++ b/src/ClawdDotNet.App/Services/TaskAdministration.cs @@ -187,7 +187,8 @@ public sealed class TaskAdministration( if (board is null) return null; - var aufgaben = await board.ListAsync(new TaskQuery { Limit = 500 }, ct); + // #716: ungekappt - vorher fehlten ab 500 Aufgaben die übrigen in der Prognose. + var aufgaben = await board.ListAllAsync(ct); var belege = new Dictionary>(StringComparer.Ordinal); if (audit is not null) diff --git a/src/ClawdDotNet.Core/Budget/BudgetGuard.cs b/src/ClawdDotNet.Core/Budget/BudgetGuard.cs index 6419b60..5e252b2 100644 --- a/src/ClawdDotNet.Core/Budget/BudgetGuard.cs +++ b/src/ClawdDotNet.Core/Budget/BudgetGuard.cs @@ -117,6 +117,10 @@ public sealed class BudgetGuard } } + /// Today's usage of an agent, or with an empty id of the whole instance. + public Task GetUsedTodayAsync(string agentId, CancellationToken ct) + => _usage.GetDailyAsync(DateOnly.FromDateTime(_now()), agentId, ct); + /// What is currently reserved by runs in flight — for the agent, or with an /// empty id for the whole instance. public RunEstimate Reserved(string agentId) diff --git a/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs b/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs new file mode 100644 index 0000000..d871ee5 --- /dev/null +++ b/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs @@ -0,0 +1,144 @@ +using ClawdDotNet.Core.Config; +using ClawdDotNet.Core.Tasks; + +namespace ClawdDotNet.Core.Budget; + +/// +/// #716: Decides whether a late run — carried-over work or a catch-up — may start now, +/// without taking budget the rest of the day still needs. +/// +/// The rule: the new day has right of way. Headroom is the limit minus what +/// was used, minus what runs in flight have reserved, minus what the day's remaining +/// scheduled runs will need, minus a safety margin. A late run starts only if its +/// estimate fits into that. Otherwise one overdrawn day would push its shortfall into the +/// next, and the next, instead of resolving it. +/// +/// This replaces the flat "at most 25 % of the new day" from the concept: a fixed +/// share is either too much on a busy day or needlessly little on a quiet one. The same +/// calculation is what fill work (#717) will run on. +/// +/// Estimates, honestly. A run is estimated from the agent's recent runs +/// (). Without history the estimate is zero — +/// then only the margin keeps a reserve. That errs towards letting late work through; +/// the budget check at the start of the run still refuses once the limit is reached. +/// +public sealed class BudgetHeadroom : ILateRunGate +{ + /// Share of each limit kept free as a reserve for surprises. + public const decimal SafetyShare = 0.10m; + + private readonly BudgetGuard _guard; + private readonly Func> _agents; + private readonly Func _instanceBudget; + private readonly TimeProvider _clock; + + public BudgetHeadroom( + BudgetGuard guard, + Func> agents, + Func instanceBudget, + TimeProvider? clock = null) + { + _guard = guard; + _agents = agents; + _instanceBudget = instanceBudget; + _clock = clock ?? TimeProvider.System; + } + + public async Task HasRoomAsync( + TaskItem lateTask, IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct) + { + var agents = _agents(); + var agent = FindAgent(agents, lateTask); + var instanceBudget = _instanceBudget(); + + // Without a known agent there is nothing to calculate against; the budget check at + // the start of the run still applies. + if (agent is null || (!agent.Budget.HasAnyLimit && !instanceBudget.HasAnyLimit)) + return true; + + var endOfDay = StartOfNextLocalDay(nowUtc); + var estimates = new Dictionary(StringComparer.OrdinalIgnoreCase); + + async Task EstimateFor(string agentId) + { + if (!estimates.TryGetValue(agentId, out var estimate)) + estimates[agentId] = estimate = await _guard.EstimateRunAsync(agentId, ct); + return estimate; + } + + var ownEstimate = await EstimateFor(agent.AgentId); + + // Regular work of the rest of the day: everything scheduled except late work + // itself — that is what has to wait for headroom, not what headroom is kept for. + var regular = runnable.Where(t => t.Id != lateTask.Id && !IsLate(t)).ToList(); + + if (agent.Budget.HasAnyLimit) + { + var demand = default(RunEstimate); + foreach (var task in regular.Where(t => IsOf(t, agent.AgentId))) + demand += Times(ownEstimate, TaskSchedule.OccurrencesBetween(task, nowUtc, endOfDay)); + + var used = await _guard.GetUsedTodayAsync(agent.AgentId, ct); + var reserved = _guard.Reserved(agent.AgentId); + + if (!Fits(agent.Budget.DailyCostUsd, used.CostUsd, reserved.CostUsd, demand.CostUsd, ownEstimate.CostUsd) + || !Fits(agent.Budget.DailyTokens, used.TotalTokens, reserved.Tokens, demand.Tokens, ownEstimate.Tokens)) + return false; + } + + if (instanceBudget.HasAnyLimit) + { + var demand = default(RunEstimate); + foreach (var task in regular) + { + var agentId = TaskAssignee.AgentId(task.Assignee); + if (agentId.Length == 0) + continue; + demand += Times(await EstimateFor(agentId), TaskSchedule.OccurrencesBetween(task, nowUtc, endOfDay)); + } + + var used = await _guard.GetUsedTodayAsync("", ct); + var reserved = _guard.Reserved(""); + + if (!Fits(instanceBudget.DailyCostUsd, used.CostUsd, reserved.CostUsd, demand.CostUsd, ownEstimate.CostUsd) + || !Fits(instanceBudget.DailyTokens, used.TotalTokens, reserved.Tokens, demand.Tokens, ownEstimate.Tokens)) + return false; + } + + return true; + } + + /// Carried-over work and pending catch-ups are the late runs. + public static bool IsLate(TaskItem task) => task.IsCarriedOver || task.CatchUpPending; + + /// 0 or less means: no limit of this kind. + private static bool Fits(decimal limit, decimal used, decimal reserved, decimal demand, decimal estimate) + => limit <= 0 || limit - used - reserved - demand - limit * SafetyShare >= estimate; + + private static bool Fits(long limit, long used, long reserved, long demand, long estimate) + => limit <= 0 || limit - used - reserved - demand - (long)(limit * SafetyShare) >= estimate; + + private static RunEstimate Times(RunEstimate estimate, int runs) + => new(estimate.CostUsd * runs, estimate.Tokens * runs); + + private static bool IsOf(TaskItem task, string agentId) + => string.Equals(TaskAssignee.AgentId(task.Assignee), agentId, StringComparison.OrdinalIgnoreCase); + + private static AgentConfig? FindAgent(IReadOnlyList agents, TaskItem task) + { + var agentId = TaskAssignee.AgentId(task.Assignee); + return agentId.Length == 0 + ? (agents.Count == 1 ? agents[0] : null) + : agents.FirstOrDefault(a => string.Equals(a.AgentId, agentId, StringComparison.OrdinalIgnoreCase)); + } + + private DateTime StartOfNextLocalDay(DateTime nowUtc) + { + var zone = _clock.LocalTimeZone; + var local = TimeZoneInfo.ConvertTimeFromUtc(DateTime.SpecifyKind(nowUtc, DateTimeKind.Utc), zone); + var midnight = DateTime.SpecifyKind(local.Date.AddDays(1), DateTimeKind.Unspecified); + while (zone.IsInvalidTime(midnight)) + midnight = midnight.AddMinutes(1); + return TimeZoneInfo.ConvertTimeToUtc(midnight, zone); + } +} diff --git a/src/ClawdDotNet.Core/Engine/AgentEngine.cs b/src/ClawdDotNet.Core/Engine/AgentEngine.cs index cc8389b..b7c44a6 100644 --- a/src/ClawdDotNet.Core/Engine/AgentEngine.cs +++ b/src/ClawdDotNet.Core/Engine/AgentEngine.cs @@ -35,6 +35,12 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut /// Tagesgrenzen der Instanz. Wird vom Host gesetzt. public InstanceBudget InstanceBudget { get; set; } = InstanceBudget.Unlimited; + /// + /// #716: The guard that checks and reserves every run — the scanner's headroom + /// calculation has to see the same reservations. null without usage recording. + /// + public BudgetGuard? BudgetGuard => _budgetGuard; + /// /// Wie viele Weitergaben eine Delegationskette höchstens haben darf (B8). /// A→B ist die Tiefe 1, A→B→C die Tiefe 2. diff --git a/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs b/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs index a8cadc2..be99078 100644 --- a/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs +++ b/src/ClawdDotNet.Core/Tasks/ITaskRepository.cs @@ -29,6 +29,13 @@ public interface ITaskRepository /// Task> ListRunnableAsync(CancellationToken ct); + /// + /// Every task, without a cap — for the few callers that must see the whole board: the + /// cost forecast, duplicate detection, the import's orphan sweep. Before #716 they used + /// with its limit of 500 and silently saw only part of it. + /// + Task> ListAllAsync(bool includeArchived, CancellationToken ct); + /// /// Beansprucht einen fälligen Termin atomar. Genau ein gleichzeitiger Aufruf gewinnt; /// jeder weitere erhält false. Setzt den Marker () diff --git a/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs b/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs index dba39e2..0a7d7ff 100644 --- a/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs +++ b/src/ClawdDotNet.Core/Tasks/SqliteTaskRepository.cs @@ -270,6 +270,22 @@ public sealed class SqliteTaskRepository : ITaskRepository return await cmd.ExecuteNonQueryAsync(ct); }, ct); + public async Task> ListAllAsync(bool includeArchived, CancellationToken ct) + { + await using var conn = await _storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT " + Columns + " FROM Tasks" + + (includeArchived ? "" : " WHERE Status <> 'archived'") + + " ORDER BY Priority DESC, UpdatedAt DESC"; + + var results = new List(); + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + results.Add(Read(reader)); + + return results; + } + public async Task> ListRunnableAsync(CancellationToken ct) { await using var conn = await _storage.OpenConnectionAsync(ct); diff --git a/src/ClawdDotNet.Core/Tasks/TaskScanner.cs b/src/ClawdDotNet.Core/Tasks/TaskScanner.cs index c1571c6..da5f36e 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskScanner.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskScanner.cs @@ -44,6 +44,15 @@ public interface IMarketCalendar bool IsOpen(DateTime nowUtc); } +/// +/// #716: Decides whether a late run — carried-over work or a catch-up — may start now. +/// Without one, late runs only wait for the regular ones of the same tick. +/// +public interface ILateRunGate +{ + Task HasRoomAsync(TaskItem lateTask, IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct); +} + /// Bis C1 kommt: der Markt gilt als immer offen. public sealed class AlwaysOpenMarketCalendar : IMarketCalendar { @@ -110,6 +119,11 @@ public sealed class TaskScanner : IAsyncDisposable private readonly Dictionary _missesToday = new(StringComparer.OrdinalIgnoreCase); private DateOnly? _missDay; + private readonly ILateRunGate? _lateRunGate; + + /// How many late runs waited for budget headroom at the last tick. + private int _lastWaitingForHeadroom; + private readonly CancellationTokenSource _cts = new(); private Task? _loop; @@ -122,7 +136,8 @@ public sealed class TaskScanner : IAsyncDisposable TimeSpan? tick = null, TimeSpan? lease = null, int maxParallelRuns = DefaultMaxParallelRuns, - Func? budgetMissPolicy = null) + Func? budgetMissPolicy = null, + ILateRunGate? lateRunGate = null) { _repo = repo; _dispatcher = dispatcher; @@ -133,6 +148,7 @@ public sealed class TaskScanner : IAsyncDisposable _lease = lease ?? TimeSpan.FromMinutes(15); _maxParallelRuns = Math.Max(1, maxParallelRuns); _missPolicy = budgetMissPolicy ?? (_ => BudgetMissPolicy.Expire); + _lateRunGate = lateRunGate; } /// How many scheduled runs may be in flight at once (#682). @@ -319,7 +335,8 @@ public sealed class TaskScanner : IAsyncDisposable var launched = new List(); var waiting = 0; - foreach (var (task, occ, _) in due) + var waitingForHeadroom = 0; + foreach (var (task, occ, late) in due) { // OpenCount also counts runs started by hand (RunTaskNowAsync) — they take a // slot, even though they did not wait for one. @@ -329,6 +346,14 @@ public sealed class TaskScanner : IAsyncDisposable continue; } + // #716: Late work only uses what the rest of the day does not need. + if (late && _lateRunGate is not null + && !await _lateRunGate.HasRoomAsync(task, candidates, now, ct)) + { + waitingForHeadroom++; + continue; + } + var token = Guid.NewGuid().ToString("N"); if (await _repo.TryClaimAsync(task.Id, occ, token, now, leaseCutoff, ct)) launched.Add(Track(RunLoggedAsync(task, token, ct))); @@ -347,6 +372,16 @@ public sealed class TaskScanner : IAsyncDisposable _lastWaiting = waiting; } + if (waitingForHeadroom != _lastWaitingForHeadroom) + { + if (waitingForHeadroom > 0) + _logger.LogInformation( + "{Waiting} carried-over or catch-up run(s) waiting for budget headroom — " + + "the rest of today's schedule comes first", + waitingForHeadroom); + _lastWaitingForHeadroom = waitingForHeadroom; + } + return launched; } diff --git a/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs b/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs index 4c87ea9..58bee19 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs @@ -201,6 +201,70 @@ public static class TaskSchedule } } + /// + /// #716: How many runs the task will still trigger from up to + /// (excluding) — including one that is due right now. The + /// basis for "how much budget does the rest of the day still need". + /// + public static int OccurrencesBetween(TaskItem task, DateTime fromUtc, DateTime toUtc, int cap = 1_440) + { + fromUtc = DateTime.SpecifyKind(fromUtc, DateTimeKind.Utc); + toUtc = DateTime.SpecifyKind(toUtc, DateTimeKind.Utc); + if (toUtc <= fromUtc) + return 0; + + var dueNow = DueOccurrence(task, fromUtc) is not null ? 1 : 0; + + if (task.When is not { } when) + return dueNow; // one-time without a schedule: due until it ran + + switch (when.Kind) + { + case TaskWhenKind.At: + { + if (dueNow == 1 || task.LastOccurrence is not null) + return dueNow; + return TryResolveInstant(when.Value, when.TimeZone, out var at) && at < toUtc ? 1 : 0; + } + + case TaskWhenKind.Every: + { + if (!TryParseInterval(when.Value, out var interval) || interval <= TimeSpan.Zero) + return dueNow; + + var next = dueNow == 1 + ? fromUtc + interval + : NextOccurrenceUtc(task, fromUtc) ?? toUtc; + var count = dueNow; + while (count < cap && next < toUtc) + { + count++; + next += interval; + } + return count; + } + + case TaskWhenKind.Cron: + { + if (!TryPrepareCron(when, out var cron, out var tz)) + return 0; + + var count = dueNow; + var cursor = fromUtc; + while (count < cap && NextCronUtc(cron, tz, cursor) is { } next && next < toUtc) + { + if (next > fromUtc) + count++; + cursor = next; + } + return count; + } + + default: + return dueNow; + } + } + private static bool TryPrepareCron(TaskWhen when, out CronExpression cron, out TimeZoneInfo tz) { cron = null!; diff --git a/src/ClawdDotNet.Core/Tasks/TaskboardService.cs b/src/ClawdDotNet.Core/Tasks/TaskboardService.cs index b0612f3..171c8e1 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskboardService.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskboardService.cs @@ -31,6 +31,10 @@ public sealed class TaskboardService public Task> ListAsync(TaskQuery query, CancellationToken ct) => _repo.ListAsync(query, ct); + /// The whole board, without the cap of . + public Task> ListAllAsync(CancellationToken ct, bool includeArchived = false) + => _repo.ListAllAsync(includeArchived, ct); + // ─── Anlegen ─── /// @@ -104,9 +108,9 @@ public sealed class TaskboardService if (task is not null) { seen.Add(task.Id); imported++; } } - // Verwaiste Zeilen archivieren. Der Deckel von 500 ist für den Start unkritisch; - // wächst das Board darüber hinaus, gehört ohnehin das Aufräumen (siehe Konzept) her. - var known = await _repo.ListAsync(new TaskQuery { IncludeArchived = true, Limit = 500 }, ct); + // Verwaiste Zeilen archivieren. Ungekappt (#716): Vorher sah dieser Abgleich nur die + // ersten 500 Zeilen, und darüber blieben verwaiste Zeilen still stehen. + var known = await _repo.ListAllAsync(includeArchived: true, ct); foreach (var task in known) if (task.Status != TaskItemStatus.Archived && !seen.Contains(task.Id)) await _repo.SetStatusAsync(task.Id, TaskItemStatus.Archived, DateTime.UtcNow, ct); diff --git a/src/ClawdDotNet.Tools.Taskboard/TaskboardTool.cs b/src/ClawdDotNet.Tools.Taskboard/TaskboardTool.cs index fb2ab8c..e0c71b3 100644 --- a/src/ClawdDotNet.Tools.Taskboard/TaskboardTool.cs +++ b/src/ClawdDotNet.Tools.Taskboard/TaskboardTool.cs @@ -210,7 +210,9 @@ public sealed class TaskboardTool : IAgentTool private static async Task> SucheAehnlicheAsync( TaskboardService board, string title, string? exceptId, CancellationToken ct) { - var alle = await board.ListAsync(new TaskQuery { Limit = 500 }, ct); + // Uncapped (#716): with a limit of 500, a duplicate among older, low-priority tasks + // stayed invisible once the board grew beyond that. + var alle = await board.ListAllAsync(ct); return TaskSimilarity.FindOpen(alle, title, exceptId); } diff --git a/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs b/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs new file mode 100644 index 0000000..d0fa360 --- /dev/null +++ b/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs @@ -0,0 +1,166 @@ +using ClawdDotNet.Core.Accounting; +using ClawdDotNet.Core.Budget; +using ClawdDotNet.Core.Config; +using ClawdDotNet.Core.Tasks; +using Shouldly; + +namespace ClawdDotNet.Core.Tests.Budget; + +/// +/// #716, stage 3: late work — carried-over tasks and catch-ups — only uses budget the rest +/// of the day does not need. The new day has right of way. +/// +public sealed class BudgetHeadroomTests +{ + // 2027-03-10 14:00 UTC; the test clock's local zone is UTC, so the day ends at 24:00 UTC. + private static readonly DateTime Now = new(2027, 3, 10, 14, 0, 0, DateTimeKind.Utc); + private static readonly DateTime CreatedEarlier = new(2027, 3, 1, 0, 0, 0, DateTimeKind.Utc); + + private static AgentConfig Agent(decimal dailyCost = 1.00m) + => new() { AgentId = "worker", Budget = new AgentBudget { DailyCostUsd = dailyCost } }; + + /// Runs averaging 0.05 USD. + private static List History(decimal cost = 0.05m) + => [.. Enumerable.Range(0, 5).Select(_ => new RunUsage { AgentId = "worker", PromptTokens = 1_000, CostUsd = cost, CostIsKnown = true })]; + + private static TaskItem Cron(string id, string expression, string? lastOccurrence = null) => new() + { + Id = id, Title = id, Assignee = "@worker", CreatedAt = CreatedEarlier, LastOccurrence = lastOccurrence, + When = new TaskWhen { Kind = TaskWhenKind.Cron, Value = expression, TimeZone = "UTC" } + }; + + private static TaskItem Carried(string id) => new() + { + Id = id, Title = id, Assignee = "@worker", CreatedAt = CreatedEarlier, BudgetMissCount = 1 + }; + + private static BudgetHeadroom Headroom(decimal used, AgentConfig agent, InstanceBudget? instance = null, decimal perRun = 0.05m) + => new(new BudgetGuard(new FakeUsage(used, History(perRun))), + () => [agent], () => instance ?? InstanceBudget.Unlimited, new UtcClock(Now)); + + // ═══════════════════ counting the rest of the day ═══════════════════ + + [Fact] + public void Hourly_from_14_30_until_midnight_is_nine_more_runs() + { + // 15:00 … 23:00. The 14:00 run is done (marker), so nothing is due right now. + var task = Cron("t", "0 * * * *", lastOccurrence: "2027-03-10T14:00:00.0000000Z"); + + TaskSchedule.OccurrencesBetween(task, Now.AddMinutes(30), Now.Date.AddDays(1)).ShouldBe(9); + } + + [Fact] + public void A_run_due_right_now_counts_as_well() + { + // Last run 13:00, so the 14:00 run is due now; then 15:00 … 23:00. + var task = Cron("t", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + + TaskSchedule.OccurrencesBetween(task, Now.AddMinutes(1), Now.Date.AddDays(1)).ShouldBe(10); + } + + [Fact] + public void A_one_time_task_counts_once_until_it_ran() + { + var open = new TaskItem { Id = "t", CreatedAt = CreatedEarlier }; + var done = open with { LastOccurrence = "2027-03-10T09:00:00.0000000Z" }; + + TaskSchedule.OccurrencesBetween(open, Now, Now.AddHours(1)).ShouldBe(1); + TaskSchedule.OccurrencesBetween(done, Now, Now.AddHours(1)).ShouldBe(0); + } + + [Fact] + public void An_interval_counts_its_remaining_ticks() + { + var task = new TaskItem + { + Id = "t", CreatedAt = CreatedEarlier, LastOccurrence = "2027-03-10T13:45:00.0000000Z", + When = new TaskWhen { Kind = TaskWhenKind.Every, Value = "30m", TimeZone = "UTC" } + }; + + // 14:15, 14:45, 15:15, 15:45 before 16:00. + TaskSchedule.OccurrencesBetween(task, Now, Now.AddHours(2)).ShouldBe(4); + } + + // ═══════════════════ headroom ═══════════════════ + + [Fact] + public async Task Late_work_fits_when_the_rest_of_the_day_leaves_room() + { + // Limit 1.00, used 0.30, margin 0.10. Rest of the day: 10 hourly runs à 0.05 = 0.50. + // Left: 1.00 - 0.30 - 0.50 - 0.10 = 0.10 ≥ 0.05 for the late run. + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + var late = Carried("t-late"); + + var room = await Headroom(used: 0.30m, Agent()).HasRoomAsync(late, [hourly, late], Now, default); + + room.ShouldBeTrue(); + } + + [Fact] + public async Task Late_work_waits_when_the_rest_of_the_day_needs_the_budget() + { + // Same, but 0.40 used: 1.00 - 0.40 - 0.50 - 0.10 = 0.00 < 0.05. + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + var late = Carried("t-late"); + + var room = await Headroom(used: 0.40m, Agent()).HasRoomAsync(late, [hourly, late], Now, default); + + room.ShouldBeFalse("the day's own schedule has right of way"); + } + + [Fact] + public async Task Other_late_work_does_not_count_as_the_days_demand() + { + // Ten carried-over tasks do not reserve budget against each other — they are what + // waits for headroom, not what headroom is kept for. + var late = Carried("t-late"); + var others = Enumerable.Range(0, 10).Select(i => Carried($"t-other-{i}")).ToList(); + + var room = await Headroom(used: 0.50m, Agent()).HasRoomAsync(late, [late, .. others], Now, default); + + room.ShouldBeTrue(); + } + + [Fact] + public async Task The_instance_limit_counts_the_schedules_of_all_agents() + { + var agent = Agent(dailyCost: 0); // no limit of its own + var instance = new InstanceBudget { DailyCostUsd = 1.00m }; + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + var late = Carried("t-late"); + + (await Headroom(used: 0.30m, agent, instance).HasRoomAsync(late, [hourly, late], Now, default)).ShouldBeTrue(); + (await Headroom(used: 0.40m, agent, instance).HasRoomAsync(late, [hourly, late], Now, default)).ShouldBeFalse(); + } + + [Fact] + public async Task Without_any_limit_there_is_always_room() + { + var late = Carried("t-late"); + + (await Headroom(used: 99m, Agent(dailyCost: 0)).HasRoomAsync(late, [late], Now, default)).ShouldBeTrue(); + } + + // ═══════════════════ test doubles ═══════════════════ + + private sealed class UtcClock(DateTime now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => new(now, TimeSpan.Zero); + public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc; + } + + private sealed class FakeUsage(decimal usedCost, IReadOnlyList recent) : IUsageRepository + { + public Task RecordAsync(RunUsage usage, CancellationToken ct) => Task.CompletedTask; + + public Task GetDailyAsync(DateOnly date, string agentId, CancellationToken ct) + => Task.FromResult(new DailyUsage(date, 0, 0, usedCost, true, 0)); + + public Task> GetRecentAsync(int limit, CancellationToken ct) => Task.FromResult(recent); + + public Task> GetRecentForAgentAsync(string agentId, int limit, CancellationToken ct) + => Task.FromResult(recent); + + public Task PurgeBeforeAsync(DateOnly date, CancellationToken ct) => Task.FromResult(0); + } +} diff --git a/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs b/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs index c3cb850..44edfa7 100644 --- a/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs +++ b/tests/ClawdDotNet.Core.Tests/Tasks/BudgetMissTests.cs @@ -238,6 +238,42 @@ public sealed class BudgetMissTests : IDisposable task.DeferredUntil.ShouldBeNull("a shutdown keeps the old 30-minute retry"); } + [Fact] + public async Task Late_work_waits_while_the_headroom_gate_says_no() + { + await SeedOneTime("t-yesterday"); + var dispatcher = new BudgetDispatcher { HasBudget = false }; + var gate = new FixedGate { Room = false }; + var scanner = new TaskScanner(_repo, dispatcher, NullLoggerFactory.Instance, + clock: _clock, lateRunGate: gate); + await scanner.ScanOnceAsync(default); + + MoveTo(NextLocalMidnightUtc().AddMinutes(1)); + await SeedOneTime("t-today"); + dispatcher.HasBudget = true; + dispatcher.Started.Clear(); + + await scanner.ScanOnceAsync(default); + dispatcher.Started.ShouldBe(["t-today"], "regular work never asks the gate"); + gate.Asked.ShouldBe(["t-yesterday"]); + + gate.Room = true; + await scanner.ScanOnceAsync(default); + dispatcher.Started.ShouldBe(["t-today", "t-yesterday"]); + } + + private sealed class FixedGate : ILateRunGate + { + public bool Room { get; set; } + public List Asked { get; } = []; + + public Task HasRoomAsync(TaskItem lateTask, IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct) + { + Asked.Add(lateTask.Id); + return Task.FromResult(Room); + } + } + // ═══════════════════ test double ═══════════════════ private sealed class BudgetDispatcher : ITaskDispatcher diff --git a/tests/ClawdDotNet.Tools.Tests/Taskboard/DublettenTests.cs b/tests/ClawdDotNet.Tools.Tests/Taskboard/DublettenTests.cs index 2051746..2c7e548 100644 --- a/tests/ClawdDotNet.Tools.Tests/Taskboard/DublettenTests.cs +++ b/tests/ClawdDotNet.Tools.Tests/Taskboard/DublettenTests.cs @@ -143,6 +143,24 @@ public sealed class DublettenTests : IDisposable (await _board.ListAsync(new TaskQuery { Limit = 50 }, default)).Count.ShouldBe(2); } + [Fact] + public async Task Die_Dublette_wird_auch_hinter_500_anderen_Aufgaben_gefunden() + { + // #716: Die Suche las ListAsync mit Limit 500, sortiert nach Prioritaet. Eine alte + // Aufgabe mit niedriger Prioritaet fiel damit aus dem Blick, sobald das Brett wuchs. + var alt = await _board.CreateAsync(new TaskItem + { + Title = "Betriebsluecke: Eur Kurse nicht aktuell", Assignee = "@human", Priority = 1 + }, default); + for (var i = 0; i < 510; i++) + await _repo.UpsertAsync(new TaskItem { Id = $"t-fill-{i}", Title = $"Anderes Thema {i}", Priority = 5 }, default); + + var result = await Run("""{"action":"task_create","title":"Betriebsluecke: Kurse EUR nicht aktuell","assignee":"@human"}"""); + + result.Success.ShouldBeFalse("die alte Aufgabe meint dasselbe"); + result.ErrorMessage.ShouldNotBeNull().ShouldContain(alt.Id); + } + [Fact] public async Task Eine_erledigte_Aufgabe_blockiert_den_naechsten_Befund_nicht() { From 32383119dae7a96bef2d939ec9df750e880d0e42 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 24 Sep 2026 12:12:44 +0200 Subject: [PATCH 4/4] Budgetende, Stufe 4: den Engpass melden, bevor er eintritt (#716) - TaskSchedule.OccurrenceTimesBetween liefert die Zeitpunkte der restlichen Laeufe; OccurrencesBetween zaehlt sie nur noch. - BudgetHeadroom implementiert IBudgetOutlook: Die restlichen geplanten Laeufe des Tages werden in zeitlicher Reihenfolge durchgegangen, je Agent mit Grenze und fuer die Instanz, bis zum ersten Lauf, der das Budget verbraucht vorfaende. Das spiegelt die Pruefung beim Start eines Laufs. Spaetwerk zaehlt nicht mit, es laeuft ohnehin nur im Spielraum. - Der Scanner schaut alle 30 Minuten voraus und nennt je Agent und Tag einmal, wann das Budget ausgeht und wie viele der geplanten Laeufe dann ausfallen, solange noch Zeit ist, nachzulegen oder zu entscheiden, was wartet. Die Meldung steht vorerst im Protokoll (Warnung). Der Weg zum Menschen ueber einen Kanal gehoert zu #646, die Anzeige ins Lagebild (#711). Tests: 4 neu fuer die Vorschau. 1.261 gruen. Co-Authored-By: Claude Opus 5.5 --- src/ClawdDotNet.App/AppHost.cs | 12 ++- src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs | 78 ++++++++++++++++++- src/ClawdDotNet.Core/Tasks/TaskScanner.cs | 70 ++++++++++++++++- src/ClawdDotNet.Core/Tasks/TaskSchedule.cs | 45 ++++++----- .../Budget/BudgetHeadroomTests.cs | 48 ++++++++++++ 5 files changed, 229 insertions(+), 24 deletions(-) diff --git a/src/ClawdDotNet.App/AppHost.cs b/src/ClawdDotNet.App/AppHost.cs index 3268f28..c9430b2 100644 --- a/src/ClawdDotNet.App/AppHost.cs +++ b/src/ClawdDotNet.App/AppHost.cs @@ -726,6 +726,10 @@ public sealed class AppHost : IAsyncDisposable var dispatcher = new EngineTaskDispatcher( engine, () => instance.Agents, instance.InstanceId, tools, stateStore, loggerFactory); + var headroom = engine.BudgetGuard is { } guard + ? new BudgetHeadroom(guard, () => instance.Agents, () => engine.InstanceBudget) + : null; + var scanner = new TaskScanner(taskRepository, dispatcher, loggerFactory, maxParallelRuns: instance.MaxParallelTaskRuns, budgetMissPolicy: task => @@ -736,10 +740,10 @@ public sealed class AppHost : IAsyncDisposable .FirstOrDefault(a => string.Equals(a.AgentId, agentId, StringComparison.OrdinalIgnoreCase)) ?.Budget.MissPolicy ?? BudgetMissPolicy.Expire; }, - // #716: Late work waits for headroom the rest of the day does not need. - lateRunGate: engine.BudgetGuard is { } guard - ? new BudgetHeadroom(guard, () => instance.Agents, () => engine.InstanceBudget) - : null); + // #716: Late work waits for headroom the rest of the day does not need, and a + // day the budget will not last is named while there is still time to act. + lateRunGate: headroom, + budgetOutlook: headroom); // Reconciliation nicht blockierend: Der Start soll nicht auf das Dateisystem warten. _ = Task.Run(async () => diff --git a/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs b/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs index d871ee5..18b9b8a 100644 --- a/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs +++ b/src/ClawdDotNet.Core/Budget/BudgetHeadroom.cs @@ -22,7 +22,7 @@ namespace ClawdDotNet.Core.Budget; /// then only the margin keeps a reserve. That errs towards letting late work through; /// the budget check at the start of the run still refuses once the limit is reached. /// -public sealed class BudgetHeadroom : ILateRunGate +public sealed class BudgetHeadroom : ILateRunGate, IBudgetOutlook { /// Share of each limit kept free as a reserve for surprises. public const decimal SafetyShare = 0.10m; @@ -108,6 +108,82 @@ public sealed class BudgetHeadroom : ILateRunGate return true; } + /// + /// #716: Walks through the rest of today's schedule in time order and finds, per agent + /// with a limit and for the instance, the first run the budget would no longer cover. + /// Late work is left out: it only ever runs in headroom, so it cannot be what makes the + /// day's own runs fail. + /// + public async Task> ShortfallsAsync( + IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct) + { + var agents = _agents(); + var instanceBudget = _instanceBudget(); + var endOfDay = StartOfNextLocalDay(nowUtc); + var result = new List(); + + // Every planned run until midnight, with whose it is. + var planned = new List<(DateTime At, string AgentId)>(); + foreach (var task in runnable.Where(t => !IsLate(t))) + { + var agent = FindAgent(agents, task); + if (agent is null) + continue; + foreach (var at in TaskSchedule.OccurrenceTimesBetween(task, nowUtc, endOfDay)) + planned.Add((at, agent.AgentId)); + } + + planned.Sort((a, b) => a.At.CompareTo(b.At)); + + var estimates = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var agentId in planned.Select(p => p.AgentId).Distinct(StringComparer.OrdinalIgnoreCase)) + estimates[agentId] = await _guard.EstimateRunAsync(agentId, ct); + + foreach (var agent in agents.Where(a => a.Budget.HasAnyLimit)) + { + var runs = planned.Where(p => string.Equals(p.AgentId, agent.AgentId, StringComparison.OrdinalIgnoreCase)).ToList(); + var used = await _guard.GetUsedTodayAsync(agent.AgentId, ct); + var reserved = _guard.Reserved(agent.AgentId); + if (FirstRefused(runs, estimates, agent.Budget.DailyCostUsd, agent.Budget.DailyTokens, used, reserved) is { } index) + result.Add(new BudgetShortfall(agent.AgentId, runs[index].At, runs.Count - index, runs.Count)); + } + + if (instanceBudget.HasAnyLimit) + { + var used = await _guard.GetUsedTodayAsync("", ct); + var reserved = _guard.Reserved(""); + if (FirstRefused(planned, estimates, instanceBudget.DailyCostUsd, instanceBudget.DailyTokens, used, reserved) is { } index) + result.Add(new BudgetShortfall("", planned[index].At, planned.Count - index, planned.Count)); + } + + return result; + } + + /// + /// The index of the first run that would find the budget used up, or null. + /// Mirrors the check at the start of a run: a run is refused once usage has reached + /// the limit, not merely because its own estimate would overshoot. + /// + private static int? FirstRefused( + List<(DateTime At, string AgentId)> runs, Dictionary estimates, + decimal costLimit, long tokenLimit, Accounting.DailyUsage used, RunEstimate reserved) + { + var cost = used.CostUsd + reserved.CostUsd; + long tokens = used.TotalTokens + reserved.Tokens; + + for (var i = 0; i < runs.Count; i++) + { + if ((costLimit > 0 && cost >= costLimit) || (tokenLimit > 0 && tokens >= tokenLimit)) + return i; + + var estimate = estimates.GetValueOrDefault(runs[i].AgentId); + cost += estimate.CostUsd; + tokens += estimate.Tokens; + } + + return null; + } + /// Carried-over work and pending catch-ups are the late runs. public static bool IsLate(TaskItem task) => task.IsCarriedOver || task.CatchUpPending; diff --git a/src/ClawdDotNet.Core/Tasks/TaskScanner.cs b/src/ClawdDotNet.Core/Tasks/TaskScanner.cs index da5f36e..f6f349a 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskScanner.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskScanner.cs @@ -53,6 +53,21 @@ public interface ILateRunGate Task HasRoomAsync(TaskItem lateTask, IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct); } +/// +/// #716: The day ahead — where the budget will not reach the end of today's schedule. +/// +public interface IBudgetOutlook +{ + Task> ShortfallsAsync(IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct); +} + +/// +/// #716: An agent whose budget runs out before today's schedule does — from when on, and +/// how many planned runs will be refused. is empty for the instance +/// limit. +/// +public sealed record BudgetShortfall(string AgentId, DateTime ExhaustsAtUtc, int RunsRefused, int RunsPlanned); + /// Bis C1 kommt: der Markt gilt als immer offen. public sealed class AlwaysOpenMarketCalendar : IMarketCalendar { @@ -121,6 +136,16 @@ public sealed class TaskScanner : IAsyncDisposable private readonly ILateRunGate? _lateRunGate; + private readonly IBudgetOutlook? _outlook; + + /// How often the day ahead is looked at — a forecast, not a watchdog. + public static readonly TimeSpan OutlookInterval = TimeSpan.FromMinutes(30); + + private DateTime _lastOutlookAt = DateTime.MinValue; + + /// Agents already warned about today, with the day — once per agent and day. + private readonly Dictionary _shortfallWarned = new(StringComparer.OrdinalIgnoreCase); + /// How many late runs waited for budget headroom at the last tick. private int _lastWaitingForHeadroom; @@ -137,7 +162,8 @@ public sealed class TaskScanner : IAsyncDisposable TimeSpan? lease = null, int maxParallelRuns = DefaultMaxParallelRuns, Func? budgetMissPolicy = null, - ILateRunGate? lateRunGate = null) + ILateRunGate? lateRunGate = null, + IBudgetOutlook? budgetOutlook = null) { _repo = repo; _dispatcher = dispatcher; @@ -149,6 +175,7 @@ public sealed class TaskScanner : IAsyncDisposable _maxParallelRuns = Math.Max(1, maxParallelRuns); _missPolicy = budgetMissPolicy ?? (_ => BudgetMissPolicy.Expire); _lateRunGate = lateRunGate; + _outlook = budgetOutlook; } /// How many scheduled runs may be in flight at once (#682). @@ -257,6 +284,7 @@ public sealed class TaskScanner : IAsyncDisposable var candidates = await _repo.ListRunnableAsync(ct); SummarizeMissesIfNewDay(now); + await WarnAboutShortfallsAsync(candidates, now, ct); var due = new List<(TaskItem Task, string Occurrence, bool Late)>(); foreach (var task in candidates) @@ -563,6 +591,46 @@ public sealed class TaskScanner : IAsyncDisposable : "occurrence lapsed (policy: expire)"); } + /// + /// #716: Prevent rather than catch up. Every half hour the day ahead is checked; an + /// agent whose budget will not last until midnight is named once a day — with the time + /// it runs out and how many planned runs that costs — while there is still time to + /// raise the budget or decide what can go. + /// + private async Task WarnAboutShortfallsAsync(IReadOnlyList runnable, DateTime nowUtc, CancellationToken ct) + { + if (_outlook is null || nowUtc - _lastOutlookAt < OutlookInterval) + return; + _lastOutlookAt = nowUtc; + + IReadOnlyList shortfalls; + try + { + shortfalls = await _outlook.ShortfallsAsync(runnable, nowUtc, ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "Budget outlook failed — no forecast this time"); + return; + } + + var today = DateOnly.FromDateTime(TimeZoneInfo.ConvertTimeFromUtc(nowUtc, _clock.LocalTimeZone)); + foreach (var shortfall in shortfalls) + { + if (_shortfallWarned.TryGetValue(shortfall.AgentId, out var warnedOn) && warnedOn == today) + continue; + _shortfallWarned[shortfall.AgentId] = today; + + var local = TimeZoneInfo.ConvertTimeFromUtc(shortfall.ExhaustsAtUtc, _clock.LocalTimeZone); + _logger.LogWarning( + "Budget forecast: the daily budget of {Who} will run out around {Time:HH:mm} — " + + "{Refused} of {Planned} runs planned for the rest of today would be refused. " + + "Raise the budget or decide what can wait.", + shortfall.AgentId.Length == 0 ? "the instance" : $"agent '{shortfall.AgentId}'", + local, shortfall.RunsRefused, shortfall.RunsPlanned); + } + } + /// /// #716: On the first tick of a new local day, one line per agent about what the /// budget refused the day before. The details are on the tasks themselves. diff --git a/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs b/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs index 58bee19..c5d01c3 100644 --- a/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs +++ b/src/ClawdDotNet.Core/Tasks/TaskSchedule.cs @@ -207,61 +207,70 @@ public static class TaskSchedule /// basis for "how much budget does the rest of the day still need". /// public static int OccurrencesBetween(TaskItem task, DateTime fromUtc, DateTime toUtc, int cap = 1_440) + => OccurrenceTimesBetween(task, fromUtc, toUtc, cap).Count; + + /// + /// #716: When the task will run from up to (excluding) + /// . A run due right now is listed at . + /// + public static IReadOnlyList OccurrenceTimesBetween(TaskItem task, DateTime fromUtc, DateTime toUtc, int cap = 1_440) { fromUtc = DateTime.SpecifyKind(fromUtc, DateTimeKind.Utc); toUtc = DateTime.SpecifyKind(toUtc, DateTimeKind.Utc); + var times = new List(); if (toUtc <= fromUtc) - return 0; + return times; - var dueNow = DueOccurrence(task, fromUtc) is not null ? 1 : 0; + var dueNow = DueOccurrence(task, fromUtc) is not null; + if (dueNow) + times.Add(fromUtc); if (task.When is not { } when) - return dueNow; // one-time without a schedule: due until it ran + return times; // one-time without a schedule: due until it ran switch (when.Kind) { case TaskWhenKind.At: { - if (dueNow == 1 || task.LastOccurrence is not null) - return dueNow; - return TryResolveInstant(when.Value, when.TimeZone, out var at) && at < toUtc ? 1 : 0; + if (!dueNow && task.LastOccurrence is null + && TryResolveInstant(when.Value, when.TimeZone, out var at) && at < toUtc) + times.Add(at); + return times; } case TaskWhenKind.Every: { if (!TryParseInterval(when.Value, out var interval) || interval <= TimeSpan.Zero) - return dueNow; + return times; - var next = dueNow == 1 + var next = dueNow ? fromUtc + interval : NextOccurrenceUtc(task, fromUtc) ?? toUtc; - var count = dueNow; - while (count < cap && next < toUtc) + while (times.Count < cap && next < toUtc) { - count++; + times.Add(next); next += interval; } - return count; + return times; } case TaskWhenKind.Cron: { if (!TryPrepareCron(when, out var cron, out var tz)) - return 0; + return times; - var count = dueNow; var cursor = fromUtc; - while (count < cap && NextCronUtc(cron, tz, cursor) is { } next && next < toUtc) + while (times.Count < cap && NextCronUtc(cron, tz, cursor) is { } next && next < toUtc) { if (next > fromUtc) - count++; + times.Add(next); cursor = next; } - return count; + return times; } default: - return dueNow; + return times; } } diff --git a/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs b/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs index d0fa360..99c6d90 100644 --- a/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs +++ b/tests/ClawdDotNet.Core.Tests/Budget/BudgetHeadroomTests.cs @@ -141,6 +141,54 @@ public sealed class BudgetHeadroomTests (await Headroom(used: 99m, Agent(dailyCost: 0)).HasRoomAsync(late, [late], Now, default)).ShouldBeTrue(); } + // ═══════════════════ the day ahead ═══════════════════ + + [Fact] + public async Task The_outlook_names_the_hour_the_budget_runs_out() + { + // Limit 1.00, used 0.60, runs at 14:00 (due now), 15:00 … 23:00 à 0.05. Before the + // 22:00 run 0.60 + 8 × 0.05 = 1.00 is used — from there on everything is refused. + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + + var shortfalls = await Headroom(used: 0.60m, Agent()).ShortfallsAsync([hourly], Now, default); + + var shortfall = shortfalls.ShouldHaveSingleItem(); + shortfall.AgentId.ShouldBe("worker"); + shortfall.ExhaustsAtUtc.ShouldBe(new DateTime(2027, 3, 10, 22, 0, 0, DateTimeKind.Utc)); + shortfall.RunsRefused.ShouldBe(2); + shortfall.RunsPlanned.ShouldBe(10); + } + + [Fact] + public async Task A_day_that_fits_has_no_shortfall() + { + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + + (await Headroom(used: 0.30m, Agent()).ShortfallsAsync([hourly], Now, default)).ShouldBeEmpty(); + } + + [Fact] + public async Task Late_work_does_not_make_the_day_fall_short() + { + // Fifty carried-over tasks only ever run in headroom — they cannot be the reason the + // day's own runs are refused. + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + var late = Enumerable.Range(0, 50).Select(i => Carried($"t-late-{i}")).ToList(); + + (await Headroom(used: 0.30m, Agent()).ShortfallsAsync([hourly, .. late], Now, default)).ShouldBeEmpty(); + } + + [Fact] + public async Task The_instance_limit_gets_its_own_forecast() + { + var hourly = Cron("t-hourly", "0 * * * *", lastOccurrence: "2027-03-10T13:00:00.0000000Z"); + var instance = new InstanceBudget { DailyCostUsd = 1.00m }; + + var shortfalls = await Headroom(used: 0.60m, Agent(dailyCost: 0), instance).ShortfallsAsync([hourly], Now, default); + + shortfalls.ShouldHaveSingleItem().AgentId.ShouldBe("", "empty means: the instance"); + } + // ═══════════════════ test doubles ═══════════════════ private sealed class UtcClock(DateTime now) : TimeProvider