Kosten je Einheit aufrechnen und Abweichungen melden (#699)
- IUsageRepository.GetDailyByAgentAsync: Tagessummen je Agent, Abo-Tokens getrennt - UnitCosts: je Einheit samt Untereinheiten heute/7/30 Tage, Monat, Vormonat, Durchschnittstag (28 Tage, ab 7 Tagen Verlauf), Maximum = Summe der Agentengrenzen - Meldungen: Abweichung vom eigenen Verlauf, Naehe zur Grenzsumme, Monatsschwellen der Instanz; CostWatch alle 15 min, je Einheit/Tag einmal (Tabelle CostAlerts), als Aufgabe an den Menschen; schaltet nichts ab - API /usage/units, /usage/cost-alerts; Zeitreihe gruppiert nach Einheit - WebUI: Karte "Kosten je Einheit" mit Meldungen in der Verbrauchsansicht; i18n en/de - Einstellungen instance.json costWatch - Tests: UnitCostsTests (8) Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
7eb211fd06
commit
da34b215f7
@@ -0,0 +1,134 @@
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Core.Accounting;
|
||||
|
||||
/// <summary>#699: An alert as it was raised — for the usage view.</summary>
|
||||
public sealed record RaisedCostAlert(string Period, string Scope, CostAlertKind Kind, string Message, DateTime RaisedAt);
|
||||
|
||||
/// <summary>
|
||||
/// #699: Looks at the costs of the units every few minutes and raises what changed — once per
|
||||
/// day and unit (a monthly threshold once per month). An alert goes to the human as a task and
|
||||
/// stays in the usage view; it switches nothing off.
|
||||
/// </summary>
|
||||
public sealed class CostWatch(
|
||||
SqliteStorage storage,
|
||||
IUsageRepository usage,
|
||||
Func<OrgChart> organisation,
|
||||
Func<CostWatchSettings> settings,
|
||||
TaskboardService? board,
|
||||
ILogger logger,
|
||||
TimeProvider? clock = null) : IAsyncDisposable
|
||||
{
|
||||
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(15);
|
||||
|
||||
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task? _loop;
|
||||
|
||||
/// <summary>The day the budget counts — the same local day as the BudgetGuard's.</summary>
|
||||
private DateOnly Today => DateOnly.FromDateTime(_clock.GetLocalNow().DateTime);
|
||||
|
||||
public async Task<IReadOnlyList<UnitCostLine>> ReportAsync(CancellationToken ct)
|
||||
{
|
||||
var today = Today;
|
||||
var rows = await usage.GetDailyByAgentAsync(new DateOnly(today.Year, today.Month, 1).AddMonths(-1).AddDays(-UnitCosts.AverageDays), today, ct);
|
||||
return UnitCosts.Build(organisation(), rows, today);
|
||||
}
|
||||
|
||||
/// <summary>Raises what is new today. Returns the alerts raised in this pass.</summary>
|
||||
public async Task<IReadOnlyList<CostAlert>> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
if (!settings().Enabled)
|
||||
return [];
|
||||
|
||||
var raised = new List<CostAlert>();
|
||||
foreach (var alert in UnitCosts.Alerts(await ReportAsync(ct), settings(), Today))
|
||||
{
|
||||
var period = alert.Kind == CostAlertKind.MonthlyThreshold ? alert.Date.ToString("yyyy-MM") : alert.Date.ToString("yyyy-MM-dd");
|
||||
if (!await TryRecordAsync(period, alert, ct))
|
||||
continue;
|
||||
raised.Add(alert);
|
||||
logger.LogWarning("Cost alert ({Kind}, {Scope}): {Message}", alert.Kind, alert.Scope, alert.Message);
|
||||
|
||||
if (board is not null)
|
||||
await board.CreateAsync(new TaskItem
|
||||
{
|
||||
Title = alert.Kind switch
|
||||
{
|
||||
CostAlertKind.Deviation => $"Cost alert: unusual day for {alert.Scope}",
|
||||
CostAlertKind.NearLimit => $"Cost alert: {alert.Scope} near its limits",
|
||||
_ => "Cost alert: monthly total crossed"
|
||||
},
|
||||
Body = alert.Message + "\n\nLook at the usage view (costs by unit). Nothing was switched off.",
|
||||
Assignee = TaskAssignee.Human,
|
||||
Priority = alert.Kind == CostAlertKind.NearLimit ? 4 : 3
|
||||
}, ct);
|
||||
}
|
||||
return raised;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<RaisedCostAlert>> RecentAsync(int days, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT Period, Scope, Kind, Message, RaisedAt FROM CostAlerts WHERE RaisedAt >= @since ORDER BY RaisedAt DESC LIMIT 200";
|
||||
cmd.Parameters.AddWithValue("@since", _clock.GetUtcNow().UtcDateTime.AddDays(-Math.Max(1, days)).ToString("O"));
|
||||
var result = new List<RaisedCostAlert>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await reader.ReadAsync(ct))
|
||||
result.Add(new RaisedCostAlert(reader.GetString(0), reader.GetString(1),
|
||||
Enum.TryParse<CostAlertKind>(reader.GetString(2), out var k) ? k : CostAlertKind.Deviation,
|
||||
reader.GetString(3), DateTime.Parse(reader.GetString(4), null, System.Globalization.DateTimeStyles.RoundtripKind)));
|
||||
return result;
|
||||
}
|
||||
|
||||
private Task<bool> TryRecordAsync(string period, CostAlert alert, CancellationToken ct)
|
||||
=> storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT OR IGNORE INTO CostAlerts (Period, Scope, Kind, Message, RaisedAt)
|
||||
VALUES (@period, @scope, @kind, @message, @at)
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@period", period);
|
||||
cmd.Parameters.AddWithValue("@scope", alert.Scope);
|
||||
cmd.Parameters.AddWithValue("@kind", alert.Kind.ToString());
|
||||
cmd.Parameters.AddWithValue("@message", alert.Message);
|
||||
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
|
||||
return await cmd.ExecuteNonQueryAsync(ct) == 1;
|
||||
}, ct);
|
||||
|
||||
public void Start() => _loop ??= RunAsync(_cts.Token);
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try { await Task.Delay(TimeSpan.FromMinutes(1), ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await CheckAsync(ct); }
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(ex, "Cost watch failed — next attempt in {Minutes} min", Interval.TotalMinutes);
|
||||
}
|
||||
|
||||
try { await Task.Delay(Interval, ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
if (_loop is not null)
|
||||
{
|
||||
try { await _loop; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -193,6 +193,41 @@ public sealed class SqliteUsageRepository : IUsageRepository
|
||||
OnSubscription = reader.GetInt32(13) != 0
|
||||
};
|
||||
|
||||
public async Task<IReadOnlyList<AgentDayUsage>> GetDailyByAgentAsync(DateOnly from, DateOnly to, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await _storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
// Amounts are stored as text — summed here, like GetDailyAsync does.
|
||||
cmd.CommandText = """
|
||||
SELECT UsageDate, AgentId, CostUsd, PromptTokens + CompletionTokens, OnSubscription
|
||||
FROM RunUsage
|
||||
WHERE UsageDate >= @from AND UsageDate <= @to
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@from", FormatDate(from));
|
||||
cmd.Parameters.AddWithValue("@to", FormatDate(to));
|
||||
|
||||
var sums = new Dictionary<(string Date, string Agent), (decimal Cost, long Tokens, long Sub, int Runs)>();
|
||||
await using (var reader = await cmd.ExecuteReaderAsync(ct))
|
||||
{
|
||||
while (await reader.ReadAsync(ct))
|
||||
{
|
||||
var key = (reader.GetString(0), reader.GetString(1));
|
||||
var cost = decimal.TryParse(reader.GetValue(2)?.ToString(), System.Globalization.NumberStyles.Any,
|
||||
System.Globalization.CultureInfo.InvariantCulture, out var c) ? c : 0m;
|
||||
var tokens = reader.GetInt64(3);
|
||||
var sub = reader.GetInt64(4) != 0;
|
||||
var current = sums.GetValueOrDefault(key);
|
||||
sums[key] = (current.Cost + cost, current.Tokens + tokens, current.Sub + (sub ? tokens : 0), current.Runs + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return sums.Select(e => new AgentDayUsage(
|
||||
DateOnly.ParseExact(e.Key.Date, "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture),
|
||||
e.Key.Agent, e.Value.Cost, e.Value.Tokens, e.Value.Sub, e.Value.Runs))
|
||||
.OrderBy(u => u.Date).ThenBy(u => u.AgentId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string FormatDate(DateOnly date) => date.ToString("yyyy-MM-dd");
|
||||
|
||||
private static string FormatDecimal(decimal value)
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
|
||||
namespace ClawdDotNet.Core.Accounting;
|
||||
|
||||
/// <summary>#699: When a change in cost is worth a look. Nothing here switches anything off.</summary>
|
||||
public sealed class CostWatchSettings
|
||||
{
|
||||
[JsonPropertyName("enabled")]
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>A day this many times the unit's average of the weeks before is a deviation.</summary>
|
||||
[JsonPropertyName("deviationFactor")]
|
||||
public decimal DeviationFactor { get; set; } = 2m;
|
||||
|
||||
/// <summary>Below this many dollars a day nothing deviates — 3 cents after 1 cent is noise.</summary>
|
||||
[JsonPropertyName("deviationMinimumUsd")]
|
||||
public decimal DeviationMinimumUsd { get; set; } = 1m;
|
||||
|
||||
/// <summary>Share of the sum of the agents' daily limits at which a unit is near its limit.</summary>
|
||||
[JsonPropertyName("nearLimitShare")]
|
||||
public decimal NearLimitShare { get; set; } = 0.8m;
|
||||
|
||||
/// <summary>Monthly totals of the instance that are reported when crossed — they report, they do not stop.</summary>
|
||||
[JsonPropertyName("monthlyThresholdsUsd")]
|
||||
public List<decimal> MonthlyThresholdsUsd { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>#699: A unit's cost — itself and everything below it.</summary>
|
||||
public sealed record UnitCostLine(
|
||||
string? UnitId,
|
||||
string Name,
|
||||
string Path,
|
||||
int Depth,
|
||||
int Agents,
|
||||
decimal Today,
|
||||
decimal Last7Days,
|
||||
decimal Last30Days,
|
||||
decimal ThisMonth,
|
||||
decimal PreviousMonth,
|
||||
// The sum of the agents' daily dollar limits; null when an agent below has none — then there is no maximum.
|
||||
decimal? MaxPerDay,
|
||||
// The average day of the 28 days before today; null without a week of history.
|
||||
decimal? AverageDay,
|
||||
long SubscriptionTokensToday,
|
||||
long SubscriptionTokens30Days)
|
||||
{
|
||||
public decimal? ShareOfMaxToday => MaxPerDay is > 0 ? Today / MaxPerDay : null;
|
||||
}
|
||||
|
||||
public enum CostAlertKind
|
||||
{
|
||||
/// <summary>Today far above the unit's own course.</summary>
|
||||
Deviation,
|
||||
|
||||
/// <summary>The agents of a unit together close to their limits — before they run into them one by one.</summary>
|
||||
NearLimit,
|
||||
|
||||
/// <summary>The instance crossed a monthly total.</summary>
|
||||
MonthlyThreshold
|
||||
}
|
||||
|
||||
public sealed record CostAlert(DateOnly Date, string Scope, CostAlertKind Kind, string Message);
|
||||
|
||||
/// <summary>
|
||||
/// #699: Costs per unit, added up — instead of budgets on the unit (decision of 2026-09-22).
|
||||
///
|
||||
/// <para><b>Limits stay at the agent; the unit reckons.</b> No unit limit, so a move changes no
|
||||
/// limit at all. The maximum shown is the sum of the agents' limits below.</para>
|
||||
///
|
||||
/// <para><b>The alert matters more than the limit.</b> A limit says when it is over; what is
|
||||
/// interesting is when something changes: a unit that needed 30 $ a day for weeks and needs
|
||||
/// 80 $ today has exceeded nothing and is exactly the case to look at. Recognising is code — an
|
||||
/// average and a threshold. Judging it is for the supervision (#680) or the human.</para>
|
||||
/// </summary>
|
||||
public static class UnitCosts
|
||||
{
|
||||
/// <summary>The scope name of the instance as a whole.</summary>
|
||||
public const string Instance = "instance";
|
||||
|
||||
/// <summary>Days of history behind the average.</summary>
|
||||
public const int AverageDays = 28;
|
||||
|
||||
/// <summary>Days with usage needed before an average means anything.</summary>
|
||||
public const int MinimumHistoryDays = 7;
|
||||
|
||||
public static IReadOnlyList<UnitCostLine> Build(OrgChart org, IReadOnlyList<AgentDayUsage> usage, DateOnly today)
|
||||
{
|
||||
var lines = new List<UnitCostLine>();
|
||||
var agents = org.Agents.ToList();
|
||||
|
||||
// The instance first, then the tree from the top, then the agents on the top level.
|
||||
lines.Add(Line(null, "Instance", "", 0, agents.Select(a => a.AgentId), agents, usage, today));
|
||||
void Walk(string? parent, int depth)
|
||||
{
|
||||
foreach (var unit in org.Children(parent).OrderBy(u => u.Name, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var below = Subtree(org, unit.Id).SelectMany(u => org.MembersOf(u.Id)).ToList();
|
||||
lines.Add(Line(unit.Id, unit.Name, org.PathOf(unit.Id), depth, below.Select(a => a.AgentId), below, usage, today));
|
||||
Walk(unit.Id, depth + 1);
|
||||
}
|
||||
}
|
||||
if (org.HasUnits)
|
||||
{
|
||||
Walk(null, 1);
|
||||
var topLevel = org.MembersOf(null);
|
||||
if (topLevel.Count > 0)
|
||||
lines.Add(Line("", "Top level", "", 1, topLevel.Select(a => a.AgentId), topLevel, usage, today));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
private static IEnumerable<UnitConfig> Subtree(OrgChart org, string unitId)
|
||||
{
|
||||
if (org.Unit(unitId) is not { } unit)
|
||||
yield break;
|
||||
yield return unit;
|
||||
foreach (var child in org.Children(unitId))
|
||||
foreach (var u in Subtree(org, child.Id))
|
||||
yield return u;
|
||||
}
|
||||
|
||||
private static UnitCostLine Line(
|
||||
string? unitId, string name, string path, int depth, IEnumerable<string> agentIds,
|
||||
IReadOnlyList<AgentConfig> agents, IReadOnlyList<AgentDayUsage> usage, DateOnly today)
|
||||
{
|
||||
var ids = agentIds.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
// The instance line counts everything — also agents removed since.
|
||||
var rows = unitId is null ? usage : usage.Where(u => ids.Contains(u.AgentId)).ToList();
|
||||
|
||||
decimal Sum(DateOnly from, DateOnly to) => rows.Where(r => r.Date >= from && r.Date <= to).Sum(r => r.CostUsd);
|
||||
var monthStart = new DateOnly(today.Year, today.Month, 1);
|
||||
var previousStart = monthStart.AddMonths(-1);
|
||||
|
||||
var history = rows.Where(r => r.Date >= today.AddDays(-AverageDays) && r.Date < today).ToList();
|
||||
var historyDays = history.Select(r => r.Date).Distinct().Count();
|
||||
decimal? average = historyDays >= MinimumHistoryDays ? history.Sum(r => r.CostUsd) / AverageDays : null;
|
||||
|
||||
decimal? max = agents.Count > 0 && agents.All(a => a.Budget.DailyCostUsd > 0) ? agents.Sum(a => a.Budget.DailyCostUsd) : null;
|
||||
|
||||
return new UnitCostLine(
|
||||
unitId, name, path, depth, agents.Count,
|
||||
Sum(today, today), Sum(today.AddDays(-6), today), Sum(today.AddDays(-29), today),
|
||||
Sum(monthStart, today), Sum(previousStart, monthStart.AddDays(-1)),
|
||||
max, average,
|
||||
rows.Where(r => r.Date == today).Sum(r => r.SubscriptionTokens),
|
||||
rows.Where(r => r.Date >= today.AddDays(-29)).Sum(r => r.SubscriptionTokens));
|
||||
}
|
||||
|
||||
/// <summary>What is worth a look today — one alert per scope and kind.</summary>
|
||||
public static IReadOnlyList<CostAlert> Alerts(IReadOnlyList<UnitCostLine> lines, CostWatchSettings settings, DateOnly today)
|
||||
{
|
||||
var alerts = new List<CostAlert>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
var scope = line.UnitId is null ? Instance : line.UnitId.Length == 0 ? "top-level" : line.UnitId;
|
||||
var label = line.UnitId is null ? "The instance" : $"'{(line.Path.Length > 0 ? line.Path : line.Name)}'";
|
||||
|
||||
if (line.AverageDay is { } average && line.Today >= settings.DeviationMinimumUsd
|
||||
&& line.Today >= average * settings.DeviationFactor)
|
||||
alerts.Add(new CostAlert(today, scope, CostAlertKind.Deviation,
|
||||
$"{label} has used {line.Today:0.00} $ today; its average day of the last {AverageDays} days is {average:0.00} $. " +
|
||||
"Nothing is exceeded — but something changed: a loop, a task off course, a more expensive model?"));
|
||||
|
||||
if (line.MaxPerDay is > 0 && line.ShareOfMaxToday >= settings.NearLimitShare)
|
||||
alerts.Add(new CostAlert(today, scope, CostAlertKind.NearLimit,
|
||||
$"{label} has used {line.Today:0.00} $ of the {line.MaxPerDay:0.00} $ its agents may spend in a day together " +
|
||||
$"({line.ShareOfMaxToday:P0}). Its agents are about to run into their limits."));
|
||||
}
|
||||
|
||||
if (lines.FirstOrDefault(l => l.UnitId is null) is { } instance)
|
||||
{
|
||||
var crossed = settings.MonthlyThresholdsUsd.Where(t => t > 0 && instance.ThisMonth >= t).DefaultIfEmpty().Max();
|
||||
if (crossed > 0)
|
||||
alerts.Add(new CostAlert(today, $"{Instance}:{crossed:0.##}", CostAlertKind.MonthlyThreshold,
|
||||
$"The instance has used {instance.ThisMonth:0.00} $ this month and crossed {crossed:0.00} $ " +
|
||||
$"(last month in all: {instance.PreviousMonth:0.00} $)."));
|
||||
}
|
||||
return alerts;
|
||||
}
|
||||
}
|
||||
@@ -80,6 +80,16 @@ public interface IUsageRepository
|
||||
Task<IReadOnlyList<RunUsage>> GetRecentForAgentAsync(string agentId, int limit, CancellationToken ct)
|
||||
=> Task.FromResult<IReadOnlyList<RunUsage>>([]);
|
||||
|
||||
/// <summary>
|
||||
/// #699: Each agent's usage per day in a range — what the units are added up from. Runs on a
|
||||
/// subscription cost 0 $ but use quota; their tokens come apart.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<AgentDayUsage>> GetDailyByAgentAsync(DateOnly from, DateOnly to, CancellationToken ct)
|
||||
=> Task.FromResult<IReadOnlyList<AgentDayUsage>>([]);
|
||||
|
||||
/// <summary>Entfernt Einträge älter als das angegebene Datum. Gibt die Anzahl zurück.</summary>
|
||||
Task<int> PurgeBeforeAsync(DateOnly date, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>#699: One agent's usage on one day.</summary>
|
||||
public sealed record AgentDayUsage(DateOnly Date, string AgentId, decimal CostUsd, long Tokens, long SubscriptionTokens, int Runs);
|
||||
|
||||
@@ -159,6 +159,10 @@ public sealed class InstanceConfig
|
||||
[JsonPropertyName("fileHistory")]
|
||||
public Files.FileHistorySettings FileHistory { get; set; } = new();
|
||||
|
||||
/// <summary>#699: When a change in the units' costs is reported.</summary>
|
||||
[JsonPropertyName("costWatch")]
|
||||
public Accounting.CostWatchSettings CostWatch { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("maxDelegationDepth")]
|
||||
public int MaxDelegationDepth { get; set; } = 3;
|
||||
|
||||
|
||||
@@ -284,6 +284,17 @@ public sealed class SqliteStorage
|
||||
|
||||
-- Staging (A2): eingefrorene, freigabepflichtige Tool-Aufrufe. Ausgeführt wird
|
||||
-- genau der gespeicherte Argument-JSON (Plan-Freeze).
|
||||
-- #699: cost alerts, once per period (day, or month for a threshold), scope and kind.
|
||||
CREATE TABLE IF NOT EXISTS CostAlerts (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
Period TEXT NOT NULL,
|
||||
Scope TEXT NOT NULL,
|
||||
Kind TEXT NOT NULL,
|
||||
Message TEXT NOT NULL,
|
||||
RaisedAt TEXT NOT NULL,
|
||||
UNIQUE (Period, Scope, Kind)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS StagedCalls (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
RunId TEXT NOT NULL,
|
||||
|
||||
Reference in New Issue
Block a user