Compare commits
2
Commits
7eb211fd06
...
c608af1abb
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c608af1abb | ||
|
|
da34b215f7 |
@@ -1311,6 +1311,17 @@ wie möglich" ist die Voreinstellung, nicht die einzige Möglichkeit.
|
||||
2026-09-19) widerspricht dieser Linie. Sie gehört auf „unbegrenzt" umgestellt; die
|
||||
Möglichkeit, je Agent kürzer oder gar nicht aufzuzeichnen, bleibt unverändert.
|
||||
|
||||
> **Umgesetzt (#699).** `UnitCosts` rechnet je Einheit samt allem darunter: heute, 7 und
|
||||
> 30 Tage, dieser Monat, Vormonat, Durchschnittstag der letzten 28 Tage (erst ab 7 Tagen
|
||||
> Verlauf) und das Maximum als Summe der Agentengrenzen — fehlt einem Agenten die Grenze,
|
||||
> gibt es kein Maximum. Abo-Läufe stehen als Kontingent (Tokens), nicht als „kostenlos".
|
||||
> `CostWatch` prüft alle 15 Minuten und meldet je Einheit und Tag einmal: Abweichung
|
||||
> (Faktor 2 über dem Durchschnitt, ab 1 $), Nähe zur Grenzsumme (80 %), dazu je Monat
|
||||
> überschrittene Instanzschwellen (`costWatch.monthlyThresholdsUsd`, Vorgabe keine). Die
|
||||
> Meldung geht als Aufgabe an den Menschen und steht in der Verbrauchsansicht; abgeschaltet
|
||||
> wird nichts. Die Zeitreihe gruppiert zusätzlich nach Einheit. **Offen:** verdichtete
|
||||
> Tagessummen für lange Zeiträume (#701) — die Auswertung liest heute die Laufzeilen.
|
||||
|
||||
---
|
||||
|
||||
## 9 — Was eine Organisation braucht und ein Agentenhaufen nicht
|
||||
|
||||
@@ -129,6 +129,9 @@ public sealed class AppHost : IAsyncDisposable
|
||||
/// <summary>#660: Usage analysis, run details and prompt history for the user interfaces.</summary>
|
||||
public RunInsights? Insights { get; private init; }
|
||||
|
||||
/// <summary>#699: Costs per unit and the alerts on them.</summary>
|
||||
public Core.Accounting.CostWatch? CostWatch { get; private set; }
|
||||
|
||||
/// <summary>Modellpreise von OpenRouter - fuer Budget und Kostenprognose (#595). Laedt im Hintergrund.</summary>
|
||||
public ModelPricingCatalog? Pricing { get; private init; }
|
||||
|
||||
@@ -704,6 +707,14 @@ public sealed class AppHost : IAsyncDisposable
|
||||
historyKeeper.Start();
|
||||
host._shutdown.Add(historyKeeper.DisposeAsync);
|
||||
|
||||
// #699: costs per unit, and what changed in them.
|
||||
var costWatch = new Core.Accounting.CostWatch(
|
||||
storage, usage, () => Core.Organisation.OrgChart.From(instance), () => instance.CostWatch, board,
|
||||
loggerFactory.CreateLogger("ClawdDotNet.Core.Accounting.CostWatch"));
|
||||
host.CostWatch = costWatch;
|
||||
costWatch.Start();
|
||||
host._shutdown.Add(costWatch.DisposeAsync);
|
||||
|
||||
// #593: Angelegt wurde der Dienst schon immer, gestartet nie — weder kopflos noch
|
||||
// mit Fenster. Die Kopfzeile der Weboberflaeche stand deshalb dauerhaft auf
|
||||
// "OpenRouter: —" und /api/status auf "Prüfe…".
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Core.Accounting;
|
||||
|
||||
/// <summary>#699: An alert as it was raised — for the usage view.</summary>
|
||||
public sealed record RaisedCostAlert(string Period, string Scope, CostAlertKind Kind, string Message, DateTime RaisedAt);
|
||||
|
||||
/// <summary>
|
||||
/// #699: Looks at the costs of the units every few minutes and raises what changed — once per
|
||||
/// day and unit (a monthly threshold once per month). An alert goes to the human as a task and
|
||||
/// stays in the usage view; it switches nothing off.
|
||||
/// </summary>
|
||||
public sealed class CostWatch(
|
||||
SqliteStorage storage,
|
||||
IUsageRepository usage,
|
||||
Func<OrgChart> organisation,
|
||||
Func<CostWatchSettings> settings,
|
||||
TaskboardService? board,
|
||||
ILogger logger,
|
||||
TimeProvider? clock = null) : IAsyncDisposable
|
||||
{
|
||||
public static readonly TimeSpan Interval = TimeSpan.FromMinutes(15);
|
||||
|
||||
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task? _loop;
|
||||
|
||||
/// <summary>The day the budget counts — the same local day as the BudgetGuard's.</summary>
|
||||
private DateOnly Today => DateOnly.FromDateTime(_clock.GetLocalNow().DateTime);
|
||||
|
||||
public async Task<IReadOnlyList<UnitCostLine>> ReportAsync(CancellationToken ct)
|
||||
{
|
||||
var today = Today;
|
||||
var rows = await usage.GetDailyByAgentAsync(new DateOnly(today.Year, today.Month, 1).AddMonths(-1).AddDays(-UnitCosts.AverageDays), today, ct);
|
||||
return UnitCosts.Build(organisation(), rows, today);
|
||||
}
|
||||
|
||||
/// <summary>Raises what is new today. Returns the alerts raised in this pass.</summary>
|
||||
public async Task<IReadOnlyList<CostAlert>> CheckAsync(CancellationToken ct)
|
||||
{
|
||||
if (!settings().Enabled)
|
||||
return [];
|
||||
|
||||
var raised = new List<CostAlert>();
|
||||
foreach (var alert in UnitCosts.Alerts(await ReportAsync(ct), settings(), Today))
|
||||
{
|
||||
var period = alert.Kind == CostAlertKind.MonthlyThreshold ? alert.Date.ToString("yyyy-MM") : alert.Date.ToString("yyyy-MM-dd");
|
||||
if (!await TryRecordAsync(period, alert, ct))
|
||||
continue;
|
||||
raised.Add(alert);
|
||||
logger.LogWarning("Cost alert ({Kind}, {Scope}): {Message}", alert.Kind, alert.Scope, alert.Message);
|
||||
|
||||
if (board is not null)
|
||||
await board.CreateAsync(new TaskItem
|
||||
{
|
||||
Title = alert.Kind switch
|
||||
{
|
||||
CostAlertKind.Deviation => $"Cost alert: unusual day for {alert.Scope}",
|
||||
CostAlertKind.NearLimit => $"Cost alert: {alert.Scope} near its limits",
|
||||
_ => "Cost alert: monthly total crossed"
|
||||
},
|
||||
Body = alert.Message + "\n\nLook at the usage view (costs by unit). Nothing was switched off.",
|
||||
Assignee = TaskAssignee.Human,
|
||||
Priority = alert.Kind == CostAlertKind.NearLimit ? 4 : 3
|
||||
}, ct);
|
||||
}
|
||||
return raised;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<RaisedCostAlert>> RecentAsync(int days, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT Period, Scope, Kind, Message, RaisedAt FROM CostAlerts WHERE RaisedAt >= @since ORDER BY RaisedAt DESC LIMIT 200";
|
||||
cmd.Parameters.AddWithValue("@since", _clock.GetUtcNow().UtcDateTime.AddDays(-Math.Max(1, days)).ToString("O"));
|
||||
var result = new List<RaisedCostAlert>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await reader.ReadAsync(ct))
|
||||
result.Add(new RaisedCostAlert(reader.GetString(0), reader.GetString(1),
|
||||
Enum.TryParse<CostAlertKind>(reader.GetString(2), out var k) ? k : CostAlertKind.Deviation,
|
||||
reader.GetString(3), DateTime.Parse(reader.GetString(4), null, System.Globalization.DateTimeStyles.RoundtripKind)));
|
||||
return result;
|
||||
}
|
||||
|
||||
private Task<bool> TryRecordAsync(string period, CostAlert alert, CancellationToken ct)
|
||||
=> storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT OR IGNORE INTO CostAlerts (Period, Scope, Kind, Message, RaisedAt)
|
||||
VALUES (@period, @scope, @kind, @message, @at)
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@period", period);
|
||||
cmd.Parameters.AddWithValue("@scope", alert.Scope);
|
||||
cmd.Parameters.AddWithValue("@kind", alert.Kind.ToString());
|
||||
cmd.Parameters.AddWithValue("@message", alert.Message);
|
||||
cmd.Parameters.AddWithValue("@at", _clock.GetUtcNow().UtcDateTime.ToString("O"));
|
||||
return await cmd.ExecuteNonQueryAsync(ct) == 1;
|
||||
}, ct);
|
||||
|
||||
public void Start() => _loop ??= RunAsync(_cts.Token);
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
try { await Task.Delay(TimeSpan.FromMinutes(1), ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await CheckAsync(ct); }
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(ex, "Cost watch failed — next attempt in {Minutes} min", Interval.TotalMinutes);
|
||||
}
|
||||
|
||||
try { await Task.Delay(Interval, ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
if (_loop is not null)
|
||||
{
|
||||
try { await _loop; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -193,6 +193,41 @@ public sealed class SqliteUsageRepository : IUsageRepository
|
||||
OnSubscription = reader.GetInt32(13) != 0
|
||||
};
|
||||
|
||||
public async Task<IReadOnlyList<AgentDayUsage>> GetDailyByAgentAsync(DateOnly from, DateOnly to, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await _storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
// Amounts are stored as text — summed here, like GetDailyAsync does.
|
||||
cmd.CommandText = """
|
||||
SELECT UsageDate, AgentId, CostUsd, PromptTokens + CompletionTokens, OnSubscription
|
||||
FROM RunUsage
|
||||
WHERE UsageDate >= @from AND UsageDate <= @to
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@from", FormatDate(from));
|
||||
cmd.Parameters.AddWithValue("@to", FormatDate(to));
|
||||
|
||||
var sums = new Dictionary<(string Date, string Agent), (decimal Cost, long Tokens, long Sub, int Runs)>();
|
||||
await using (var reader = await cmd.ExecuteReaderAsync(ct))
|
||||
{
|
||||
while (await reader.ReadAsync(ct))
|
||||
{
|
||||
var key = (reader.GetString(0), reader.GetString(1));
|
||||
var cost = decimal.TryParse(reader.GetValue(2)?.ToString(), System.Globalization.NumberStyles.Any,
|
||||
System.Globalization.CultureInfo.InvariantCulture, out var c) ? c : 0m;
|
||||
var tokens = reader.GetInt64(3);
|
||||
var sub = reader.GetInt64(4) != 0;
|
||||
var current = sums.GetValueOrDefault(key);
|
||||
sums[key] = (current.Cost + cost, current.Tokens + tokens, current.Sub + (sub ? tokens : 0), current.Runs + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return sums.Select(e => new AgentDayUsage(
|
||||
DateOnly.ParseExact(e.Key.Date, "yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture),
|
||||
e.Key.Agent, e.Value.Cost, e.Value.Tokens, e.Value.Sub, e.Value.Runs))
|
||||
.OrderBy(u => u.Date).ThenBy(u => u.AgentId, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string FormatDate(DateOnly date) => date.ToString("yyyy-MM-dd");
|
||||
|
||||
private static string FormatDecimal(decimal value)
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
using System.Text.Json.Serialization;
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
|
||||
namespace ClawdDotNet.Core.Accounting;
|
||||
|
||||
/// <summary>#699: When a change in cost is worth a look. Nothing here switches anything off.</summary>
|
||||
public sealed class CostWatchSettings
|
||||
{
|
||||
[JsonPropertyName("enabled")]
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>A day this many times the unit's average of the weeks before is a deviation.</summary>
|
||||
[JsonPropertyName("deviationFactor")]
|
||||
public decimal DeviationFactor { get; set; } = 2m;
|
||||
|
||||
/// <summary>Below this many dollars a day nothing deviates — 3 cents after 1 cent is noise.</summary>
|
||||
[JsonPropertyName("deviationMinimumUsd")]
|
||||
public decimal DeviationMinimumUsd { get; set; } = 1m;
|
||||
|
||||
/// <summary>Share of the sum of the agents' daily limits at which a unit is near its limit.</summary>
|
||||
[JsonPropertyName("nearLimitShare")]
|
||||
public decimal NearLimitShare { get; set; } = 0.8m;
|
||||
|
||||
/// <summary>Monthly totals of the instance that are reported when crossed — they report, they do not stop.</summary>
|
||||
[JsonPropertyName("monthlyThresholdsUsd")]
|
||||
public List<decimal> MonthlyThresholdsUsd { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>#699: A unit's cost — itself and everything below it.</summary>
|
||||
public sealed record UnitCostLine(
|
||||
string? UnitId,
|
||||
string Name,
|
||||
string Path,
|
||||
int Depth,
|
||||
int Agents,
|
||||
decimal Today,
|
||||
decimal Last7Days,
|
||||
decimal Last30Days,
|
||||
decimal ThisMonth,
|
||||
decimal PreviousMonth,
|
||||
// The sum of the agents' daily dollar limits; null when an agent below has none — then there is no maximum.
|
||||
decimal? MaxPerDay,
|
||||
// The average day of the 28 days before today; null without a week of history.
|
||||
decimal? AverageDay,
|
||||
long SubscriptionTokensToday,
|
||||
long SubscriptionTokens30Days)
|
||||
{
|
||||
public decimal? ShareOfMaxToday => MaxPerDay is > 0 ? Today / MaxPerDay : null;
|
||||
}
|
||||
|
||||
public enum CostAlertKind
|
||||
{
|
||||
/// <summary>Today far above the unit's own course.</summary>
|
||||
Deviation,
|
||||
|
||||
/// <summary>The agents of a unit together close to their limits — before they run into them one by one.</summary>
|
||||
NearLimit,
|
||||
|
||||
/// <summary>The instance crossed a monthly total.</summary>
|
||||
MonthlyThreshold
|
||||
}
|
||||
|
||||
public sealed record CostAlert(DateOnly Date, string Scope, CostAlertKind Kind, string Message);
|
||||
|
||||
/// <summary>
|
||||
/// #699: Costs per unit, added up — instead of budgets on the unit (decision of 2026-09-22).
|
||||
///
|
||||
/// <para><b>Limits stay at the agent; the unit reckons.</b> No unit limit, so a move changes no
|
||||
/// limit at all. The maximum shown is the sum of the agents' limits below.</para>
|
||||
///
|
||||
/// <para><b>The alert matters more than the limit.</b> A limit says when it is over; what is
|
||||
/// interesting is when something changes: a unit that needed 30 $ a day for weeks and needs
|
||||
/// 80 $ today has exceeded nothing and is exactly the case to look at. Recognising is code — an
|
||||
/// average and a threshold. Judging it is for the supervision (#680) or the human.</para>
|
||||
/// </summary>
|
||||
public static class UnitCosts
|
||||
{
|
||||
/// <summary>The scope name of the instance as a whole.</summary>
|
||||
public const string Instance = "instance";
|
||||
|
||||
/// <summary>Days of history behind the average.</summary>
|
||||
public const int AverageDays = 28;
|
||||
|
||||
/// <summary>Days with usage needed before an average means anything.</summary>
|
||||
public const int MinimumHistoryDays = 7;
|
||||
|
||||
public static IReadOnlyList<UnitCostLine> Build(OrgChart org, IReadOnlyList<AgentDayUsage> usage, DateOnly today)
|
||||
{
|
||||
var lines = new List<UnitCostLine>();
|
||||
var agents = org.Agents.ToList();
|
||||
|
||||
// The instance first, then the tree from the top, then the agents on the top level.
|
||||
lines.Add(Line(null, "Instance", "", 0, agents.Select(a => a.AgentId), agents, usage, today));
|
||||
void Walk(string? parent, int depth)
|
||||
{
|
||||
foreach (var unit in org.Children(parent).OrderBy(u => u.Name, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var below = Subtree(org, unit.Id).SelectMany(u => org.MembersOf(u.Id)).ToList();
|
||||
lines.Add(Line(unit.Id, unit.Name, org.PathOf(unit.Id), depth, below.Select(a => a.AgentId), below, usage, today));
|
||||
Walk(unit.Id, depth + 1);
|
||||
}
|
||||
}
|
||||
if (org.HasUnits)
|
||||
{
|
||||
Walk(null, 1);
|
||||
var topLevel = org.MembersOf(null);
|
||||
if (topLevel.Count > 0)
|
||||
lines.Add(Line("", "Top level", "", 1, topLevel.Select(a => a.AgentId), topLevel, usage, today));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
private static IEnumerable<UnitConfig> Subtree(OrgChart org, string unitId)
|
||||
{
|
||||
if (org.Unit(unitId) is not { } unit)
|
||||
yield break;
|
||||
yield return unit;
|
||||
foreach (var child in org.Children(unitId))
|
||||
foreach (var u in Subtree(org, child.Id))
|
||||
yield return u;
|
||||
}
|
||||
|
||||
private static UnitCostLine Line(
|
||||
string? unitId, string name, string path, int depth, IEnumerable<string> agentIds,
|
||||
IReadOnlyList<AgentConfig> agents, IReadOnlyList<AgentDayUsage> usage, DateOnly today)
|
||||
{
|
||||
var ids = agentIds.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
// The instance line counts everything — also agents removed since.
|
||||
var rows = unitId is null ? usage : usage.Where(u => ids.Contains(u.AgentId)).ToList();
|
||||
|
||||
decimal Sum(DateOnly from, DateOnly to) => rows.Where(r => r.Date >= from && r.Date <= to).Sum(r => r.CostUsd);
|
||||
var monthStart = new DateOnly(today.Year, today.Month, 1);
|
||||
var previousStart = monthStart.AddMonths(-1);
|
||||
|
||||
var history = rows.Where(r => r.Date >= today.AddDays(-AverageDays) && r.Date < today).ToList();
|
||||
var historyDays = history.Select(r => r.Date).Distinct().Count();
|
||||
decimal? average = historyDays >= MinimumHistoryDays ? history.Sum(r => r.CostUsd) / AverageDays : null;
|
||||
|
||||
decimal? max = agents.Count > 0 && agents.All(a => a.Budget.DailyCostUsd > 0) ? agents.Sum(a => a.Budget.DailyCostUsd) : null;
|
||||
|
||||
return new UnitCostLine(
|
||||
unitId, name, path, depth, agents.Count,
|
||||
Sum(today, today), Sum(today.AddDays(-6), today), Sum(today.AddDays(-29), today),
|
||||
Sum(monthStart, today), Sum(previousStart, monthStart.AddDays(-1)),
|
||||
max, average,
|
||||
rows.Where(r => r.Date == today).Sum(r => r.SubscriptionTokens),
|
||||
rows.Where(r => r.Date >= today.AddDays(-29)).Sum(r => r.SubscriptionTokens));
|
||||
}
|
||||
|
||||
/// <summary>What is worth a look today — one alert per scope and kind.</summary>
|
||||
public static IReadOnlyList<CostAlert> Alerts(IReadOnlyList<UnitCostLine> lines, CostWatchSettings settings, DateOnly today)
|
||||
{
|
||||
var alerts = new List<CostAlert>();
|
||||
foreach (var line in lines)
|
||||
{
|
||||
var scope = line.UnitId is null ? Instance : line.UnitId.Length == 0 ? "top-level" : line.UnitId;
|
||||
var label = line.UnitId is null ? "The instance" : $"'{(line.Path.Length > 0 ? line.Path : line.Name)}'";
|
||||
|
||||
if (line.AverageDay is { } average && line.Today >= settings.DeviationMinimumUsd
|
||||
&& line.Today >= average * settings.DeviationFactor)
|
||||
alerts.Add(new CostAlert(today, scope, CostAlertKind.Deviation,
|
||||
$"{label} has used {line.Today:0.00} $ today; its average day of the last {AverageDays} days is {average:0.00} $. " +
|
||||
"Nothing is exceeded — but something changed: a loop, a task off course, a more expensive model?"));
|
||||
|
||||
if (line.MaxPerDay is > 0 && line.ShareOfMaxToday >= settings.NearLimitShare)
|
||||
alerts.Add(new CostAlert(today, scope, CostAlertKind.NearLimit,
|
||||
$"{label} has used {line.Today:0.00} $ of the {line.MaxPerDay:0.00} $ its agents may spend in a day together " +
|
||||
$"({line.ShareOfMaxToday:P0}). Its agents are about to run into their limits."));
|
||||
}
|
||||
|
||||
if (lines.FirstOrDefault(l => l.UnitId is null) is { } instance)
|
||||
{
|
||||
var crossed = settings.MonthlyThresholdsUsd.Where(t => t > 0 && instance.ThisMonth >= t).DefaultIfEmpty().Max();
|
||||
if (crossed > 0)
|
||||
alerts.Add(new CostAlert(today, $"{Instance}:{crossed:0.##}", CostAlertKind.MonthlyThreshold,
|
||||
$"The instance has used {instance.ThisMonth:0.00} $ this month and crossed {crossed:0.00} $ " +
|
||||
$"(last month in all: {instance.PreviousMonth:0.00} $)."));
|
||||
}
|
||||
return alerts;
|
||||
}
|
||||
}
|
||||
@@ -80,6 +80,16 @@ public interface IUsageRepository
|
||||
Task<IReadOnlyList<RunUsage>> GetRecentForAgentAsync(string agentId, int limit, CancellationToken ct)
|
||||
=> Task.FromResult<IReadOnlyList<RunUsage>>([]);
|
||||
|
||||
/// <summary>
|
||||
/// #699: Each agent's usage per day in a range — what the units are added up from. Runs on a
|
||||
/// subscription cost 0 $ but use quota; their tokens come apart.
|
||||
/// </summary>
|
||||
Task<IReadOnlyList<AgentDayUsage>> GetDailyByAgentAsync(DateOnly from, DateOnly to, CancellationToken ct)
|
||||
=> Task.FromResult<IReadOnlyList<AgentDayUsage>>([]);
|
||||
|
||||
/// <summary>Entfernt Einträge älter als das angegebene Datum. Gibt die Anzahl zurück.</summary>
|
||||
Task<int> PurgeBeforeAsync(DateOnly date, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>#699: One agent's usage on one day.</summary>
|
||||
public sealed record AgentDayUsage(DateOnly Date, string AgentId, decimal CostUsd, long Tokens, long SubscriptionTokens, int Runs);
|
||||
|
||||
@@ -159,6 +159,10 @@ public sealed class InstanceConfig
|
||||
[JsonPropertyName("fileHistory")]
|
||||
public Files.FileHistorySettings FileHistory { get; set; } = new();
|
||||
|
||||
/// <summary>#699: When a change in the units' costs is reported.</summary>
|
||||
[JsonPropertyName("costWatch")]
|
||||
public Accounting.CostWatchSettings CostWatch { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("maxDelegationDepth")]
|
||||
public int MaxDelegationDepth { get; set; } = 3;
|
||||
|
||||
|
||||
@@ -284,6 +284,17 @@ public sealed class SqliteStorage
|
||||
|
||||
-- Staging (A2): eingefrorene, freigabepflichtige Tool-Aufrufe. Ausgeführt wird
|
||||
-- genau der gespeicherte Argument-JSON (Plan-Freeze).
|
||||
-- #699: cost alerts, once per period (day, or month for a threshold), scope and kind.
|
||||
CREATE TABLE IF NOT EXISTS CostAlerts (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
Period TEXT NOT NULL,
|
||||
Scope TEXT NOT NULL,
|
||||
Kind TEXT NOT NULL,
|
||||
Message TEXT NOT NULL,
|
||||
RaisedAt TEXT NOT NULL,
|
||||
UNIQUE (Period, Scope, Kind)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS StagedCalls (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
RunId TEXT NOT NULL,
|
||||
|
||||
@@ -671,6 +671,22 @@
|
||||
"usage.cost": "Kosten",
|
||||
"usage.unknownPrice": "(Preis unbekannt)",
|
||||
"usage.notAvailable": "Verbrauchsdaten nicht verfügbar",
|
||||
"usage.units.title": "Kosten je Einheit",
|
||||
"usage.units.hint": "Jede Einheit mit allem, was darunter liegt. Das Maximum ist die Summe der Tagesgrenzen ihrer Agenten — Grenzen bleiben am Agenten. Wichtig ist die Veränderung: Ein Tag weit über dem eigenen Durchschnitt der Einheit wird markiert, auch wenn nichts überschritten ist.",
|
||||
"usage.units.unit": "Einheit",
|
||||
"usage.units.today": "Heute",
|
||||
"usage.units.average": "Durchschnittstag",
|
||||
"usage.units.max": "Max. je Tag",
|
||||
"usage.units.last7": "7 Tage",
|
||||
"usage.units.thisMonth": "Dieser Monat",
|
||||
"usage.units.previousMonth": "Vormonat",
|
||||
"usage.units.quota": "Kontingent (30 T.)",
|
||||
"usage.units.instance": "Instanz",
|
||||
"usage.units.topLevel": "Oberste Ebene",
|
||||
"usage.units.alerts": "Meldungen der letzten 7 Tage",
|
||||
"usage.units.kind.Deviation": "Ungewöhnlicher Tag",
|
||||
"usage.units.kind.NearLimit": "Nahe an den Grenzen",
|
||||
"usage.units.kind.MonthlyThreshold": "Monatssumme überschritten",
|
||||
"usage.forecastTitle": "Kostenprognose",
|
||||
"usage.forecastDay": "Prognose je Tag",
|
||||
"usage.forecastMonth": "Prognose je Monat (30 Tage)",
|
||||
@@ -783,6 +799,7 @@
|
||||
"timeline.metric.runs": "Läufe",
|
||||
"timeline.groupBy": "Nach",
|
||||
"timeline.groupBy.agent": "Agent",
|
||||
"timeline.groupBy.unit": "Einheit",
|
||||
"timeline.groupBy.model": "Modell",
|
||||
"timeline.groupBy.source": "Ereignis",
|
||||
"timeline.groupBy.budget": "Budget",
|
||||
|
||||
@@ -671,6 +671,22 @@
|
||||
"usage.cost": "Cost",
|
||||
"usage.unknownPrice": "(price unknown)",
|
||||
"usage.notAvailable": "Usage data not available",
|
||||
"usage.units.title": "Costs by unit",
|
||||
"usage.units.hint": "Each unit with everything below it. The maximum is the sum of its agents' daily limits — limits stay at the agents. What matters is the change: a day far above the unit's own average is marked even when nothing is exceeded.",
|
||||
"usage.units.unit": "Unit",
|
||||
"usage.units.today": "Today",
|
||||
"usage.units.average": "Average day",
|
||||
"usage.units.max": "Max per day",
|
||||
"usage.units.last7": "7 days",
|
||||
"usage.units.thisMonth": "This month",
|
||||
"usage.units.previousMonth": "Last month",
|
||||
"usage.units.quota": "Quota (30 d)",
|
||||
"usage.units.instance": "Instance",
|
||||
"usage.units.topLevel": "Top level",
|
||||
"usage.units.alerts": "Alerts of the last 7 days",
|
||||
"usage.units.kind.Deviation": "Unusual day",
|
||||
"usage.units.kind.NearLimit": "Near the limits",
|
||||
"usage.units.kind.MonthlyThreshold": "Monthly total crossed",
|
||||
"usage.forecastTitle": "Cost Forecast",
|
||||
"usage.forecastDay": "Forecast per day",
|
||||
"usage.forecastMonth": "Forecast per month (30 days)",
|
||||
@@ -783,6 +799,7 @@
|
||||
"timeline.metric.runs": "Runs",
|
||||
"timeline.groupBy": "By",
|
||||
"timeline.groupBy.agent": "Agent",
|
||||
"timeline.groupBy.unit": "Unit",
|
||||
"timeline.groupBy.model": "Model",
|
||||
"timeline.groupBy.source": "Event",
|
||||
"timeline.groupBy.budget": "Budget",
|
||||
|
||||
@@ -91,7 +91,7 @@ function buildControls(filters, bucket, groupBy, metric) {
|
||||
|
||||
bar.append(
|
||||
group("metric", ["cost", "tokens", "runs"], metric),
|
||||
group("groupBy", ["agent", "model", "source", "budget"], groupBy),
|
||||
group("groupBy", ["agent", "unit", "model", "source", "budget"], groupBy),
|
||||
group("bucket", ["hour", "day"], bucket));
|
||||
return bar;
|
||||
}
|
||||
|
||||
@@ -44,8 +44,9 @@ async function ladeVerbrauch() {
|
||||
);
|
||||
|
||||
try {
|
||||
const [laeufe, prognose] = await Promise.all([api("/usage?limit=100"), erstellePrognoseKarte()]);
|
||||
const [laeufe, prognose, einheiten] = await Promise.all([api("/usage?limit=100"), erstellePrognoseKarte(), erstelleEinheitenKarte()]);
|
||||
container.replaceChildren();
|
||||
if (einheiten) container.append(einheiten);
|
||||
if (prognose) container.append(prognose);
|
||||
|
||||
if (!laeufe || laeufe.length === 0) {
|
||||
@@ -164,6 +165,59 @@ async function ladeVerbrauch() {
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Costs by unit (#699) ───
|
||||
|
||||
/**
|
||||
* What each unit costs — itself and everything below it — against the sum of its agents'
|
||||
* limits and against its own course. The alert matters more than the limit: a unit far
|
||||
* above its average has exceeded nothing and is exactly the one to look at.
|
||||
*/
|
||||
async function erstelleEinheitenKarte() {
|
||||
let zeilen, warnungen;
|
||||
try {
|
||||
[zeilen, warnungen] = await Promise.all([api("/usage/units"), api("/usage/cost-alerts?days=7")]);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!Array.isArray(zeilen) || zeilen.length === 0) return null;
|
||||
|
||||
const usd = (v) => (v == null ? "—" : `${Number(v).toFixed(2)} $`);
|
||||
const tabelle = erstelleElement("table", { klasse: "daten-tabelle einheiten-kosten" });
|
||||
tabelle.append(erstelleElement("thead", {}, erstelleElement("tr", {},
|
||||
...["usage.units.unit", "usage.units.today", "usage.units.average", "usage.units.max", "usage.units.last7", "usage.units.thisMonth", "usage.units.previousMonth", "usage.units.quota"]
|
||||
.map((k) => erstelleElement("th", { klasse: k === "usage.units.unit" ? "" : "zahl", text: t(k) })))));
|
||||
|
||||
const koerper = erstelleElement("tbody");
|
||||
for (const z of zeilen) {
|
||||
const name = z.unitId == null ? t("usage.units.instance") : z.unitId === "" ? t("usage.units.topLevel") : z.name;
|
||||
const auffaellig = z.averageDay != null && z.today >= 1 && z.today >= 2 * z.averageDay;
|
||||
const nahe = z.shareOfMaxToday != null && z.shareOfMaxToday >= 0.8;
|
||||
const anteil = z.shareOfMaxToday == null ? "" : ` (${Math.round(z.shareOfMaxToday * 100)} %)`;
|
||||
koerper.append(erstelleElement("tr", { klasse: auffaellig || nahe ? "zeile-warnung" : undefined },
|
||||
erstelleElement("td", { stil: `padding-left: ${14 + z.depth * 16}px`, text: name, title: z.path || "" }),
|
||||
erstelleElement("td", { klasse: "zahl", text: usd(z.today) + anteil }),
|
||||
erstelleElement("td", { klasse: "zahl", text: usd(z.averageDay) }),
|
||||
erstelleElement("td", { klasse: "zahl", text: usd(z.maxPerDay) }),
|
||||
erstelleElement("td", { klasse: "zahl", text: usd(z.last7Days) }),
|
||||
erstelleElement("td", { klasse: "zahl", text: usd(z.thisMonth) }),
|
||||
erstelleElement("td", { klasse: "zahl", text: usd(z.previousMonth) }),
|
||||
// #712: a subscription costs 0 $ but uses quota — shown as tokens, not as "free".
|
||||
erstelleElement("td", { klasse: "zahl", text: z.subscriptionTokens30Days ? formatiereTokens(z.subscriptionTokens30Days) : "—" })));
|
||||
}
|
||||
tabelle.append(koerper);
|
||||
|
||||
const inhalt = [erstelleElement("p", { klasse: "gedaempft text-klein", text: t("usage.units.hint") }), erstelleElement("div", { klasse: "tabelle-scroll" }, tabelle)];
|
||||
if (Array.isArray(warnungen) && warnungen.length > 0) {
|
||||
inhalt.push(erstelleElement("h3", { klasse: "fall-verlauf-titel", text: t("usage.units.alerts") }));
|
||||
for (const w of warnungen) {
|
||||
inhalt.push(erstelleElement("div", { klasse: "hinweis-box warnung" },
|
||||
erstelleElement("strong", { text: `${formatiereDatumZeit(w.raisedAt)} · ${t(`usage.units.kind.${w.kind}`)}` }),
|
||||
erstelleElement("div", { text: w.message })));
|
||||
}
|
||||
}
|
||||
return erstelleKarte(t("usage.units.title"), inhalt);
|
||||
}
|
||||
|
||||
// ─── Kostenprognose (#595) ───
|
||||
|
||||
function getGrundlageText(basis, sample) {
|
||||
|
||||
@@ -395,3 +395,13 @@ public sealed record FileVersionDto(string? Text, bool Binary, bool TooLarge, bo
|
||||
|
||||
/// <summary>#719: Putting a version back — which one, and why. Without a reason it is refused.</summary>
|
||||
public sealed record FileRestoreInputDto(string? Version, string? Reason);
|
||||
|
||||
/// <summary>#699: A unit's costs, itself and everything below it. UnitId null: the instance; "": the top level.</summary>
|
||||
public sealed record UnitCostDto(
|
||||
string? UnitId, string Name, string Path, int Depth, int Agents,
|
||||
decimal Today, decimal Last7Days, decimal Last30Days, decimal ThisMonth, decimal PreviousMonth,
|
||||
decimal? MaxPerDay, decimal? AverageDay, decimal? ShareOfMaxToday,
|
||||
long SubscriptionTokensToday, long SubscriptionTokens30Days);
|
||||
|
||||
/// <summary>#699: A cost alert as it was raised.</summary>
|
||||
public sealed record CostAlertDto(string Period, string Scope, string Kind, string Message, DateTime RaisedAt);
|
||||
|
||||
@@ -58,9 +58,39 @@ public static class AnalyticsEndpoints
|
||||
groupBy is "model" or "source" or "budget" ? groupBy : "agent",
|
||||
TimeSpan.FromMinutes(Math.Clamp(tzOffset, -14 * 60, 14 * 60)),
|
||||
ct);
|
||||
|
||||
// #699: "What does marketing cost me?" — the agents' series, added up by their unit as it is now.
|
||||
if (groupBy == "unit")
|
||||
{
|
||||
var org = host.Organisation.Current();
|
||||
return Results.Ok(points
|
||||
.GroupBy(p => (p.BucketStart, Unit: org.UnitOf(p.Group) is { } u ? org.PathOf(u.Id) : "—"))
|
||||
.Select(g => new SeriesPointDto(g.Key.BucketStart, g.Key.Unit, g.Sum(p => p.Runs), g.Sum(p => p.Tokens), g.Sum(p => p.CostUsd)))
|
||||
.OrderBy(p => p.BucketStart)
|
||||
.ToList());
|
||||
}
|
||||
return Results.Ok(points.Select(p => new SeriesPointDto(p.BucketStart, p.Group, p.Runs, p.Tokens, p.CostUsd)).ToList());
|
||||
});
|
||||
|
||||
// #699: Costs per unit — itself and everything below it — with the alerts raised on them.
|
||||
api.MapGet("/usage/units", async (AppHost host, CancellationToken ct) =>
|
||||
{
|
||||
if (host.CostWatch is not { } watch)
|
||||
return ApiEndpoints.Unavailable("No usage data: agents are not active.");
|
||||
var lines = await watch.ReportAsync(ct);
|
||||
return Results.Ok(lines.Select(l => new UnitCostDto(
|
||||
l.UnitId, l.Name, l.Path, l.Depth, l.Agents, l.Today, l.Last7Days, l.Last30Days, l.ThisMonth, l.PreviousMonth,
|
||||
l.MaxPerDay, l.AverageDay, l.ShareOfMaxToday, l.SubscriptionTokensToday, l.SubscriptionTokens30Days)).ToList());
|
||||
});
|
||||
|
||||
api.MapGet("/usage/cost-alerts", async (AppHost host, CancellationToken ct, int days = 30) =>
|
||||
{
|
||||
if (host.CostWatch is not { } watch)
|
||||
return Results.Ok(Array.Empty<CostAlertDto>());
|
||||
return Results.Ok((await watch.RecentAsync(days, ct))
|
||||
.Select(a => new CostAlertDto(a.Period, a.Scope, a.Kind.ToString(), a.Message, a.RaisedAt)).ToList());
|
||||
});
|
||||
|
||||
api.MapGet("/usage/events", async (AppHost host, HttpRequest request, CancellationToken ct) =>
|
||||
{
|
||||
if (host.Insights is not { } insights)
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
using ClawdDotNet.Core.Accounting;
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using ClawdDotNet.Core.Tasks;
|
||||
using ClawdDotNet.Core.Tests.Infrastructure;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Accounting;
|
||||
|
||||
/// <summary>
|
||||
/// #699: Costs per unit are added up, not capped — and what changed is reported, once.
|
||||
/// </summary>
|
||||
public sealed class UnitCostsTests : IDisposable
|
||||
{
|
||||
private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", "unitcosts-" + Guid.NewGuid().ToString("N")[..8]);
|
||||
private static readonly DateOnly Today = new(2027, 3, 10);
|
||||
|
||||
/// <summary>marketing (ads, listing) › amazon (seller); bookkeeper on the top level.</summary>
|
||||
private static OrgChart Org(decimal adsLimit = 10) => new(
|
||||
[new UnitConfig { Id = "marketing", Name = "Marketing" }, new UnitConfig { Id = "amazon", Name = "Amazon", ParentId = "marketing" }],
|
||||
[
|
||||
new AgentConfig { AgentId = "ads", UnitId = "marketing", Budget = { DailyCostUsd = adsLimit } },
|
||||
new AgentConfig { AgentId = "listing", UnitId = "marketing", Budget = { DailyCostUsd = 5 } },
|
||||
new AgentConfig { AgentId = "seller", UnitId = "amazon", Budget = { DailyCostUsd = 5 } },
|
||||
new AgentConfig { AgentId = "bookkeeper", Budget = { DailyCostUsd = 2 } }
|
||||
]);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
SqliteStorage.ReleasePooledConnections(_directory);
|
||||
try { Directory.Delete(_directory, recursive: true); }
|
||||
catch { /* cleaning up is secondary */ }
|
||||
}
|
||||
|
||||
private static AgentDayUsage Day(int daysAgo, string agent, decimal cost, long subscriptionTokens = 0)
|
||||
=> new(Today.AddDays(-daysAgo), agent, cost, 1000, subscriptionTokens, 1);
|
||||
|
||||
/// <summary>Four weeks of the same course, then today.</summary>
|
||||
private static List<AgentDayUsage> Course(decimal perDay, decimal today, string agent = "ads")
|
||||
{
|
||||
var rows = Enumerable.Range(1, 28).Select(d => Day(d, agent, perDay)).ToList();
|
||||
rows.Add(Day(0, agent, today));
|
||||
return rows;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_unit_adds_up_itself_and_everything_below_it()
|
||||
{
|
||||
var lines = UnitCosts.Build(Org(), [Day(0, "ads", 3), Day(0, "seller", 2), Day(0, "bookkeeper", 1)], Today);
|
||||
|
||||
lines.Single(l => l.UnitId is null).Today.ShouldBe(6);
|
||||
lines.Single(l => l.UnitId == "marketing").Today.ShouldBe(5, "amazon is below marketing");
|
||||
lines.Single(l => l.UnitId == "amazon").Today.ShouldBe(2);
|
||||
lines.Single(l => l.UnitId == "").Today.ShouldBe(1, "the top level stands apart");
|
||||
lines.Select(l => l.Depth).ShouldBe([0, 1, 2, 1]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void The_maximum_is_the_sum_of_the_agents_limits_and_unknown_when_one_has_none()
|
||||
{
|
||||
UnitCosts.Build(Org(), [], Today).Single(l => l.UnitId == "marketing").MaxPerDay.ShouldBe(20);
|
||||
UnitCosts.Build(Org(adsLimit: 0), [], Today).Single(l => l.UnitId == "marketing").MaxPerDay.ShouldBeNull();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Months_are_compared_and_subscription_runs_count_as_quota_not_as_free()
|
||||
{
|
||||
var lines = UnitCosts.Build(Org(), [Day(0, "ads", 4, subscriptionTokens: 900), Day(12, "ads", 6)], Today);
|
||||
var marketing = lines.Single(l => l.UnitId == "marketing");
|
||||
|
||||
marketing.ThisMonth.ShouldBe(4);
|
||||
marketing.PreviousMonth.ShouldBe(6, "12 days before 10 March lies in February");
|
||||
marketing.SubscriptionTokensToday.ShouldBe(900);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_day_far_above_the_own_course_is_reported_though_nothing_is_exceeded()
|
||||
{
|
||||
var alerts = UnitCosts.Alerts(UnitCosts.Build(Org(), Course(1.5m, 4m), Today), new CostWatchSettings(), Today);
|
||||
|
||||
alerts.ShouldContain(a => a.Kind == CostAlertKind.Deviation && a.Scope == "marketing");
|
||||
alerts.ShouldNotContain(a => a.Kind == CostAlertKind.NearLimit, "4 $ of 20 $ is far from the limits");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Small_amounts_and_a_short_history_raise_nothing()
|
||||
{
|
||||
UnitCosts.Alerts(UnitCosts.Build(Org(), Course(0.01m, 0.5m), Today), new CostWatchSettings(), Today)
|
||||
.ShouldNotContain(a => a.Kind == CostAlertKind.Deviation, "below a dollar a day nothing deviates");
|
||||
|
||||
var shortHistory = new List<AgentDayUsage> { Day(1, "ads", 1), Day(2, "ads", 1), Day(0, "ads", 9) };
|
||||
UnitCosts.Alerts(UnitCosts.Build(Org(), shortHistory, Today), new CostWatchSettings(), Today)
|
||||
.ShouldNotContain(a => a.Kind == CostAlertKind.Deviation, "two days are no course");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_unit_near_the_sum_of_its_limits_is_reported_before_its_agents_run_into_them()
|
||||
{
|
||||
var alerts = UnitCosts.Alerts(UnitCosts.Build(Org(), [Day(0, "ads", 9), Day(0, "listing", 4), Day(0, "seller", 4)], Today),
|
||||
new CostWatchSettings(), Today);
|
||||
|
||||
alerts.ShouldContain(a => a.Kind == CostAlertKind.NearLimit && a.Scope == "marketing");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_monthly_total_is_reported_at_the_highest_threshold_crossed()
|
||||
{
|
||||
var settings = new CostWatchSettings { MonthlyThresholdsUsd = [5, 10, 50] };
|
||||
|
||||
var alert = UnitCosts.Alerts(UnitCosts.Build(Org(), [Day(0, "ads", 7), Day(3, "seller", 5)], Today), settings, Today)
|
||||
.Single(a => a.Kind == CostAlertKind.MonthlyThreshold);
|
||||
|
||||
alert.Scope.ShouldBe("instance:10");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_watch_raises_an_alert_once_and_hands_it_to_the_human()
|
||||
{
|
||||
var storage = new SqliteStorage(Path.Combine(_directory, "state.db"));
|
||||
var usage = new SqliteUsageRepository(storage);
|
||||
var tasks = new SqliteTaskRepository(storage);
|
||||
var board = new TaskboardService(tasks, Path.Combine(_directory, "tasks"));
|
||||
var now = DateTime.Now;
|
||||
foreach (var d in Enumerable.Range(1, 28))
|
||||
await usage.RecordAsync(new RunUsage { AgentId = "ads", Model = "m", CostUsd = 1m, CostIsKnown = true, Status = "Completed", OccurredAt = now.AddDays(-d) }, default);
|
||||
await usage.RecordAsync(new RunUsage { AgentId = "ads", Model = "m", CostUsd = 5m, CostIsKnown = true, Status = "Completed", OccurredAt = now }, default);
|
||||
|
||||
var watch = new CostWatch(storage, usage, () => Org(), () => new CostWatchSettings(), board, NullLogger.Instance);
|
||||
|
||||
(await watch.CheckAsync(default)).ShouldContain(a => a.Kind == CostAlertKind.Deviation && a.Scope == "marketing");
|
||||
(await watch.CheckAsync(default)).ShouldBeEmpty("once a day, not every quarter of an hour");
|
||||
|
||||
(await board.ListAsync(new TaskQuery { Assignee = TaskAssignee.Human }, default)).ShouldContain(t => t.Title.Contains("marketing"));
|
||||
(await watch.RecentAsync(7, default)).ShouldNotBeEmpty();
|
||||
(await watch.ReportAsync(default)).Single(l => l.UnitId == "marketing").Today.ShouldBe(5);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user