Prompt history: every run records its transcript (task, each model step with tokens and cost, tool calls and results, engine follow-ups) - bounded in size, scrubbed of recognisable secrets, system prompts stored once per text. Retention in days per instance (default 14) and per agent (0 = never, -1 = forever); an hourly janitor deletes what is past it, "never" also purges. Usage analysis over run receipts: filter by period, agent, model, event, status, cost and free text; sort by every column; totals over the whole filter; hourly or daily series grouped by agent, model or event; markers for budget denials and model changes; run detail with receipt, transcript, tool calls and approvals. WebUI: Analysis > Token usage > Runs / Timeline, run detail, Prompt History with retention settings. New code, comments and texts in English; texts come from language packs (i18n/en.json, de.json) via modules/i18n.js. CLAUDE.md records English as the main language. Version 0.1.12. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Richard
@@ -0,0 +1,224 @@
|
||||
using ClawdDotNet.Core.Audit;
|
||||
|
||||
namespace ClawdDotNet.Core.Analytics;
|
||||
|
||||
/// <summary>Filter, sort order and page of <see cref="UsageAnalytics.QueryRunsAsync"/>.</summary>
|
||||
public sealed record RunQuery
|
||||
{
|
||||
public DateTime FromUtc { get; init; }
|
||||
public DateTime ToUtc { get; init; }
|
||||
public IReadOnlyCollection<string> Agents { get; init; } = [];
|
||||
public IReadOnlyCollection<string> Models { get; init; } = [];
|
||||
public IReadOnlyCollection<string> Sources { get; init; } = [];
|
||||
public IReadOnlyCollection<string> Statuses { get; init; } = [];
|
||||
public decimal? MinCostUsd { get; init; }
|
||||
|
||||
/// <summary>Free text over run id, agent, model, task id, task title and result.</summary>
|
||||
public string? Search { get; init; }
|
||||
|
||||
/// <summary>time, agent, model, source, status, steps, prompt, completion, tokens, cost, duration.</summary>
|
||||
public string Sort { get; init; } = "time";
|
||||
public bool Descending { get; init; } = true;
|
||||
public int Offset { get; init; }
|
||||
public int Limit { get; init; } = 50;
|
||||
}
|
||||
|
||||
public sealed record RunRow(RunReceipt Receipt, string? TaskTitle)
|
||||
{
|
||||
public int TotalTokens => Receipt.PromptTokens + Receipt.CompletionTokens;
|
||||
}
|
||||
|
||||
/// <summary>Sums over the whole filter, not only the current page.</summary>
|
||||
public sealed record RunTotals(
|
||||
int Runs, long PromptTokens, long CompletionTokens, long CachedTokens,
|
||||
decimal CostUsd, int RunsWithUnknownCost, int NotRun);
|
||||
|
||||
public sealed record RunPage(IReadOnlyList<RunRow> Items, int Total, RunTotals Totals);
|
||||
|
||||
/// <summary>One stacked bar segment: <see cref="Group"/> within the bucket starting at <see cref="BucketStart"/>.</summary>
|
||||
public sealed record SeriesPoint(DateTime BucketStart, string Group, int Runs, long Tokens, decimal CostUsd);
|
||||
|
||||
/// <summary>Something worth a marker on the time axis.</summary>
|
||||
public sealed record UsageEvent(DateTime At, string Kind, string AgentId, string Text);
|
||||
|
||||
/// <summary>Values that occur in the range - the options of the filter dropdowns.</summary>
|
||||
public sealed record UsageFacets(
|
||||
IReadOnlyList<string> Agents, IReadOnlyList<string> Models,
|
||||
IReadOnlyList<string> Sources, IReadOnlyList<string> Statuses);
|
||||
|
||||
/// <summary>
|
||||
/// Usage analysis over run receipts (#660). Receipts rather than RunUsage: only they carry
|
||||
/// run id, source, task and the runs that never happened (budget denials, #608).
|
||||
///
|
||||
/// <para>Filtering, sorting and bucketing happen in memory over the receipts of the
|
||||
/// requested range. That keeps every rule in one testable place; the volume of a single
|
||||
/// instance (a few hundred runs a day) is far below where this would matter. The range is
|
||||
/// capped at <see cref="MaxReceipts"/> receipts.</para>
|
||||
/// </summary>
|
||||
public sealed class UsageAnalytics(
|
||||
IAuditRepository audit,
|
||||
Func<string, CancellationToken, Task<string?>>? taskTitle = null)
|
||||
{
|
||||
public const int MaxReceipts = 50_000;
|
||||
|
||||
/// <summary>Statuses of runs that were refused before any model call.</summary>
|
||||
private static readonly HashSet<string> NotRunStatuses =
|
||||
new(StringComparer.OrdinalIgnoreCase) { "BudgetExceeded", "ShuttingDown", "TargetBusy" };
|
||||
|
||||
public async Task<RunPage> QueryRunsAsync(RunQuery q, CancellationToken ct)
|
||||
{
|
||||
var rows = await LoadAsync(q.FromUtc, q.ToUtc, ct);
|
||||
var filtered = rows.Where(r => Matches(r, q)).ToList();
|
||||
|
||||
var totals = new RunTotals(
|
||||
filtered.Count,
|
||||
filtered.Sum(r => (long)r.Receipt.PromptTokens),
|
||||
filtered.Sum(r => (long)r.Receipt.CompletionTokens),
|
||||
filtered.Sum(r => (long)r.Receipt.CachedTokens),
|
||||
filtered.Sum(r => r.Receipt.CostUsd),
|
||||
filtered.Count(r => !r.Receipt.CostIsKnown && !NotRunStatuses.Contains(r.Receipt.Status)),
|
||||
filtered.Count(r => NotRunStatuses.Contains(r.Receipt.Status)));
|
||||
|
||||
var page = Sort(filtered, q.Sort, q.Descending)
|
||||
.Skip(Math.Max(0, q.Offset))
|
||||
.Take(Math.Clamp(q.Limit, 1, 500))
|
||||
.ToList();
|
||||
|
||||
return new RunPage(page, filtered.Count, totals);
|
||||
}
|
||||
|
||||
/// <param name="bucket"><c>hour</c> or <c>day</c>.</param>
|
||||
/// <param name="groupBy"><c>agent</c>, <c>model</c> or <c>source</c>.</param>
|
||||
/// <param name="utcOffset">Offset of the viewer's local time - a "day" is the viewer's day.</param>
|
||||
public async Task<IReadOnlyList<SeriesPoint>> SeriesAsync(
|
||||
RunQuery q, string bucket, string groupBy, TimeSpan utcOffset, CancellationToken ct)
|
||||
{
|
||||
var rows = (await LoadAsync(q.FromUtc, q.ToUtc, ct)).Where(r => Matches(r, q));
|
||||
|
||||
return rows
|
||||
.GroupBy(r => (Start: BucketStart(r.Receipt.OccurredAt, bucket, utcOffset), Group: GroupKey(r.Receipt, groupBy)))
|
||||
.Select(g => new SeriesPoint(
|
||||
g.Key.Start, g.Key.Group, g.Count(),
|
||||
g.Sum(r => (long)r.TotalTokens), g.Sum(r => r.Receipt.CostUsd)))
|
||||
.OrderBy(p => p.BucketStart).ThenBy(p => p.Group, StringComparer.Ordinal)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <summary>Budget denials and model changes - the usual explanation for a jump in a chart.</summary>
|
||||
public async Task<IReadOnlyList<UsageEvent>> EventsAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct)
|
||||
{
|
||||
var rows = await LoadAsync(fromUtc, toUtc, ct);
|
||||
var events = new List<UsageEvent>();
|
||||
|
||||
// One marker per agent and hour is enough for a denial streak.
|
||||
foreach (var g in rows.Where(r => r.Receipt.Status == "BudgetExceeded")
|
||||
.GroupBy(r => (r.Receipt.AgentId, Hour: Truncate(r.Receipt.OccurredAt, TimeSpan.FromHours(1)))))
|
||||
{
|
||||
events.Add(new UsageEvent(g.Min(r => r.Receipt.OccurredAt), "budget_exceeded", g.Key.AgentId,
|
||||
$"{g.Count()} run(s) refused: daily budget exhausted"));
|
||||
}
|
||||
|
||||
foreach (var agent in rows.GroupBy(r => r.Receipt.AgentId))
|
||||
{
|
||||
string? previous = null;
|
||||
foreach (var r in agent.Where(r => !string.IsNullOrWhiteSpace(r.Receipt.Model)))
|
||||
{
|
||||
if (previous is not null && previous != r.Receipt.Model)
|
||||
events.Add(new UsageEvent(r.Receipt.OccurredAt, "model_changed", agent.Key,
|
||||
$"{previous} → {r.Receipt.Model}"));
|
||||
previous = r.Receipt.Model;
|
||||
}
|
||||
}
|
||||
|
||||
return events.OrderBy(e => e.At).ToList();
|
||||
}
|
||||
|
||||
public async Task<UsageFacets> FacetsAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct)
|
||||
{
|
||||
var rows = await LoadAsync(fromUtc, toUtc, ct);
|
||||
static IReadOnlyList<string> Distinct(IEnumerable<string> values)
|
||||
=> values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct().Order(StringComparer.Ordinal).ToList();
|
||||
|
||||
return new UsageFacets(
|
||||
Distinct(rows.Select(r => r.Receipt.AgentId)),
|
||||
Distinct(rows.Select(r => r.Receipt.Model)),
|
||||
Distinct(rows.Select(r => r.Receipt.Source)),
|
||||
Distinct(rows.Select(r => r.Receipt.Status)));
|
||||
}
|
||||
|
||||
// ─── Building blocks ───
|
||||
|
||||
private async Task<IReadOnlyList<RunRow>> LoadAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct)
|
||||
{
|
||||
var receipts = await audit.ListReceiptsAsync(fromUtc, toUtc, MaxReceipts, ct);
|
||||
if (taskTitle is null)
|
||||
return receipts.Select(r => new RunRow(r, null)).ToList();
|
||||
|
||||
var titles = new Dictionary<string, string?>();
|
||||
foreach (var id in receipts.Select(r => r.TaskId).OfType<string>().Distinct())
|
||||
titles[id] = await taskTitle(id, ct);
|
||||
|
||||
return receipts.Select(r => new RunRow(r, r.TaskId is { } id ? titles.GetValueOrDefault(id) : null)).ToList();
|
||||
}
|
||||
|
||||
internal static bool Matches(RunRow row, RunQuery q)
|
||||
{
|
||||
var r = row.Receipt;
|
||||
if (q.Agents.Count > 0 && !q.Agents.Contains(r.AgentId)) return false;
|
||||
if (q.Models.Count > 0 && !q.Models.Contains(r.Model)) return false;
|
||||
if (q.Sources.Count > 0 && !q.Sources.Contains(r.Source)) return false;
|
||||
if (q.Statuses.Count > 0 && !q.Statuses.Contains(r.Status)) return false;
|
||||
if (q.MinCostUsd is { } min && r.CostUsd < min) return false;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(q.Search))
|
||||
{
|
||||
var s = q.Search.Trim();
|
||||
bool Has(string? v) => v?.Contains(s, StringComparison.OrdinalIgnoreCase) == true;
|
||||
if (!(Has(r.RunId) || Has(r.AgentId) || Has(r.Model) || Has(r.TaskId) || Has(row.TaskTitle) || Has(r.ResultRef)))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static IEnumerable<RunRow> Sort(IEnumerable<RunRow> rows, string sort, bool descending)
|
||||
{
|
||||
Func<RunRow, IComparable> key = (sort ?? "time").ToLowerInvariant() switch
|
||||
{
|
||||
"agent" => r => r.Receipt.AgentId,
|
||||
"model" => r => r.Receipt.Model,
|
||||
"source" => r => r.Receipt.Source,
|
||||
"status" => r => r.Receipt.Status,
|
||||
"steps" => r => r.Receipt.StepCount,
|
||||
"prompt" => r => r.Receipt.PromptTokens,
|
||||
"completion" => r => r.Receipt.CompletionTokens,
|
||||
"tokens" => r => r.TotalTokens,
|
||||
"cost" => r => r.Receipt.CostUsd,
|
||||
"duration" => r => r.Receipt.DurationMs,
|
||||
_ => r => r.Receipt.OccurredAt
|
||||
};
|
||||
|
||||
// Newest first as the tie-breaker, whatever the primary order.
|
||||
return descending
|
||||
? rows.OrderByDescending(key).ThenByDescending(r => r.Receipt.OccurredAt)
|
||||
: rows.OrderBy(key).ThenByDescending(r => r.Receipt.OccurredAt);
|
||||
}
|
||||
|
||||
internal static DateTime BucketStart(DateTime occurredUtc, string bucket, TimeSpan utcOffset)
|
||||
{
|
||||
var local = DateTime.SpecifyKind(occurredUtc, DateTimeKind.Utc) + utcOffset;
|
||||
var start = bucket == "day"
|
||||
? local.Date
|
||||
: new DateTime(local.Year, local.Month, local.Day, local.Hour, 0, 0);
|
||||
return DateTime.SpecifyKind(start - utcOffset, DateTimeKind.Utc);
|
||||
}
|
||||
|
||||
private static string GroupKey(RunReceipt r, string groupBy) => groupBy switch
|
||||
{
|
||||
"model" => string.IsNullOrWhiteSpace(r.Model) ? "(unknown)" : r.Model,
|
||||
"source" => r.Source,
|
||||
_ => r.AgentId
|
||||
};
|
||||
|
||||
private static DateTime Truncate(DateTime value, TimeSpan unit)
|
||||
=> new(value.Ticks - value.Ticks % unit.Ticks, value.Kind);
|
||||
}
|
||||
Reference in New Issue
Block a user