Prompt history: every run records its transcript (task, each model step with tokens and cost, tool calls and results, engine follow-ups) - bounded in size, scrubbed of recognisable secrets, system prompts stored once per text. Retention in days per instance (default 14) and per agent (0 = never, -1 = forever); an hourly janitor deletes what is past it, "never" also purges. Usage analysis over run receipts: filter by period, agent, model, event, status, cost and free text; sort by every column; totals over the whole filter; hourly or daily series grouped by agent, model or event; markers for budget denials and model changes; run detail with receipt, transcript, tool calls and approvals. WebUI: Analysis > Token usage > Runs / Timeline, run detail, Prompt History with retention settings. New code, comments and texts in English; texts come from language packs (i18n/en.json, de.json) via modules/i18n.js. CLAUDE.md records English as the main language. Version 0.1.12. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Richard
@@ -0,0 +1,143 @@
|
||||
using ClawdDotNet.Core.Analytics;
|
||||
using ClawdDotNet.Core.Audit;
|
||||
using ClawdDotNet.Core.Tests.Infrastructure;
|
||||
using Shouldly;
|
||||
|
||||
namespace ClawdDotNet.Core.Tests.Analytics;
|
||||
|
||||
/// <summary>
|
||||
/// Usage analysis (#660): filter, sort, page and bucket run receipts. Totals cover the whole
|
||||
/// filter, not only the page; runs refused before any model call are counted separately.
|
||||
/// </summary>
|
||||
public sealed class UsageAnalyticsTests
|
||||
{
|
||||
private static readonly DateTime Day = new(2026, 9, 19, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private static RunReceipt Receipt(string agent, int hour, decimal cost, string model = "gemini", string status = "Completed",
|
||||
string source = "task", string? task = null, int prompt = 1000, string result = "ERGEBNIS: ok")
|
||||
=> new()
|
||||
{
|
||||
RunId = Guid.NewGuid().ToString("N"),
|
||||
AgentId = agent,
|
||||
Model = model,
|
||||
Source = source,
|
||||
TaskId = task,
|
||||
Status = status,
|
||||
StepCount = 3,
|
||||
PromptTokens = prompt,
|
||||
CompletionTokens = 100,
|
||||
CostUsd = cost,
|
||||
CostIsKnown = cost > 0,
|
||||
ResultRef = result,
|
||||
OccurredAt = Day.AddHours(hour)
|
||||
};
|
||||
|
||||
private static UsageAnalytics Analytics(params RunReceipt[] receipts)
|
||||
{
|
||||
var audit = new InMemoryAuditRepository();
|
||||
foreach (var r in receipts) audit.Receipts.Enqueue(r);
|
||||
return new UsageAnalytics(audit, (id, _) => Task.FromResult<string?>(id == "t-1" ? "Wetter abrufen" : null));
|
||||
}
|
||||
|
||||
private static RunQuery All => new() { FromUtc = Day, ToUtc = Day.AddDays(1) };
|
||||
|
||||
[Fact]
|
||||
public async Task Totals_cover_the_whole_filter_not_only_the_page()
|
||||
{
|
||||
var analytics = Analytics(
|
||||
Receipt("wetter", 1, 0.01m), Receipt("wetter", 2, 0.02m), Receipt("kurse", 3, 0.04m),
|
||||
Receipt("kurse", 4, 0m, status: "BudgetExceeded", prompt: 0));
|
||||
|
||||
var page = await analytics.QueryRunsAsync(All with { Limit = 1 }, default);
|
||||
|
||||
page.Items.Count.ShouldBe(1);
|
||||
page.Total.ShouldBe(4);
|
||||
page.Totals.CostUsd.ShouldBe(0.07m);
|
||||
page.Totals.NotRun.ShouldBe(1);
|
||||
page.Totals.RunsWithUnknownCost.ShouldBe(0, "a refused run has no cost to know");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Filters_combine()
|
||||
{
|
||||
var analytics = Analytics(
|
||||
Receipt("wetter", 1, 0.01m, model: "haiku"), Receipt("wetter", 2, 0.02m, model: "gemini"),
|
||||
Receipt("kurse", 3, 0.04m, model: "gemini"));
|
||||
|
||||
var page = await analytics.QueryRunsAsync(All with { Agents = ["wetter"], Models = ["gemini"] }, default);
|
||||
|
||||
page.Total.ShouldBe(1);
|
||||
page.Items.Single().Receipt.CostUsd.ShouldBe(0.02m);
|
||||
(await analytics.QueryRunsAsync(All with { MinCostUsd = 0.015m }, default)).Total.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task The_search_covers_task_title_and_result()
|
||||
{
|
||||
var analytics = Analytics(
|
||||
Receipt("wetter", 1, 0.01m, task: "t-1"),
|
||||
Receipt("kurse", 2, 0.01m, result: "ERGEBNIS: NVDA bei 120"));
|
||||
|
||||
(await analytics.QueryRunsAsync(All with { Search = "wetter abrufen" }, default)).Items.Single().TaskTitle.ShouldBe("Wetter abrufen");
|
||||
(await analytics.QueryRunsAsync(All with { Search = "nvda" }, default)).Items.Single().Receipt.AgentId.ShouldBe("kurse");
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("cost", true, 0.04)]
|
||||
[InlineData("cost", false, 0.01)]
|
||||
[InlineData("time", true, 0.02)]
|
||||
public async Task Every_column_sorts(string sort, bool descending, double firstCost)
|
||||
{
|
||||
var analytics = Analytics(Receipt("a", 1, 0.01m), Receipt("b", 5, 0.02m), Receipt("c", 3, 0.04m));
|
||||
|
||||
var page = await analytics.QueryRunsAsync(All with { Sort = sort, Descending = descending }, default);
|
||||
|
||||
page.Items[0].Receipt.CostUsd.ShouldBe((decimal)firstCost);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Series_stack_per_bucket_and_group()
|
||||
{
|
||||
var analytics = Analytics(Receipt("wetter", 1, 0.01m), Receipt("wetter", 1, 0.02m), Receipt("kurse", 1, 0.04m), Receipt("wetter", 5, 0.1m));
|
||||
|
||||
var points = await analytics.SeriesAsync(All, "hour", "agent", TimeSpan.Zero, default);
|
||||
|
||||
points.Count.ShouldBe(3);
|
||||
points.Single(p => p.Group == "wetter" && p.BucketStart == Day.AddHours(1)).CostUsd.ShouldBe(0.03m);
|
||||
points.Single(p => p.Group == "wetter" && p.BucketStart == Day.AddHours(1)).Runs.ShouldBe(2);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void A_day_is_the_viewers_day()
|
||||
{
|
||||
// 23:30 UTC is already the next day in Berlin (UTC+2).
|
||||
var bucket = UsageAnalytics.BucketStart(Day.AddHours(23.5), "day", TimeSpan.FromHours(2));
|
||||
bucket.ShouldBe(Day.AddDays(1).AddHours(-2), "local midnight of 20.09 expressed in UTC");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Budget_denials_and_model_changes_become_events()
|
||||
{
|
||||
var analytics = Analytics(
|
||||
Receipt("wetter", 1, 0.01m, model: "haiku"),
|
||||
Receipt("wetter", 2, 0.01m, model: "gemini"),
|
||||
Receipt("wetter", 3, 0m, status: "BudgetExceeded", model: "gemini"),
|
||||
Receipt("wetter", 3, 0m, status: "BudgetExceeded", model: "gemini"));
|
||||
|
||||
var events = await analytics.EventsAsync(Day, Day.AddDays(1), default);
|
||||
|
||||
events.Single(e => e.Kind == "model_changed").Text.ShouldBe("haiku → gemini");
|
||||
events.Single(e => e.Kind == "budget_exceeded").Text.ShouldContain("2 run(s)");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Facets_list_what_occurs()
|
||||
{
|
||||
var analytics = Analytics(Receipt("wetter", 1, 0.01m, source: "task"), Receipt("kurse", 1, 0.01m, source: "webui", model: "haiku"));
|
||||
|
||||
var facets = await analytics.FacetsAsync(Day, Day.AddDays(1), default);
|
||||
|
||||
facets.Agents.ShouldBe(["kurse", "wetter"]);
|
||||
facets.Sources.ShouldBe(["task", "webui"]);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user