Prompt history: every run records its transcript (task, each model step with tokens and cost, tool calls and results, engine follow-ups) - bounded in size, scrubbed of recognisable secrets, system prompts stored once per text. Retention in days per instance (default 14) and per agent (0 = never, -1 = forever); an hourly janitor deletes what is past it, "never" also purges. Usage analysis over run receipts: filter by period, agent, model, event, status, cost and free text; sort by every column; totals over the whole filter; hourly or daily series grouped by agent, model or event; markers for budget denials and model changes; run detail with receipt, transcript, tool calls and approvals. WebUI: Analysis > Token usage > Runs / Timeline, run detail, Prompt History with retention settings. New code, comments and texts in English; texts come from language packs (i18n/en.json, de.json) via modules/i18n.js. CLAUDE.md records English as the main language. Version 0.1.12. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Richard
@@ -0,0 +1,279 @@
|
||||
using ClawdDotNet.App;
|
||||
using ClawdDotNet.App.Services;
|
||||
using ClawdDotNet.Core.Analytics;
|
||||
using ClawdDotNet.Core.Transcripts;
|
||||
using ClawdDotNet.Web.Contracts;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Routing;
|
||||
|
||||
namespace ClawdDotNet.Web.Endpoints;
|
||||
|
||||
/// <summary>
|
||||
/// Usage analysis and prompt history (#660).
|
||||
///
|
||||
/// <list type="bullet">
|
||||
/// <item><c>GET /api/usage/runs</c> - filterable, sortable, paged list of runs with totals over the whole filter</item>
|
||||
/// <item><c>GET /api/usage/series</c> - stacked buckets per hour or day for the timeline</item>
|
||||
/// <item><c>GET /api/usage/events</c> - markers for the timeline (budget denials, model changes)</item>
|
||||
/// <item><c>GET /api/usage/facets</c> - the values the filter dropdowns offer</item>
|
||||
/// <item><c>GET /api/runs/{runId}</c> - one run: receipt, tool calls, approvals, transcript</item>
|
||||
/// <item><c>GET /api/prompt-history</c> - recorded transcripts with full-text search</item>
|
||||
/// <item><c>GET|PUT /api/prompt-history/settings</c> - retention per instance and per agent</item>
|
||||
/// </list>
|
||||
///
|
||||
/// Filters use the same query parameters everywhere: <c>from</c>, <c>to</c> (ISO 8601),
|
||||
/// <c>agent</c>, <c>model</c>, <c>source</c>, <c>status</c> (comma separated),
|
||||
/// <c>minCost</c>, <c>q</c>. Without <c>from</c> the last 7 days are shown.
|
||||
/// </summary>
|
||||
public static class AnalyticsEndpoints
|
||||
{
|
||||
private static readonly TimeSpan DefaultRange = TimeSpan.FromDays(7);
|
||||
|
||||
public static void MapAnalytics(this RouteGroupBuilder api)
|
||||
{
|
||||
api.MapGet("/usage/runs", async (AppHost host, HttpRequest request, CancellationToken ct) =>
|
||||
{
|
||||
if (host.Insights is not { } insights)
|
||||
return ApiEndpoints.Unavailable("No usage data: agents are not active.");
|
||||
|
||||
var page = await insights.Analytics.QueryRunsAsync(ParseQuery(request), ct);
|
||||
return Results.Ok(new RunPageDto(
|
||||
page.Items.Select(ToDto).ToList(),
|
||||
page.Total,
|
||||
new RunTotalsDto(page.Totals.Runs, page.Totals.PromptTokens, page.Totals.CompletionTokens,
|
||||
page.Totals.CachedTokens, page.Totals.CostUsd, page.Totals.RunsWithUnknownCost, page.Totals.NotRun)));
|
||||
});
|
||||
|
||||
api.MapGet("/usage/series", async (
|
||||
AppHost host, HttpRequest request, CancellationToken ct,
|
||||
string bucket = "hour", string groupBy = "agent", int tzOffset = 0) =>
|
||||
{
|
||||
if (host.Insights is not { } insights)
|
||||
return ApiEndpoints.Unavailable("No usage data: agents are not active.");
|
||||
|
||||
var points = await insights.Analytics.SeriesAsync(
|
||||
ParseQuery(request),
|
||||
bucket == "day" ? "day" : "hour",
|
||||
groupBy is "model" or "source" ? groupBy : "agent",
|
||||
TimeSpan.FromMinutes(Math.Clamp(tzOffset, -14 * 60, 14 * 60)),
|
||||
ct);
|
||||
return Results.Ok(points.Select(p => new SeriesPointDto(p.BucketStart, p.Group, p.Runs, p.Tokens, p.CostUsd)).ToList());
|
||||
});
|
||||
|
||||
api.MapGet("/usage/events", async (AppHost host, HttpRequest request, CancellationToken ct) =>
|
||||
{
|
||||
if (host.Insights is not { } insights)
|
||||
return ApiEndpoints.Unavailable("No usage data: agents are not active.");
|
||||
|
||||
var (from, to) = ParseRange(request);
|
||||
var events = await insights.Analytics.EventsAsync(from, to, ct);
|
||||
return Results.Ok(events.Select(e => new UsageEventDto(e.At, e.Kind, e.AgentId, e.Text)).ToList());
|
||||
});
|
||||
|
||||
api.MapGet("/usage/facets", async (AppHost host, HttpRequest request, CancellationToken ct) =>
|
||||
{
|
||||
if (host.Insights is not { } insights)
|
||||
return ApiEndpoints.Unavailable("No usage data: agents are not active.");
|
||||
|
||||
var (from, to) = ParseRange(request);
|
||||
var f = await insights.Analytics.FacetsAsync(from, to, ct);
|
||||
return Results.Ok(new UsageFacetsDto(f.Agents, f.Models, f.Sources, f.Statuses));
|
||||
});
|
||||
|
||||
api.MapGet("/runs/{runId}", async (AppHost host, string runId, CancellationToken ct) =>
|
||||
{
|
||||
if (host.Insights is not { } insights)
|
||||
return ApiEndpoints.Unavailable("No usage data: agents are not active.");
|
||||
|
||||
return await insights.GetRunAsync(runId, ct) is { } detail
|
||||
? Results.Ok(ToDto(detail))
|
||||
: Results.NotFound(new OutcomeDto(false, $"No run with id '{runId}'."));
|
||||
});
|
||||
|
||||
api.MapGet("/prompt-history", async (
|
||||
AppHost host, HttpRequest request, CancellationToken ct, int offset = 0, int limit = 50) =>
|
||||
{
|
||||
if (host.Insights?.Transcripts is not { } store)
|
||||
return ApiEndpoints.Unavailable("Prompt history is not available in this instance.");
|
||||
|
||||
var (from, to) = ParseRange(request);
|
||||
var (items, total) = await store.ListAsync(new TranscriptQuery
|
||||
{
|
||||
AgentId = request.Query["agent"].FirstOrDefault(),
|
||||
From = from,
|
||||
To = to,
|
||||
Search = request.Query["q"].FirstOrDefault(),
|
||||
Offset = offset,
|
||||
Limit = limit
|
||||
}, ct);
|
||||
|
||||
return Results.Ok(new TranscriptPageDto(items.Select(i => new TranscriptSummaryDto(
|
||||
i.RunId, i.AgentId, i.Model, i.Source, i.TaskId, i.StartedAt, i.CompletedAt,
|
||||
i.Status, i.Prompt, i.Response, i.SizeChars, i.Truncated)).ToList(), total));
|
||||
});
|
||||
|
||||
api.MapGet("/prompt-history/settings", (AppHost host) => Results.Ok(SettingsDto(host)));
|
||||
|
||||
api.MapPut("/prompt-history/settings", (AppHost host, PromptHistorySettingsInputDto input) =>
|
||||
{
|
||||
var messages = new List<string>();
|
||||
|
||||
if (input.DefaultRetentionDays is { } days)
|
||||
{
|
||||
var result = host.AgentAdmin.SetDefaultPromptHistoryRetention(days);
|
||||
if (!result.Ok) return result.ToResponse();
|
||||
messages.Add(result.Message);
|
||||
}
|
||||
|
||||
foreach (var agent in input.Agents ?? [])
|
||||
{
|
||||
var result = host.AgentAdmin.SetPromptHistoryRetention(agent.AgentId, agent.RetentionDays);
|
||||
if (!result.Ok) return result.ToResponse();
|
||||
messages.Add(result.Message);
|
||||
}
|
||||
|
||||
return Results.Ok(new OutcomeDto(true, messages.Count == 0 ? "Nothing changed." : string.Join(" ", messages)));
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Parsing ───
|
||||
|
||||
internal static (DateTime From, DateTime To) ParseRange(HttpRequest request)
|
||||
{
|
||||
var to = ParseDate(request.Query["to"].FirstOrDefault()) ?? DateTime.UtcNow.AddMinutes(1);
|
||||
var from = ParseDate(request.Query["from"].FirstOrDefault()) ?? to - DefaultRange;
|
||||
return (from, to);
|
||||
}
|
||||
|
||||
internal static RunQuery ParseQuery(HttpRequest request)
|
||||
{
|
||||
var (from, to) = ParseRange(request);
|
||||
var query = request.Query;
|
||||
|
||||
static IReadOnlyCollection<string> List(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value)
|
||||
? []
|
||||
: value.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToHashSet();
|
||||
|
||||
return new RunQuery
|
||||
{
|
||||
FromUtc = from,
|
||||
ToUtc = to,
|
||||
Agents = List(query["agent"].FirstOrDefault()),
|
||||
Models = List(query["model"].FirstOrDefault()),
|
||||
Sources = List(query["source"].FirstOrDefault()),
|
||||
Statuses = List(query["status"].FirstOrDefault()),
|
||||
MinCostUsd = decimal.TryParse(query["minCost"].FirstOrDefault(), System.Globalization.NumberStyles.Any,
|
||||
System.Globalization.CultureInfo.InvariantCulture, out var min) ? min : null,
|
||||
Search = query["q"].FirstOrDefault(),
|
||||
Sort = query["sort"].FirstOrDefault() ?? "time",
|
||||
Descending = !string.Equals(query["dir"].FirstOrDefault(), "asc", StringComparison.OrdinalIgnoreCase),
|
||||
Offset = int.TryParse(query["offset"].FirstOrDefault(), out var o) ? o : 0,
|
||||
Limit = int.TryParse(query["limit"].FirstOrDefault(), out var l) ? l : 50
|
||||
};
|
||||
}
|
||||
|
||||
private static DateTime? ParseDate(string? value)
|
||||
=> DateTime.TryParse(value, System.Globalization.CultureInfo.InvariantCulture,
|
||||
System.Globalization.DateTimeStyles.AdjustToUniversal | System.Globalization.DateTimeStyles.AssumeUniversal, out var d)
|
||||
? DateTime.SpecifyKind(d, DateTimeKind.Utc)
|
||||
: null;
|
||||
|
||||
// ─── Mapping ───
|
||||
|
||||
private static RunRowDto ToDto(RunRow row)
|
||||
{
|
||||
var r = row.Receipt;
|
||||
return new RunRowDto(r.RunId, r.AgentId, r.Model, r.Source, r.TaskId, row.TaskTitle, r.Status,
|
||||
r.StepCount, r.PromptTokens, r.CompletionTokens, r.CachedTokens, r.CostUsd, r.CostIsKnown,
|
||||
r.DurationMs, r.ResultRef, r.OccurredAt);
|
||||
}
|
||||
|
||||
private static RunDetailDto ToDto(RunDetail d) => new(
|
||||
d.RunId,
|
||||
d.Receipt is { } r
|
||||
? new RunRowDto(r.RunId, r.AgentId, r.Model, r.Source, r.TaskId, d.TaskTitle, r.Status, r.StepCount,
|
||||
r.PromptTokens, r.CompletionTokens, r.CachedTokens, r.CostUsd, r.CostIsKnown, r.DurationMs, r.ResultRef, r.OccurredAt)
|
||||
: null,
|
||||
d.ToolCalls.Select(c => new RunToolCallDto(c.OccurredAt, c.Tool, c.Status.ToString(), c.Arguments, c.Summary, c.DurationMs, c.Source)).ToList(),
|
||||
d.Approvals.Select(c => c.ToDto()).ToList(),
|
||||
d.Transcript is { } t
|
||||
? new TranscriptDto(t.RunId, t.AgentId, t.Model, t.Source, t.TaskId, t.StartedAt, t.CompletedAt, t.Status,
|
||||
t.Entries, t.SizeChars, t.Truncated)
|
||||
: null,
|
||||
d.SystemPrompt,
|
||||
d.TranscriptState);
|
||||
|
||||
private static PromptHistorySettingsDto SettingsDto(AppHost host)
|
||||
{
|
||||
var instanceDays = PromptHistorySettings.Normalize(host.Instance.PromptHistory.RetentionDays);
|
||||
return new PromptHistorySettingsDto(
|
||||
instanceDays,
|
||||
host.Instance.Agents.Select(a => new AgentRetentionDto(
|
||||
a.AgentId,
|
||||
string.IsNullOrWhiteSpace(a.DisplayName) ? a.AgentId : a.DisplayName,
|
||||
a.PromptHistory.RetentionDays,
|
||||
PromptHistorySettings.Resolve(a.PromptHistory.RetentionDays, instanceDays))).ToList());
|
||||
}
|
||||
}
|
||||
|
||||
// ─── DTOs (#660) ───
|
||||
|
||||
public sealed record RunRowDto(
|
||||
string RunId, string AgentId, string Model, string Source, string? TaskId, string? TaskTitle,
|
||||
string Status, int StepCount, int PromptTokens, int CompletionTokens, int CachedTokens,
|
||||
decimal CostUsd, bool CostIsKnown, long DurationMs, string Result, DateTime OccurredAt);
|
||||
|
||||
public sealed record RunTotalsDto(
|
||||
int Runs, long PromptTokens, long CompletionTokens, long CachedTokens,
|
||||
decimal CostUsd, int RunsWithUnknownCost, int NotRun);
|
||||
|
||||
public sealed record RunPageDto(IReadOnlyList<RunRowDto> Items, int Total, RunTotalsDto Totals);
|
||||
|
||||
public sealed record SeriesPointDto(DateTime BucketStart, string Group, int Runs, long Tokens, decimal CostUsd);
|
||||
|
||||
public sealed record UsageEventDto(DateTime At, string Kind, string AgentId, string Text);
|
||||
|
||||
public sealed record UsageFacetsDto(
|
||||
IReadOnlyList<string> Agents, IReadOnlyList<string> Models,
|
||||
IReadOnlyList<string> Sources, IReadOnlyList<string> Statuses);
|
||||
|
||||
public sealed record RunToolCallDto(
|
||||
DateTime At, string Tool, string Status, string Arguments, string Summary, long DurationMs, string Source);
|
||||
|
||||
/// <summary>
|
||||
/// Entries go out as recorded. They contain model output and tool results - untrusted
|
||||
/// text; the UI inserts them as text, never as HTML.
|
||||
/// </summary>
|
||||
public sealed record TranscriptDto(
|
||||
string RunId, string AgentId, string Model, string Source, string? TaskId,
|
||||
DateTime StartedAt, DateTime? CompletedAt, string Status,
|
||||
IReadOnlyList<TranscriptEntry> Entries, int SizeChars, bool Truncated);
|
||||
|
||||
/// <param name="TranscriptState">recorded, disabled (retention "never"), missing, unavailable.</param>
|
||||
public sealed record RunDetailDto(
|
||||
string RunId,
|
||||
RunRowDto? Receipt,
|
||||
IReadOnlyList<RunToolCallDto> ToolCalls,
|
||||
IReadOnlyList<StagedCallDto> Approvals,
|
||||
TranscriptDto? Transcript,
|
||||
string? SystemPrompt,
|
||||
string TranscriptState);
|
||||
|
||||
public sealed record TranscriptSummaryDto(
|
||||
string RunId, string AgentId, string Model, string Source, string? TaskId,
|
||||
DateTime StartedAt, DateTime? CompletedAt, string Status,
|
||||
string Prompt, string Response, int SizeChars, bool Truncated);
|
||||
|
||||
public sealed record TranscriptPageDto(IReadOnlyList<TranscriptSummaryDto> Items, int Total);
|
||||
|
||||
/// <param name="RetentionDays">Own setting of the agent; <c>null</c> = instance default.</param>
|
||||
/// <param name="EffectiveRetentionDays">What applies: 0 = never, -1 = forever.</param>
|
||||
public sealed record AgentRetentionDto(string AgentId, string DisplayName, int? RetentionDays, int EffectiveRetentionDays);
|
||||
|
||||
public sealed record PromptHistorySettingsDto(int DefaultRetentionDays, IReadOnlyList<AgentRetentionDto> Agents);
|
||||
|
||||
public sealed record AgentRetentionInputDto(string AgentId, int? RetentionDays);
|
||||
|
||||
public sealed record PromptHistorySettingsInputDto(int? DefaultRetentionDays, IReadOnlyList<AgentRetentionInputDto>? Agents);
|
||||
@@ -29,6 +29,7 @@ internal static class ApiEndpoints
|
||||
api.MapTasks();
|
||||
api.MapStaging();
|
||||
api.MapUsage();
|
||||
api.MapAnalytics();
|
||||
api.MapLogs();
|
||||
|
||||
app.MapEvents();
|
||||
|
||||
Reference in New Issue
Block a user