Prompt History und Verbrauchsauswertung (#660), englische Hauptsprache (#661)

Prompt history: every run records its transcript (task, each model step with
tokens and cost, tool calls and results, engine follow-ups) - bounded in size,
scrubbed of recognisable secrets, system prompts stored once per text.
Retention in days per instance (default 14) and per agent (0 = never, -1 =
forever); an hourly janitor deletes what is past it, "never" also purges.

Usage analysis over run receipts: filter by period, agent, model, event, status,
cost and free text; sort by every column; totals over the whole filter; hourly
or daily series grouped by agent, model or event; markers for budget denials and
model changes; run detail with receipt, transcript, tool calls and approvals.

WebUI: Analysis > Token usage > Runs / Timeline, run detail, Prompt History with
retention settings. New code, comments and texts in English; texts come from
language packs (i18n/en.json, de.json) via modules/i18n.js. CLAUDE.md records
English as the main language.

Version 0.1.12.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Richard
2026-09-19 15:33:09 +02:00
co-authored by Claude Opus 5
parent 2f0361278b
commit 5adf4c1ca6
40 changed files with 3521 additions and 10 deletions
@@ -0,0 +1,224 @@
using ClawdDotNet.Core.Audit;
namespace ClawdDotNet.Core.Analytics;
/// <summary>Filter, sort order and page of <see cref="UsageAnalytics.QueryRunsAsync"/>.</summary>
public sealed record RunQuery
{
public DateTime FromUtc { get; init; }
public DateTime ToUtc { get; init; }
public IReadOnlyCollection<string> Agents { get; init; } = [];
public IReadOnlyCollection<string> Models { get; init; } = [];
public IReadOnlyCollection<string> Sources { get; init; } = [];
public IReadOnlyCollection<string> Statuses { get; init; } = [];
public decimal? MinCostUsd { get; init; }
/// <summary>Free text over run id, agent, model, task id, task title and result.</summary>
public string? Search { get; init; }
/// <summary>time, agent, model, source, status, steps, prompt, completion, tokens, cost, duration.</summary>
public string Sort { get; init; } = "time";
public bool Descending { get; init; } = true;
public int Offset { get; init; }
public int Limit { get; init; } = 50;
}
public sealed record RunRow(RunReceipt Receipt, string? TaskTitle)
{
public int TotalTokens => Receipt.PromptTokens + Receipt.CompletionTokens;
}
/// <summary>Sums over the whole filter, not only the current page.</summary>
public sealed record RunTotals(
int Runs, long PromptTokens, long CompletionTokens, long CachedTokens,
decimal CostUsd, int RunsWithUnknownCost, int NotRun);
public sealed record RunPage(IReadOnlyList<RunRow> Items, int Total, RunTotals Totals);
/// <summary>One stacked bar segment: <see cref="Group"/> within the bucket starting at <see cref="BucketStart"/>.</summary>
public sealed record SeriesPoint(DateTime BucketStart, string Group, int Runs, long Tokens, decimal CostUsd);
/// <summary>Something worth a marker on the time axis.</summary>
public sealed record UsageEvent(DateTime At, string Kind, string AgentId, string Text);
/// <summary>Values that occur in the range - the options of the filter dropdowns.</summary>
public sealed record UsageFacets(
IReadOnlyList<string> Agents, IReadOnlyList<string> Models,
IReadOnlyList<string> Sources, IReadOnlyList<string> Statuses);
/// <summary>
/// Usage analysis over run receipts (#660). Receipts rather than RunUsage: only they carry
/// run id, source, task and the runs that never happened (budget denials, #608).
///
/// <para>Filtering, sorting and bucketing happen in memory over the receipts of the
/// requested range. That keeps every rule in one testable place; the volume of a single
/// instance (a few hundred runs a day) is far below where this would matter. The range is
/// capped at <see cref="MaxReceipts"/> receipts.</para>
/// </summary>
public sealed class UsageAnalytics(
IAuditRepository audit,
Func<string, CancellationToken, Task<string?>>? taskTitle = null)
{
public const int MaxReceipts = 50_000;
/// <summary>Statuses of runs that were refused before any model call.</summary>
private static readonly HashSet<string> NotRunStatuses =
new(StringComparer.OrdinalIgnoreCase) { "BudgetExceeded", "ShuttingDown", "TargetBusy" };
public async Task<RunPage> QueryRunsAsync(RunQuery q, CancellationToken ct)
{
var rows = await LoadAsync(q.FromUtc, q.ToUtc, ct);
var filtered = rows.Where(r => Matches(r, q)).ToList();
var totals = new RunTotals(
filtered.Count,
filtered.Sum(r => (long)r.Receipt.PromptTokens),
filtered.Sum(r => (long)r.Receipt.CompletionTokens),
filtered.Sum(r => (long)r.Receipt.CachedTokens),
filtered.Sum(r => r.Receipt.CostUsd),
filtered.Count(r => !r.Receipt.CostIsKnown && !NotRunStatuses.Contains(r.Receipt.Status)),
filtered.Count(r => NotRunStatuses.Contains(r.Receipt.Status)));
var page = Sort(filtered, q.Sort, q.Descending)
.Skip(Math.Max(0, q.Offset))
.Take(Math.Clamp(q.Limit, 1, 500))
.ToList();
return new RunPage(page, filtered.Count, totals);
}
/// <param name="bucket"><c>hour</c> or <c>day</c>.</param>
/// <param name="groupBy"><c>agent</c>, <c>model</c> or <c>source</c>.</param>
/// <param name="utcOffset">Offset of the viewer's local time - a "day" is the viewer's day.</param>
public async Task<IReadOnlyList<SeriesPoint>> SeriesAsync(
RunQuery q, string bucket, string groupBy, TimeSpan utcOffset, CancellationToken ct)
{
var rows = (await LoadAsync(q.FromUtc, q.ToUtc, ct)).Where(r => Matches(r, q));
return rows
.GroupBy(r => (Start: BucketStart(r.Receipt.OccurredAt, bucket, utcOffset), Group: GroupKey(r.Receipt, groupBy)))
.Select(g => new SeriesPoint(
g.Key.Start, g.Key.Group, g.Count(),
g.Sum(r => (long)r.TotalTokens), g.Sum(r => r.Receipt.CostUsd)))
.OrderBy(p => p.BucketStart).ThenBy(p => p.Group, StringComparer.Ordinal)
.ToList();
}
/// <summary>Budget denials and model changes - the usual explanation for a jump in a chart.</summary>
public async Task<IReadOnlyList<UsageEvent>> EventsAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct)
{
var rows = await LoadAsync(fromUtc, toUtc, ct);
var events = new List<UsageEvent>();
// One marker per agent and hour is enough for a denial streak.
foreach (var g in rows.Where(r => r.Receipt.Status == "BudgetExceeded")
.GroupBy(r => (r.Receipt.AgentId, Hour: Truncate(r.Receipt.OccurredAt, TimeSpan.FromHours(1)))))
{
events.Add(new UsageEvent(g.Min(r => r.Receipt.OccurredAt), "budget_exceeded", g.Key.AgentId,
$"{g.Count()} run(s) refused: daily budget exhausted"));
}
foreach (var agent in rows.GroupBy(r => r.Receipt.AgentId))
{
string? previous = null;
foreach (var r in agent.Where(r => !string.IsNullOrWhiteSpace(r.Receipt.Model)))
{
if (previous is not null && previous != r.Receipt.Model)
events.Add(new UsageEvent(r.Receipt.OccurredAt, "model_changed", agent.Key,
$"{previous} → {r.Receipt.Model}"));
previous = r.Receipt.Model;
}
}
return events.OrderBy(e => e.At).ToList();
}
public async Task<UsageFacets> FacetsAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct)
{
var rows = await LoadAsync(fromUtc, toUtc, ct);
static IReadOnlyList<string> Distinct(IEnumerable<string> values)
=> values.Where(v => !string.IsNullOrWhiteSpace(v)).Distinct().Order(StringComparer.Ordinal).ToList();
return new UsageFacets(
Distinct(rows.Select(r => r.Receipt.AgentId)),
Distinct(rows.Select(r => r.Receipt.Model)),
Distinct(rows.Select(r => r.Receipt.Source)),
Distinct(rows.Select(r => r.Receipt.Status)));
}
// ─── Building blocks ───
private async Task<IReadOnlyList<RunRow>> LoadAsync(DateTime fromUtc, DateTime toUtc, CancellationToken ct)
{
var receipts = await audit.ListReceiptsAsync(fromUtc, toUtc, MaxReceipts, ct);
if (taskTitle is null)
return receipts.Select(r => new RunRow(r, null)).ToList();
var titles = new Dictionary<string, string?>();
foreach (var id in receipts.Select(r => r.TaskId).OfType<string>().Distinct())
titles[id] = await taskTitle(id, ct);
return receipts.Select(r => new RunRow(r, r.TaskId is { } id ? titles.GetValueOrDefault(id) : null)).ToList();
}
internal static bool Matches(RunRow row, RunQuery q)
{
var r = row.Receipt;
if (q.Agents.Count > 0 && !q.Agents.Contains(r.AgentId)) return false;
if (q.Models.Count > 0 && !q.Models.Contains(r.Model)) return false;
if (q.Sources.Count > 0 && !q.Sources.Contains(r.Source)) return false;
if (q.Statuses.Count > 0 && !q.Statuses.Contains(r.Status)) return false;
if (q.MinCostUsd is { } min && r.CostUsd < min) return false;
if (!string.IsNullOrWhiteSpace(q.Search))
{
var s = q.Search.Trim();
bool Has(string? v) => v?.Contains(s, StringComparison.OrdinalIgnoreCase) == true;
if (!(Has(r.RunId) || Has(r.AgentId) || Has(r.Model) || Has(r.TaskId) || Has(row.TaskTitle) || Has(r.ResultRef)))
return false;
}
return true;
}
private static IEnumerable<RunRow> Sort(IEnumerable<RunRow> rows, string sort, bool descending)
{
Func<RunRow, IComparable> key = (sort ?? "time").ToLowerInvariant() switch
{
"agent" => r => r.Receipt.AgentId,
"model" => r => r.Receipt.Model,
"source" => r => r.Receipt.Source,
"status" => r => r.Receipt.Status,
"steps" => r => r.Receipt.StepCount,
"prompt" => r => r.Receipt.PromptTokens,
"completion" => r => r.Receipt.CompletionTokens,
"tokens" => r => r.TotalTokens,
"cost" => r => r.Receipt.CostUsd,
"duration" => r => r.Receipt.DurationMs,
_ => r => r.Receipt.OccurredAt
};
// Newest first as the tie-breaker, whatever the primary order.
return descending
? rows.OrderByDescending(key).ThenByDescending(r => r.Receipt.OccurredAt)
: rows.OrderBy(key).ThenByDescending(r => r.Receipt.OccurredAt);
}
internal static DateTime BucketStart(DateTime occurredUtc, string bucket, TimeSpan utcOffset)
{
var local = DateTime.SpecifyKind(occurredUtc, DateTimeKind.Utc) + utcOffset;
var start = bucket == "day"
? local.Date
: new DateTime(local.Year, local.Month, local.Day, local.Hour, 0, 0);
return DateTime.SpecifyKind(start - utcOffset, DateTimeKind.Utc);
}
private static string GroupKey(RunReceipt r, string groupBy) => groupBy switch
{
"model" => string.IsNullOrWhiteSpace(r.Model) ? "(unknown)" : r.Model,
"source" => r.Source,
_ => r.AgentId
};
private static DateTime Truncate(DateTime value, TimeSpan unit)
=> new(value.Ticks - value.Ticks % unit.Ticks, value.Kind);
}
@@ -24,6 +24,12 @@ public interface IAuditRepository
/// <summary>Alle Belege zu einem Task — die Kosten pro Ergebnis (C7).</summary>
Task<IReadOnlyList<RunReceipt>> ListReceiptsForTaskAsync(string taskId, CancellationToken ct);
/// <summary>
/// #660: All receipts in <c>[fromUtc, toUtc)</c>, oldest first, at most
/// <paramref name="limit"/> - the input of the usage analysis.
/// </summary>
Task<IReadOnlyList<RunReceipt>> ListReceiptsAsync(DateTime fromUtc, DateTime toUtc, int limit, CancellationToken ct);
/// <summary>Zahl der Log-Einträge insgesamt.</summary>
Task<int> CountAsync(CancellationToken ct);
@@ -99,6 +99,24 @@ public sealed class SqliteAuditRepository : IAuditRepository
return await reader.ReadAsync(ct) ? ReadReceipt(reader) : null;
}
public async Task<IReadOnlyList<RunReceipt>> ListReceiptsAsync(DateTime fromUtc, DateTime toUtc, int limit, CancellationToken ct)
{
await using var conn = await _storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
// OccurredAt is ISO-8601 UTC ("O"), so the text comparison is a time comparison.
cmd.CommandText = "SELECT " + ReceiptColumns +
" FROM RunReceipts WHERE OccurredAt >= @from AND OccurredAt < @to ORDER BY OccurredAt LIMIT @limit";
cmd.Parameters.AddWithValue("@from", Format(fromUtc));
cmd.Parameters.AddWithValue("@to", Format(toUtc));
cmd.Parameters.AddWithValue("@limit", limit);
var results = new List<RunReceipt>();
await using var reader = await cmd.ExecuteReaderAsync(ct);
while (await reader.ReadAsync(ct))
results.Add(ReadReceipt(reader));
return results;
}
public async Task<IReadOnlyList<RunReceipt>> ListReceiptsForTaskAsync(string taskId, CancellationToken ct)
{
await using var conn = await _storage.OpenConnectionAsync(ct);
@@ -145,6 +145,12 @@ public sealed class AgentConfig
[JsonPropertyName("budget")]
public AgentBudget Budget { get; set; } = new();
/// <summary>
/// #660: Per-agent retention of the prompt history. <c>null</c> retention = instance default.
/// </summary>
[JsonPropertyName("promptHistory")]
public Transcripts.AgentPromptHistorySettings PromptHistory { get; set; } = new();
/// <summary>
/// Übersetzt <see cref="ReasoningEffort"/> in das Anfragefeld. Null heißt:
/// nichts mitschicken, das Modell denkt wie voreingestellt.
@@ -124,6 +124,13 @@ public sealed class InstanceConfig
[JsonPropertyName("budget")]
public InstanceBudget Budget { get; set; } = new();
/// <summary>
/// #660: How long prompt and response of every run are kept. Default for all agents;
/// each agent can override it.
/// </summary>
[JsonPropertyName("promptHistory")]
public Transcripts.PromptHistorySettings PromptHistory { get; set; } = new();
/// <summary>
/// Wie viele Weitergaben eine Delegationskette höchstens haben darf (B8, #206).
/// A→B ist die Tiefe 1. Zyklen fängt die Kette selbst ab; dieser Wert begrenzt die
+66 -4
View File
@@ -127,6 +127,15 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
public event Action<string, string>? OnStepCompleted;
/// <summary>#660: Where prompt and response of every run are kept. <c>null</c> = not at all.</summary>
public Transcripts.IRunTranscriptStore? TranscriptStore { get; init; }
/// <summary>
/// #660: Instance-wide retention in days (0 = never, -1 = forever). A function, so a
/// changed setting applies to the next run without a restart.
/// </summary>
public Func<int> DefaultTranscriptRetention { get; init; } = () => Transcripts.PromptHistorySettings.DefaultRetentionDays;
/// <summary>
/// Wird ausgelöst wenn ein neuer ChatEntry hinzugefügt wird (agentId, role, content, source).
/// Erlaubt der UI, Nachrichten aus Hintergrund-Runs (ToolJobs, AgentComm) live anzuzeigen.
@@ -161,9 +170,11 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
var runId = Guid.NewGuid().ToString("N");
var chain = delegation ?? DelegationChain.Root(agentConfig.AgentId);
var result = await RunCoreAsync(agentConfig, userMessage, instanceId, externalCt, runId, source, chain);
var recorder = CreateRecorder(agentConfig);
var result = await RunCoreAsync(agentConfig, userMessage, instanceId, externalCt, runId, source, chain, recorder);
await RecordUsageAsync(agentConfig, result);
await RecordReceiptAsync(runId, agentConfig, result, source ?? AuditSource.Direct, taskId);
await SaveTranscriptAsync(recorder, runId, agentConfig, result, source ?? AuditSource.Direct, taskId);
return result;
}
@@ -174,7 +185,8 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
CancellationToken externalCt,
string runId,
string? source,
DelegationChain chain)
DelegationChain chain,
Transcripts.RunTranscriptRecorder recorder)
{
var logger = _loggerFactory.CreateLogger($"ClawdDotNet.Core.Engine.{agentConfig.AgentId}");
var loopGuard = new LoopGuard(agentConfig.LoopGuard);
@@ -205,6 +217,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
messages.Add(ChatMessage.System(systemPrompt));
messages.Add(ChatMessage.User(MitZeit(userMessage, source)));
recorder.Begin(systemPrompt, messages[^1].Content?.ToString() ?? userMessage);
string? finalMessage = null;
var truncated = false;
@@ -229,6 +242,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
// sie bezahlt ist statt danach.
if (await _compactor.CompactIfNeededAsync(messages, agentConfig.LoopGuard, agentConfig.Model, ct, tally.Add))
{
recorder.Event("Context compacted");
OnStepCompleted?.Invoke(agentConfig.AgentId, "Context kompaktiert (RunAsync)");
}
@@ -249,6 +263,8 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
}
var choice = response.Choices.FirstOrDefault();
if (choice is not null)
recorder.ModelStep(choice.Message, response.Usage, choice.FinishReason);
if (choice is null)
{
finalMessage = "[No response from model]";
@@ -284,6 +300,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
finally
{
AppendToolResponses(messages, assistantMessage.ToolCalls, toolResults);
recorder.ToolResults(assistantMessage.ToolCalls, toolResults);
}
if (abbruch is not null)
@@ -304,6 +321,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
logger.LogWarning("Leere Antwort von {AgentId} nach {Steps} Schritten - einmal nachgefasst",
agentConfig.AgentId, loopGuard.Steps);
messages.Add(ChatMessage.User(LaufAbschluss.LeerNachfrage));
recorder.EngineNote(LaufAbschluss.LeerNachfrage);
continue;
}
@@ -314,6 +332,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
agentConfig.AgentId);
messages.Add(ChatMessage.Assistant(text));
messages.Add(ChatMessage.User(LaufAbschluss.ErgebnisNachfrage));
recorder.EngineNote(LaufAbschluss.ErgebnisNachfrage);
continue;
}
@@ -468,9 +487,11 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
try
{
var result = await ChatCoreAsync(agentConfig, userMessage, instanceId, runCts.Token, source, runId, chain);
var recorder = CreateRecorder(agentConfig);
var result = await ChatCoreAsync(agentConfig, userMessage, instanceId, runCts.Token, source, runId, chain, recorder);
await RecordUsageAsync(agentConfig, result);
await RecordReceiptAsync(runId, agentConfig, result, source, taskId);
await SaveTranscriptAsync(recorder, runId, agentConfig, result, source ?? AuditSource.Direct, taskId);
return result;
}
finally
@@ -487,7 +508,8 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
CancellationToken runCt,
string? source,
string runId,
DelegationChain chain)
DelegationChain chain,
Transcripts.RunTranscriptRecorder recorder)
{
var logger = _loggerFactory.CreateLogger($"ClawdDotNet.Core.Engine.Chat.{agentConfig.AgentId}");
var loopGuard = new LoopGuard(agentConfig.LoopGuard);
@@ -532,6 +554,9 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
messages.Add(ChatMessage.User(MitZeit(routedMessage, source)));
AddChatEntry(agentConfig.AgentId, "user", userMessage, source);
recorder.Begin(
messages.Count > 0 && messages[0].Role == "system" ? messages[0].Content?.ToString() : null,
messages[^1].Content?.ToString() ?? userMessage);
string? finalMessage = null;
var truncated = false;
@@ -557,6 +582,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
// nächsten Schritt. Genau ein Schritt zu spät, und der teuerste.
if (await _compactor.CompactIfNeededAsync(messages, agentConfig.LoopGuard, agentConfig.Model, ct, tally.Add))
{
recorder.Event("Context compacted");
PersistChatState(agentConfig.AgentId);
OnStepCompleted?.Invoke(agentConfig.AgentId, "Context kompaktiert");
}
@@ -581,6 +607,8 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
}
var choice = response.Choices.FirstOrDefault();
if (choice is not null)
recorder.ModelStep(choice.Message, response.Usage, choice.FinishReason);
if (choice is null)
{
finalMessage = "[No response from model]";
@@ -614,6 +642,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
finally
{
AppendToolResponses(messages, assistantMessage.ToolCalls, toolResults);
recorder.ToolResults(assistantMessage.ToolCalls, toolResults);
}
if (abbruch is not null)
@@ -636,6 +665,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
logger.LogWarning("Leere Antwort von {AgentId} nach {Steps} Schritten - einmal nachgefasst",
agentConfig.AgentId, loopGuard.Steps);
messages.Add(ChatMessage.User(LaufAbschluss.LeerNachfrage));
recorder.EngineNote(LaufAbschluss.LeerNachfrage);
continue;
}
@@ -647,6 +677,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
messages.Add(ChatMessage.Assistant(text));
AddChatEntry(agentConfig.AgentId, "assistant", text, source);
messages.Add(ChatMessage.User(LaufAbschluss.ErgebnisNachfrage));
recorder.EngineNote(LaufAbschluss.ErgebnisNachfrage);
continue;
}
@@ -1746,6 +1777,37 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
/// <summary>
/// Hält den Abschluss-Beleg eines Laufs fest (Receipt). Best effort, wie beim Audit.
/// </summary>
// ─── Prompt history (#660) ───
/// <summary>Retention that applies to this agent: its own setting or the instance default.</summary>
public int TranscriptRetentionFor(AgentConfig agentConfig)
=> Transcripts.PromptHistorySettings.Resolve(agentConfig.PromptHistory?.RetentionDays, DefaultTranscriptRetention());
private Transcripts.RunTranscriptRecorder CreateRecorder(AgentConfig agentConfig)
=> TranscriptStore is not null && TranscriptRetentionFor(agentConfig) != Transcripts.PromptHistorySettings.Never
? new Transcripts.RunTranscriptRecorder(enabled: true, _clock)
: Transcripts.RunTranscriptRecorder.Disabled;
/// <summary>Best effort: a transcript that cannot be written must not fail the run.</summary>
private async Task SaveTranscriptAsync(
Transcripts.RunTranscriptRecorder recorder, string runId, AgentConfig agentConfig,
AgentRunResult result, string source, string? taskId)
{
if (!recorder.Enabled || TranscriptStore is null)
return;
try
{
var transcript = recorder.Build(runId, agentConfig.AgentId, agentConfig.Model, source, taskId, result.Status.ToString());
await TranscriptStore.SaveAsync(transcript, recorder.SystemPrompt, CancellationToken.None);
}
catch (Exception ex)
{
_loggerFactory.CreateLogger("ClawdDotNet.Core.Engine.Transcripts")
.LogWarning(ex, "Transcript of run {RunId} ({AgentId}) could not be stored", runId, agentConfig.AgentId);
}
}
private async Task RecordReceiptAsync(
string runId, AgentConfig agentConfig, AgentRunResult result, string? source, string? taskId)
{
@@ -122,6 +122,20 @@ public sealed class SqliteStagingRepository : IStagingRepository
return results;
}
public async Task<IReadOnlyList<StagedCall>> ListForRunAsync(string runId, CancellationToken ct)
{
await using var conn = await _storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT " + Columns + " FROM StagedCalls WHERE RunId = @runId ORDER BY Id";
cmd.Parameters.AddWithValue("@runId", runId);
var results = new List<StagedCall>();
await using var reader = await cmd.ExecuteReaderAsync(ct);
while (await reader.ReadAsync(ct))
results.Add(Read(reader));
return results;
}
public async Task<int> CountExecutedAsync(string agentId, string tool, string? action, DateTime since, CancellationToken ct)
{
await using var conn = await _storage.OpenConnectionAsync(ct);
@@ -113,4 +113,7 @@ public interface IStagingRepository
/// ausgefuehrt wurde (#641) - Grundlage fuer den Vorschlag "Gleichartiges kuenftig erlauben".
/// </summary>
Task<int> CountExecutedAsync(string agentId, string tool, string? action, DateTime since, CancellationToken ct);
/// <summary>#660: Everything a run proposed for approval - for the run detail view.</summary>
Task<IReadOnlyList<StagedCall>> ListForRunAsync(string runId, CancellationToken ct);
}
@@ -93,6 +93,7 @@ public sealed class StagingService
}
public Task<StagedCall?> GetAsync(long id, CancellationToken ct) => _repo.GetAsync(id, ct);
public Task<int> CountPendingAsync(CancellationToken ct) => _repo.CountPendingAsync(ct);
public Task<IReadOnlyList<StagedCall>> ListForRunAsync(string runId, CancellationToken ct) => _repo.ListForRunAsync(runId, ct);
// ─── Dauerfreigaben (#641) ───
@@ -288,6 +288,37 @@ public sealed class SqliteStorage
CREATE INDEX IF NOT EXISTS IX_Dauerfreigaben_Agent
ON Dauerfreigaben (AgentId, Tool, Operation);
-- #660: Prompt history - prompt and response of every run, kept as long as the
-- retention of the agent allows. System prompts are stored once per text.
CREATE TABLE IF NOT EXISTS RunTranscripts (
RunId TEXT PRIMARY KEY,
AgentId TEXT NOT NULL,
Model TEXT NOT NULL DEFAULT '',
Source TEXT NOT NULL DEFAULT '',
TaskId TEXT NULL,
StartedAt TEXT NOT NULL,
CompletedAt TEXT NULL,
Status TEXT NOT NULL DEFAULT '',
SystemPromptHash TEXT NULL,
Prompt TEXT NOT NULL DEFAULT '',
Response TEXT NOT NULL DEFAULT '',
EntriesJson TEXT NOT NULL DEFAULT '[]',
SizeChars INTEGER NOT NULL DEFAULT 0,
Truncated INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS IX_RunTranscripts_Agent
ON RunTranscripts (AgentId, StartedAt);
CREATE INDEX IF NOT EXISTS IX_RunTranscripts_Started
ON RunTranscripts (StartedAt DESC);
CREATE TABLE IF NOT EXISTS SystemPrompts (
Hash TEXT PRIMARY KEY,
Text TEXT NOT NULL,
FirstSeenAt TEXT NOT NULL
);
""";
cmd.ExecuteNonQuery();
@@ -0,0 +1,49 @@
using System.Text.Json.Serialization;
namespace ClawdDotNet.Core.Transcripts;
/// <summary>
/// How long run transcripts (prompt and response of every run) are kept (#660).
///
/// <para>One number of days covers the whole range the user asked for:
/// <see cref="Never"/> (0) records nothing, <see cref="Forever"/> (-1) keeps everything,
/// any positive value keeps that many days.</para>
/// </summary>
public sealed class PromptHistorySettings
{
public const int Never = 0;
public const int Forever = -1;
public const int DefaultRetentionDays = 14;
/// <summary>Instance default for all agents without their own setting.</summary>
[JsonPropertyName("retentionDays")]
public int RetentionDays { get; set; } = DefaultRetentionDays;
/// <summary>
/// The retention that applies to an agent: its own value if set, otherwise the
/// instance default. Values below <see cref="Forever"/> are treated as "forever".
/// </summary>
public static int Resolve(int? agentRetentionDays, int instanceRetentionDays)
=> Normalize(agentRetentionDays ?? instanceRetentionDays);
public static int Normalize(int days) => days < Forever ? Forever : days;
/// <summary>Human readable form for logs and the UI fallback.</summary>
public static string Describe(int days) => Normalize(days) switch
{
Never => "never recorded",
Forever => "kept forever",
var d => $"kept for {d} day(s)"
};
}
/// <summary>
/// Per-agent override (#660): debug a single agent over a long period, or switch
/// recording off completely for an agent that handles sensitive secrets.
/// </summary>
public sealed class AgentPromptHistorySettings
{
/// <summary><c>null</c> = use the instance default.</summary>
[JsonPropertyName("retentionDays")]
public int? RetentionDays { get; set; }
}
@@ -0,0 +1,103 @@
namespace ClawdDotNet.Core.Transcripts;
/// <summary>What a transcript entry is.</summary>
public static class TranscriptEntryKind
{
/// <summary>The task or message the run started with.</summary>
public const string User = "user";
/// <summary>A follow-up the engine itself sent (empty answer, missing result line).</summary>
public const string EngineNote = "engine";
/// <summary>One model response: text and/or tool calls, with the tokens of this step.</summary>
public const string Assistant = "assistant";
/// <summary>The result of one tool call.</summary>
public const string ToolResult = "tool_result";
/// <summary>Something that happened to the context, e.g. compaction.</summary>
public const string Event = "event";
}
/// <summary>One tool call as the model issued it.</summary>
public sealed record TranscriptToolCall(string Id, string Name, string Arguments);
/// <summary>
/// One entry of a run transcript. Texts are stored as the model saw or wrote them, after
/// secret scrubbing and clipping (<see cref="Clipped"/>).
/// </summary>
public sealed record TranscriptEntry
{
public string Kind { get; init; } = "";
public DateTime At { get; init; }
public string? Text { get; init; }
public IReadOnlyList<TranscriptToolCall>? ToolCalls { get; init; }
/// <summary>For <see cref="TranscriptEntryKind.ToolResult"/>: which call and tool.</summary>
public string? ToolCallId { get; init; }
public string? ToolName { get; init; }
/// <summary>For <see cref="TranscriptEntryKind.Assistant"/>: usage of this single step.</summary>
public int? PromptTokens { get; init; }
public int? CompletionTokens { get; init; }
public int? CachedTokens { get; init; }
public decimal? CostUsd { get; init; }
public string? FinishReason { get; init; }
/// <summary>The text was shortened to stay within the limits.</summary>
public bool Clipped { get; init; }
}
/// <summary>
/// Prompt and response of one run (#660), joined to receipts and audit via
/// <see cref="RunId"/>. The system prompt is stored once per distinct text and referenced
/// by <see cref="SystemPromptHash"/>.
/// </summary>
public sealed record RunTranscript
{
public string RunId { get; init; } = "";
public string AgentId { get; init; } = "";
public string Model { get; init; } = "";
public string Source { get; init; } = "";
public string? TaskId { get; init; }
public DateTime StartedAt { get; init; }
public DateTime? CompletedAt { get; init; }
public string Status { get; init; } = "";
public string? SystemPromptHash { get; init; }
public IReadOnlyList<TranscriptEntry> Entries { get; init; } = [];
/// <summary>Characters stored, after clipping.</summary>
public int SizeChars { get; init; }
/// <summary>The run exceeded the size limit; later entries are shortened or omitted.</summary>
public bool Truncated { get; init; }
}
/// <summary>A transcript without its entries — for lists.</summary>
public sealed record RunTranscriptSummary(
string RunId,
string AgentId,
string Model,
string Source,
string? TaskId,
DateTime StartedAt,
DateTime? CompletedAt,
string Status,
string Prompt,
string Response,
int SizeChars,
bool Truncated);
/// <summary>Filter for <see cref="IRunTranscriptStore.ListAsync"/>.</summary>
public sealed record TranscriptQuery
{
public string? AgentId { get; init; }
public DateTime? From { get; init; }
public DateTime? To { get; init; }
/// <summary>Full text over the whole transcript.</summary>
public string? Search { get; init; }
public int Limit { get; init; } = 50;
public int Offset { get; init; }
}
@@ -0,0 +1,212 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using ClawdDotNet.Core.Api.Models;
namespace ClawdDotNet.Core.Transcripts;
/// <summary>
/// Collects the transcript of one run while it happens (#660). The engine calls it at the
/// few points where something enters or leaves the conversation; nothing here talks to
/// the model or the database.
///
/// <para>Recording is bounded: tool results are clipped to <see cref="MaxToolResultChars"/>,
/// any single text to <see cref="MaxTextChars"/>, and the whole transcript to
/// <see cref="MaxTotalChars"/>. Beyond that only the entry heads are kept (who called what,
/// how many tokens), which is what an analysis of a runaway run needs anyway.</para>
///
/// <para>A disabled recorder (<see cref="Disabled"/>) accepts every call and keeps
/// nothing, so the engine never has to branch.</para>
/// </summary>
public sealed class RunTranscriptRecorder
{
public const int MaxToolResultChars = 4_000;
public const int MaxTextChars = 32_000;
public const int MaxTotalChars = 256_000;
private readonly Lock _lock = new();
private readonly List<TranscriptEntry> _entries = [];
private readonly TimeProvider _clock;
private int _size;
private bool _truncated;
public static RunTranscriptRecorder Disabled { get; } = new(enabled: false);
public bool Enabled { get; }
public string? SystemPrompt { get; private set; }
public DateTime StartedAt { get; }
public RunTranscriptRecorder(bool enabled = true, TimeProvider? clock = null)
{
Enabled = enabled;
_clock = clock ?? TimeProvider.System;
StartedAt = _clock.GetUtcNow().UtcDateTime;
}
// ─── Engine hooks ───
public void Begin(string? systemPrompt, string userMessage)
{
if (!Enabled) return;
SystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? null : SecretScrubber.Scrub(systemPrompt);
Add(TranscriptEntryKind.User, userMessage, MaxTextChars);
}
/// <summary>A message the engine sent on its own (follow-up after an empty answer etc.).</summary>
public void EngineNote(string text)
{
if (!Enabled) return;
Add(TranscriptEntryKind.EngineNote, text, MaxTextChars);
}
public void Event(string text)
{
if (!Enabled) return;
Add(TranscriptEntryKind.Event, text, MaxTextChars);
}
/// <summary>One model response with the usage of exactly this request.</summary>
public void ModelStep(ChatMessage response, Usage? usage, string? finishReason)
{
if (!Enabled) return;
var calls = response.ToolCalls?
.Select(c => new TranscriptToolCall(
c.Id, c.Function.Name, Clip(SecretScrubber.Scrub(c.Function.Arguments ?? ""), MaxTextChars, out _)))
.ToList();
var (text, clipped) = Prepare(response.Content, MaxTextChars);
lock (_lock)
{
var groesse = (text?.Length ?? 0) + (calls?.Sum(c => c.Arguments.Length + c.Name.Length) ?? 0);
var ueber = _size + groesse > MaxTotalChars;
if (ueber) _truncated = true;
_entries.Add(new TranscriptEntry
{
Kind = TranscriptEntryKind.Assistant,
At = _clock.GetUtcNow().UtcDateTime,
Text = ueber ? null : text,
ToolCalls = ueber ? calls?.Select(c => c with { Arguments = "" }).ToList() : calls,
PromptTokens = usage?.PromptTokens,
CompletionTokens = usage?.CompletionTokens,
CachedTokens = usage?.CachedTokens,
CostUsd = usage?.Cost,
FinishReason = finishReason,
Clipped = clipped || ueber
});
if (!ueber) _size += groesse;
}
}
public void ToolResults(IReadOnlyList<ToolCall> calls, IReadOnlyList<string?> results)
{
if (!Enabled) return;
for (var i = 0; i < calls.Count; i++)
{
Add(TranscriptEntryKind.ToolResult, results.ElementAtOrDefault(i) ?? "(no result)", MaxToolResultChars,
calls[i].Id, calls[i].Function.Name);
}
}
// ─── Result ───
public RunTranscript Build(string runId, string agentId, string model, string source, string? taskId, string status)
{
lock (_lock)
{
return new RunTranscript
{
RunId = runId,
AgentId = agentId,
Model = model,
Source = source,
TaskId = taskId,
StartedAt = StartedAt,
CompletedAt = _clock.GetUtcNow().UtcDateTime,
Status = status,
SystemPromptHash = SystemPrompt is null ? null : Hash(SystemPrompt),
Entries = _entries.ToList(),
SizeChars = _size,
Truncated = _truncated
};
}
}
public static string Hash(string text)
=> Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text)))[..32].ToLowerInvariant();
// ─── Helpers ───
private void Add(string kind, string? text, int max, string? toolCallId = null, string? toolName = null)
{
var (prepared, clipped) = Prepare(text, max);
lock (_lock)
{
var ueber = _size + (prepared?.Length ?? 0) > MaxTotalChars;
if (ueber) _truncated = true;
_entries.Add(new TranscriptEntry
{
Kind = kind,
At = _clock.GetUtcNow().UtcDateTime,
Text = ueber ? null : prepared,
ToolCallId = toolCallId,
ToolName = toolName,
Clipped = clipped || ueber
});
if (!ueber) _size += prepared?.Length ?? 0;
}
}
private static (string? Text, bool Clipped) Prepare(string? text, int max)
{
if (string.IsNullOrEmpty(text)) return (text, false);
var clipped = Clip(SecretScrubber.Scrub(text), max, out var wasClipped);
return (clipped, wasClipped);
}
/// <summary>Keeps start and end — the end of a tool result is often the interesting part.</summary>
internal static string Clip(string text, int max, out bool clipped)
{
clipped = text.Length > max;
if (!clipped) return text;
var head = max * 3 / 4;
var tail = max - head;
return text[..head] + $"\n[… {text.Length - max} characters omitted …]\n" + text[^tail..];
}
}
/// <summary>
/// Removes the kinds of secrets that can end up in prompts or tool results before a
/// transcript is stored. Deliberately conservative: it targets recognisable secret
/// formats, not arbitrary text.
/// </summary>
public static partial class SecretScrubber
{
public const string Mask = "«redacted»";
public static string Scrub(string text)
{
if (string.IsNullOrEmpty(text)) return text;
text = ApiKey().Replace(text, Mask);
text = Bearer().Replace(text, "Bearer " + Mask);
text = PasswordAssignment().Replace(text, m => m.Groups["key"].Value + m.Groups["sep"].Value + Mask);
text = LicenseKey().Replace(text, Mask);
return text;
}
[GeneratedRegex(@"\b(?:sk-or-v1-|sk-or-|sk-ant-|sk-proj-|sk-|ghp_|gho_|github_pat_|xox[baprs]-)[A-Za-z0-9_\-]{16,}")]
private static partial Regex ApiKey();
[GeneratedRegex(@"Bearer\s+[A-Za-z0-9._~+/\-]{16,}=*", RegexOptions.IgnoreCase)]
private static partial Regex Bearer();
[GeneratedRegex(@"(?<key>\b(?:password|passwort|pwd|secret|api[_-]?key|token)\b)(?<sep>\s*[=:]\s*""?)[^;\s""',}]+", RegexOptions.IgnoreCase)]
private static partial Regex PasswordAssignment();
[GeneratedRegex(@"\b[A-Z0-9]{5}(?:-[A-Z0-9]{5}){4}\b")]
private static partial Regex LicenseKey();
}
@@ -0,0 +1,211 @@
using System.Globalization;
using System.Text.Json;
using ClawdDotNet.Core.Storage;
using Microsoft.Data.Sqlite;
namespace ClawdDotNet.Core.Transcripts;
public interface IRunTranscriptStore
{
Task SaveAsync(RunTranscript transcript, string? systemPrompt, CancellationToken ct);
Task<RunTranscript?> GetAsync(string runId, CancellationToken ct);
Task<string?> GetSystemPromptAsync(string hash, CancellationToken ct);
Task<(IReadOnlyList<RunTranscriptSummary> Items, int Total)> ListAsync(TranscriptQuery query, CancellationToken ct);
/// <summary>
/// Deletes what is older than each agent's retention allows. <paramref name="retentionFor"/>
/// returns the resolved retention in days for an agent id (0 = delete all, -1 = keep).
/// </summary>
Task<int> PurgeAsync(Func<string, int> retentionFor, DateTime now, CancellationToken ct);
}
/// <summary>
/// Run transcripts in the instance database (#660). Entries are one JSON column per run:
/// a transcript is always read and written as a whole, and the full-text search over
/// it is a plain LIKE — enough for the volumes of a single instance.
/// </summary>
public sealed class SqliteRunTranscriptStore(SqliteStorage storage) : IRunTranscriptStore
{
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web)
{
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
// Unescaped, so the LIKE search finds "Grüße" as typed. Never rendered as HTML.
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping
};
private const string SummaryColumns =
"RunId, AgentId, Model, Source, TaskId, StartedAt, CompletedAt, Status, Prompt, Response, SizeChars, Truncated";
public Task SaveAsync(RunTranscript t, string? systemPrompt, CancellationToken ct)
=> storage.WriteAsync(async conn =>
{
if (t.SystemPromptHash is not null && systemPrompt is not null)
{
using var sp = conn.CreateCommand();
sp.CommandText = """
INSERT OR IGNORE INTO SystemPrompts (Hash, Text, FirstSeenAt) VALUES (@hash, @text, @at)
""";
sp.Parameters.AddWithValue("@hash", t.SystemPromptHash);
sp.Parameters.AddWithValue("@text", systemPrompt);
sp.Parameters.AddWithValue("@at", Format(t.StartedAt));
await sp.ExecuteNonQueryAsync(ct);
}
using var cmd = conn.CreateCommand();
cmd.CommandText = """
INSERT OR REPLACE INTO RunTranscripts
(RunId, AgentId, Model, Source, TaskId, StartedAt, CompletedAt, Status,
SystemPromptHash, Prompt, Response, EntriesJson, SizeChars, Truncated)
VALUES
(@runId, @agent, @model, @source, @task, @started, @completed, @status,
@hash, @prompt, @response, @entries, @size, @truncated)
""";
cmd.Parameters.AddWithValue("@runId", t.RunId);
cmd.Parameters.AddWithValue("@agent", t.AgentId);
cmd.Parameters.AddWithValue("@model", t.Model);
cmd.Parameters.AddWithValue("@source", t.Source);
cmd.Parameters.AddWithValue("@task", (object?)t.TaskId ?? DBNull.Value);
cmd.Parameters.AddWithValue("@started", Format(t.StartedAt));
cmd.Parameters.AddWithValue("@completed", t.CompletedAt is { } c ? Format(c) : DBNull.Value);
cmd.Parameters.AddWithValue("@status", t.Status);
cmd.Parameters.AddWithValue("@hash", (object?)t.SystemPromptHash ?? DBNull.Value);
cmd.Parameters.AddWithValue("@prompt", Snippet(t.Entries.FirstOrDefault(e => e.Kind == TranscriptEntryKind.User)?.Text));
cmd.Parameters.AddWithValue("@response", Snippet(t.Entries.LastOrDefault(e => e.Kind == TranscriptEntryKind.Assistant && !string.IsNullOrWhiteSpace(e.Text))?.Text));
cmd.Parameters.AddWithValue("@entries", JsonSerializer.Serialize(t.Entries, Json));
cmd.Parameters.AddWithValue("@size", t.SizeChars);
cmd.Parameters.AddWithValue("@truncated", t.Truncated ? 1 : 0);
await cmd.ExecuteNonQueryAsync(ct);
return true;
}, ct);
public async Task<RunTranscript?> GetAsync(string runId, CancellationToken ct)
{
await using var conn = await storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = """
SELECT RunId, AgentId, Model, Source, TaskId, StartedAt, CompletedAt, Status,
SystemPromptHash, EntriesJson, SizeChars, Truncated
FROM RunTranscripts WHERE RunId = @runId
""";
cmd.Parameters.AddWithValue("@runId", runId);
await using var r = await cmd.ExecuteReaderAsync(ct);
if (!await r.ReadAsync(ct)) return null;
return new RunTranscript
{
RunId = r.GetString(0),
AgentId = r.GetString(1),
Model = r.GetString(2),
Source = r.GetString(3),
TaskId = r.IsDBNull(4) ? null : r.GetString(4),
StartedAt = Parse(r.GetString(5)),
CompletedAt = r.IsDBNull(6) ? null : Parse(r.GetString(6)),
Status = r.GetString(7),
SystemPromptHash = r.IsDBNull(8) ? null : r.GetString(8),
Entries = JsonSerializer.Deserialize<List<TranscriptEntry>>(r.GetString(9), Json) ?? [],
SizeChars = r.GetInt32(10),
Truncated = r.GetInt32(11) != 0
};
}
public async Task<string?> GetSystemPromptAsync(string hash, CancellationToken ct)
{
await using var conn = await storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT Text FROM SystemPrompts WHERE Hash = @hash";
cmd.Parameters.AddWithValue("@hash", hash);
return await cmd.ExecuteScalarAsync(ct) as string;
}
public async Task<(IReadOnlyList<RunTranscriptSummary> Items, int Total)> ListAsync(TranscriptQuery q, CancellationToken ct)
{
await using var conn = await storage.OpenConnectionAsync(ct);
var where = new List<string>();
void Bind(SqliteCommand c)
{
if (!string.IsNullOrWhiteSpace(q.AgentId)) c.Parameters.AddWithValue("@agent", q.AgentId);
if (q.From is { } f) c.Parameters.AddWithValue("@from", Format(f));
if (q.To is { } t) c.Parameters.AddWithValue("@to", Format(t));
if (!string.IsNullOrWhiteSpace(q.Search)) c.Parameters.AddWithValue("@q", "%" + Escape(q.Search.Trim()) + "%");
}
if (!string.IsNullOrWhiteSpace(q.AgentId)) where.Add("AgentId = @agent");
if (q.From is not null) where.Add("StartedAt >= @from");
if (q.To is not null) where.Add("StartedAt < @to");
if (!string.IsNullOrWhiteSpace(q.Search)) where.Add("EntriesJson LIKE @q ESCAPE '\\'");
var filter = where.Count == 0 ? "" : " WHERE " + string.Join(" AND ", where);
using var count = conn.CreateCommand();
count.CommandText = "SELECT COUNT(*) FROM RunTranscripts" + filter;
Bind(count);
var total = Convert.ToInt32(await count.ExecuteScalarAsync(ct));
using var cmd = conn.CreateCommand();
cmd.CommandText = $"SELECT {SummaryColumns} FROM RunTranscripts{filter} ORDER BY StartedAt DESC LIMIT @limit OFFSET @offset";
Bind(cmd);
cmd.Parameters.AddWithValue("@limit", Math.Clamp(q.Limit, 1, 500));
cmd.Parameters.AddWithValue("@offset", Math.Max(0, q.Offset));
var items = new List<RunTranscriptSummary>();
await using var r = await cmd.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
{
items.Add(new RunTranscriptSummary(
r.GetString(0), r.GetString(1), r.GetString(2), r.GetString(3),
r.IsDBNull(4) ? null : r.GetString(4),
Parse(r.GetString(5)),
r.IsDBNull(6) ? null : Parse(r.GetString(6)),
r.GetString(7), r.GetString(8), r.GetString(9), r.GetInt32(10), r.GetInt32(11) != 0));
}
return (items, total);
}
public Task<int> PurgeAsync(Func<string, int> retentionFor, DateTime now, CancellationToken ct)
=> storage.WriteAsync(async conn =>
{
var agents = new List<string>();
using (var list = conn.CreateCommand())
{
list.CommandText = "SELECT DISTINCT AgentId FROM RunTranscripts";
await using var r = await list.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct)) agents.Add(r.GetString(0));
}
var deleted = 0;
foreach (var agent in agents)
{
var days = PromptHistorySettings.Normalize(retentionFor(agent));
if (days == PromptHistorySettings.Forever) continue;
using var del = conn.CreateCommand();
del.CommandText = "DELETE FROM RunTranscripts WHERE AgentId = @agent AND StartedAt < @cutoff";
del.Parameters.AddWithValue("@agent", agent);
// Never (0): everything goes, including what was recorded before the switch.
del.Parameters.AddWithValue("@cutoff", Format(days == PromptHistorySettings.Never ? DateTime.MaxValue : now.AddDays(-days)));
deleted += await del.ExecuteNonQueryAsync(ct);
}
using var orphans = conn.CreateCommand();
orphans.CommandText = """
DELETE FROM SystemPrompts
WHERE Hash NOT IN (SELECT SystemPromptHash FROM RunTranscripts WHERE SystemPromptHash IS NOT NULL)
""";
await orphans.ExecuteNonQueryAsync(ct);
return deleted;
}, ct);
private static string Snippet(string? text)
{
if (string.IsNullOrWhiteSpace(text)) return "";
var t = text.Trim();
return t.Length <= 300 ? t : t[..300] + "…";
}
private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("%", "\\%").Replace("_", "\\_");
private static string Format(DateTime value)
=> (value == DateTime.MaxValue ? value : value.ToUniversalTime()).ToString("O", CultureInfo.InvariantCulture);
private static DateTime Parse(string value)
=> DateTime.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToUniversalTime();
}
@@ -0,0 +1,58 @@
using Microsoft.Extensions.Logging;
namespace ClawdDotNet.Core.Transcripts;
/// <summary>
/// Enforces the prompt history retention (#660): once at start and then every hour,
/// transcripts older than their agent's retention are deleted. An agent switched to
/// "never" loses everything recorded before the switch — that is the point of the switch.
/// </summary>
public sealed class TranscriptJanitor(
IRunTranscriptStore store,
Func<string, int> retentionFor,
ILogger logger,
TimeProvider? clock = null,
TimeSpan? interval = null) : IAsyncDisposable
{
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
private readonly CancellationTokenSource _cts = new();
private Task? _loop;
public void Start() => _loop ??= RunAsync(_cts.Token);
public async Task<int> PurgeOnceAsync(CancellationToken ct)
{
var deleted = await store.PurgeAsync(retentionFor, _clock.GetUtcNow().UtcDateTime, ct);
if (deleted > 0)
logger.LogInformation("Prompt history: {Count} transcript(s) past their retention deleted", deleted);
return deleted;
}
private async Task RunAsync(CancellationToken ct)
{
using var timer = new PeriodicTimer(interval ?? TimeSpan.FromHours(1), _clock);
do
{
try
{
await PurgeOnceAsync(ct);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Prompt history cleanup failed");
}
}
while (await timer.WaitForNextTickAsync(ct));
}
public async ValueTask DisposeAsync()
{
await _cts.CancelAsync();
if (_loop is not null)
{
try { await _loop; }
catch (OperationCanceledException) { }
}
_cts.Dispose();
}
}