Prompt history: every run records its transcript (task, each model step with tokens and cost, tool calls and results, engine follow-ups) - bounded in size, scrubbed of recognisable secrets, system prompts stored once per text. Retention in days per instance (default 14) and per agent (0 = never, -1 = forever); an hourly janitor deletes what is past it, "never" also purges. Usage analysis over run receipts: filter by period, agent, model, event, status, cost and free text; sort by every column; totals over the whole filter; hourly or daily series grouped by agent, model or event; markers for budget denials and model changes; run detail with receipt, transcript, tool calls and approvals. WebUI: Analysis > Token usage > Runs / Timeline, run detail, Prompt History with retention settings. New code, comments and texts in English; texts come from language packs (i18n/en.json, de.json) via modules/i18n.js. CLAUDE.md records English as the main language. Version 0.1.12. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Richard
@@ -0,0 +1,49 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace ClawdDotNet.Core.Transcripts;
|
||||
|
||||
/// <summary>
|
||||
/// How long run transcripts (prompt and response of every run) are kept (#660).
|
||||
///
|
||||
/// <para>One number of days covers the whole range the user asked for:
|
||||
/// <see cref="Never"/> (0) records nothing, <see cref="Forever"/> (-1) keeps everything,
|
||||
/// any positive value keeps that many days.</para>
|
||||
/// </summary>
|
||||
public sealed class PromptHistorySettings
|
||||
{
|
||||
public const int Never = 0;
|
||||
public const int Forever = -1;
|
||||
public const int DefaultRetentionDays = 14;
|
||||
|
||||
/// <summary>Instance default for all agents without their own setting.</summary>
|
||||
[JsonPropertyName("retentionDays")]
|
||||
public int RetentionDays { get; set; } = DefaultRetentionDays;
|
||||
|
||||
/// <summary>
|
||||
/// The retention that applies to an agent: its own value if set, otherwise the
|
||||
/// instance default. Values below <see cref="Forever"/> are treated as "forever".
|
||||
/// </summary>
|
||||
public static int Resolve(int? agentRetentionDays, int instanceRetentionDays)
|
||||
=> Normalize(agentRetentionDays ?? instanceRetentionDays);
|
||||
|
||||
public static int Normalize(int days) => days < Forever ? Forever : days;
|
||||
|
||||
/// <summary>Human readable form for logs and the UI fallback.</summary>
|
||||
public static string Describe(int days) => Normalize(days) switch
|
||||
{
|
||||
Never => "never recorded",
|
||||
Forever => "kept forever",
|
||||
var d => $"kept for {d} day(s)"
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Per-agent override (#660): debug a single agent over a long period, or switch
|
||||
/// recording off completely for an agent that handles sensitive secrets.
|
||||
/// </summary>
|
||||
public sealed class AgentPromptHistorySettings
|
||||
{
|
||||
/// <summary><c>null</c> = use the instance default.</summary>
|
||||
[JsonPropertyName("retentionDays")]
|
||||
public int? RetentionDays { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
namespace ClawdDotNet.Core.Transcripts;
|
||||
|
||||
/// <summary>What a transcript entry is.</summary>
|
||||
public static class TranscriptEntryKind
|
||||
{
|
||||
/// <summary>The task or message the run started with.</summary>
|
||||
public const string User = "user";
|
||||
|
||||
/// <summary>A follow-up the engine itself sent (empty answer, missing result line).</summary>
|
||||
public const string EngineNote = "engine";
|
||||
|
||||
/// <summary>One model response: text and/or tool calls, with the tokens of this step.</summary>
|
||||
public const string Assistant = "assistant";
|
||||
|
||||
/// <summary>The result of one tool call.</summary>
|
||||
public const string ToolResult = "tool_result";
|
||||
|
||||
/// <summary>Something that happened to the context, e.g. compaction.</summary>
|
||||
public const string Event = "event";
|
||||
}
|
||||
|
||||
/// <summary>One tool call as the model issued it.</summary>
|
||||
public sealed record TranscriptToolCall(string Id, string Name, string Arguments);
|
||||
|
||||
/// <summary>
|
||||
/// One entry of a run transcript. Texts are stored as the model saw or wrote them, after
|
||||
/// secret scrubbing and clipping (<see cref="Clipped"/>).
|
||||
/// </summary>
|
||||
public sealed record TranscriptEntry
|
||||
{
|
||||
public string Kind { get; init; } = "";
|
||||
public DateTime At { get; init; }
|
||||
public string? Text { get; init; }
|
||||
public IReadOnlyList<TranscriptToolCall>? ToolCalls { get; init; }
|
||||
|
||||
/// <summary>For <see cref="TranscriptEntryKind.ToolResult"/>: which call and tool.</summary>
|
||||
public string? ToolCallId { get; init; }
|
||||
public string? ToolName { get; init; }
|
||||
|
||||
/// <summary>For <see cref="TranscriptEntryKind.Assistant"/>: usage of this single step.</summary>
|
||||
public int? PromptTokens { get; init; }
|
||||
public int? CompletionTokens { get; init; }
|
||||
public int? CachedTokens { get; init; }
|
||||
public decimal? CostUsd { get; init; }
|
||||
public string? FinishReason { get; init; }
|
||||
|
||||
/// <summary>The text was shortened to stay within the limits.</summary>
|
||||
public bool Clipped { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prompt and response of one run (#660), joined to receipts and audit via
|
||||
/// <see cref="RunId"/>. The system prompt is stored once per distinct text and referenced
|
||||
/// by <see cref="SystemPromptHash"/>.
|
||||
/// </summary>
|
||||
public sealed record RunTranscript
|
||||
{
|
||||
public string RunId { get; init; } = "";
|
||||
public string AgentId { get; init; } = "";
|
||||
public string Model { get; init; } = "";
|
||||
public string Source { get; init; } = "";
|
||||
public string? TaskId { get; init; }
|
||||
public DateTime StartedAt { get; init; }
|
||||
public DateTime? CompletedAt { get; init; }
|
||||
public string Status { get; init; } = "";
|
||||
public string? SystemPromptHash { get; init; }
|
||||
public IReadOnlyList<TranscriptEntry> Entries { get; init; } = [];
|
||||
|
||||
/// <summary>Characters stored, after clipping.</summary>
|
||||
public int SizeChars { get; init; }
|
||||
|
||||
/// <summary>The run exceeded the size limit; later entries are shortened or omitted.</summary>
|
||||
public bool Truncated { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>A transcript without its entries — for lists.</summary>
|
||||
public sealed record RunTranscriptSummary(
|
||||
string RunId,
|
||||
string AgentId,
|
||||
string Model,
|
||||
string Source,
|
||||
string? TaskId,
|
||||
DateTime StartedAt,
|
||||
DateTime? CompletedAt,
|
||||
string Status,
|
||||
string Prompt,
|
||||
string Response,
|
||||
int SizeChars,
|
||||
bool Truncated);
|
||||
|
||||
/// <summary>Filter for <see cref="IRunTranscriptStore.ListAsync"/>.</summary>
|
||||
public sealed record TranscriptQuery
|
||||
{
|
||||
public string? AgentId { get; init; }
|
||||
public DateTime? From { get; init; }
|
||||
public DateTime? To { get; init; }
|
||||
|
||||
/// <summary>Full text over the whole transcript.</summary>
|
||||
public string? Search { get; init; }
|
||||
|
||||
public int Limit { get; init; } = 50;
|
||||
public int Offset { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using ClawdDotNet.Core.Api.Models;
|
||||
|
||||
namespace ClawdDotNet.Core.Transcripts;
|
||||
|
||||
/// <summary>
|
||||
/// Collects the transcript of one run while it happens (#660). The engine calls it at the
|
||||
/// few points where something enters or leaves the conversation; nothing here talks to
|
||||
/// the model or the database.
|
||||
///
|
||||
/// <para>Recording is bounded: tool results are clipped to <see cref="MaxToolResultChars"/>,
|
||||
/// any single text to <see cref="MaxTextChars"/>, and the whole transcript to
|
||||
/// <see cref="MaxTotalChars"/>. Beyond that only the entry heads are kept (who called what,
|
||||
/// how many tokens), which is what an analysis of a runaway run needs anyway.</para>
|
||||
///
|
||||
/// <para>A disabled recorder (<see cref="Disabled"/>) accepts every call and keeps
|
||||
/// nothing, so the engine never has to branch.</para>
|
||||
/// </summary>
|
||||
public sealed class RunTranscriptRecorder
|
||||
{
|
||||
public const int MaxToolResultChars = 4_000;
|
||||
public const int MaxTextChars = 32_000;
|
||||
public const int MaxTotalChars = 256_000;
|
||||
|
||||
private readonly Lock _lock = new();
|
||||
private readonly List<TranscriptEntry> _entries = [];
|
||||
private readonly TimeProvider _clock;
|
||||
private int _size;
|
||||
private bool _truncated;
|
||||
|
||||
public static RunTranscriptRecorder Disabled { get; } = new(enabled: false);
|
||||
|
||||
public bool Enabled { get; }
|
||||
public string? SystemPrompt { get; private set; }
|
||||
public DateTime StartedAt { get; }
|
||||
|
||||
public RunTranscriptRecorder(bool enabled = true, TimeProvider? clock = null)
|
||||
{
|
||||
Enabled = enabled;
|
||||
_clock = clock ?? TimeProvider.System;
|
||||
StartedAt = _clock.GetUtcNow().UtcDateTime;
|
||||
}
|
||||
|
||||
// ─── Engine hooks ───
|
||||
|
||||
public void Begin(string? systemPrompt, string userMessage)
|
||||
{
|
||||
if (!Enabled) return;
|
||||
SystemPrompt = string.IsNullOrWhiteSpace(systemPrompt) ? null : SecretScrubber.Scrub(systemPrompt);
|
||||
Add(TranscriptEntryKind.User, userMessage, MaxTextChars);
|
||||
}
|
||||
|
||||
/// <summary>A message the engine sent on its own (follow-up after an empty answer etc.).</summary>
|
||||
public void EngineNote(string text)
|
||||
{
|
||||
if (!Enabled) return;
|
||||
Add(TranscriptEntryKind.EngineNote, text, MaxTextChars);
|
||||
}
|
||||
|
||||
public void Event(string text)
|
||||
{
|
||||
if (!Enabled) return;
|
||||
Add(TranscriptEntryKind.Event, text, MaxTextChars);
|
||||
}
|
||||
|
||||
/// <summary>One model response with the usage of exactly this request.</summary>
|
||||
public void ModelStep(ChatMessage response, Usage? usage, string? finishReason)
|
||||
{
|
||||
if (!Enabled) return;
|
||||
|
||||
var calls = response.ToolCalls?
|
||||
.Select(c => new TranscriptToolCall(
|
||||
c.Id, c.Function.Name, Clip(SecretScrubber.Scrub(c.Function.Arguments ?? ""), MaxTextChars, out _)))
|
||||
.ToList();
|
||||
|
||||
var (text, clipped) = Prepare(response.Content, MaxTextChars);
|
||||
|
||||
lock (_lock)
|
||||
{
|
||||
var groesse = (text?.Length ?? 0) + (calls?.Sum(c => c.Arguments.Length + c.Name.Length) ?? 0);
|
||||
var ueber = _size + groesse > MaxTotalChars;
|
||||
if (ueber) _truncated = true;
|
||||
|
||||
_entries.Add(new TranscriptEntry
|
||||
{
|
||||
Kind = TranscriptEntryKind.Assistant,
|
||||
At = _clock.GetUtcNow().UtcDateTime,
|
||||
Text = ueber ? null : text,
|
||||
ToolCalls = ueber ? calls?.Select(c => c with { Arguments = "" }).ToList() : calls,
|
||||
PromptTokens = usage?.PromptTokens,
|
||||
CompletionTokens = usage?.CompletionTokens,
|
||||
CachedTokens = usage?.CachedTokens,
|
||||
CostUsd = usage?.Cost,
|
||||
FinishReason = finishReason,
|
||||
Clipped = clipped || ueber
|
||||
});
|
||||
if (!ueber) _size += groesse;
|
||||
}
|
||||
}
|
||||
|
||||
public void ToolResults(IReadOnlyList<ToolCall> calls, IReadOnlyList<string?> results)
|
||||
{
|
||||
if (!Enabled) return;
|
||||
for (var i = 0; i < calls.Count; i++)
|
||||
{
|
||||
Add(TranscriptEntryKind.ToolResult, results.ElementAtOrDefault(i) ?? "(no result)", MaxToolResultChars,
|
||||
calls[i].Id, calls[i].Function.Name);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Result ───
|
||||
|
||||
public RunTranscript Build(string runId, string agentId, string model, string source, string? taskId, string status)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
return new RunTranscript
|
||||
{
|
||||
RunId = runId,
|
||||
AgentId = agentId,
|
||||
Model = model,
|
||||
Source = source,
|
||||
TaskId = taskId,
|
||||
StartedAt = StartedAt,
|
||||
CompletedAt = _clock.GetUtcNow().UtcDateTime,
|
||||
Status = status,
|
||||
SystemPromptHash = SystemPrompt is null ? null : Hash(SystemPrompt),
|
||||
Entries = _entries.ToList(),
|
||||
SizeChars = _size,
|
||||
Truncated = _truncated
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public static string Hash(string text)
|
||||
=> Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(text)))[..32].ToLowerInvariant();
|
||||
|
||||
// ─── Helpers ───
|
||||
|
||||
private void Add(string kind, string? text, int max, string? toolCallId = null, string? toolName = null)
|
||||
{
|
||||
var (prepared, clipped) = Prepare(text, max);
|
||||
lock (_lock)
|
||||
{
|
||||
var ueber = _size + (prepared?.Length ?? 0) > MaxTotalChars;
|
||||
if (ueber) _truncated = true;
|
||||
_entries.Add(new TranscriptEntry
|
||||
{
|
||||
Kind = kind,
|
||||
At = _clock.GetUtcNow().UtcDateTime,
|
||||
Text = ueber ? null : prepared,
|
||||
ToolCallId = toolCallId,
|
||||
ToolName = toolName,
|
||||
Clipped = clipped || ueber
|
||||
});
|
||||
if (!ueber) _size += prepared?.Length ?? 0;
|
||||
}
|
||||
}
|
||||
|
||||
private static (string? Text, bool Clipped) Prepare(string? text, int max)
|
||||
{
|
||||
if (string.IsNullOrEmpty(text)) return (text, false);
|
||||
var clipped = Clip(SecretScrubber.Scrub(text), max, out var wasClipped);
|
||||
return (clipped, wasClipped);
|
||||
}
|
||||
|
||||
/// <summary>Keeps start and end — the end of a tool result is often the interesting part.</summary>
|
||||
internal static string Clip(string text, int max, out bool clipped)
|
||||
{
|
||||
clipped = text.Length > max;
|
||||
if (!clipped) return text;
|
||||
|
||||
var head = max * 3 / 4;
|
||||
var tail = max - head;
|
||||
return text[..head] + $"\n[… {text.Length - max} characters omitted …]\n" + text[^tail..];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the kinds of secrets that can end up in prompts or tool results before a
|
||||
/// transcript is stored. Deliberately conservative: it targets recognisable secret
|
||||
/// formats, not arbitrary text.
|
||||
/// </summary>
|
||||
public static partial class SecretScrubber
|
||||
{
|
||||
public const string Mask = "«redacted»";
|
||||
|
||||
public static string Scrub(string text)
|
||||
{
|
||||
if (string.IsNullOrEmpty(text)) return text;
|
||||
|
||||
text = ApiKey().Replace(text, Mask);
|
||||
text = Bearer().Replace(text, "Bearer " + Mask);
|
||||
text = PasswordAssignment().Replace(text, m => m.Groups["key"].Value + m.Groups["sep"].Value + Mask);
|
||||
text = LicenseKey().Replace(text, Mask);
|
||||
return text;
|
||||
}
|
||||
|
||||
[GeneratedRegex(@"\b(?:sk-or-v1-|sk-or-|sk-ant-|sk-proj-|sk-|ghp_|gho_|github_pat_|xox[baprs]-)[A-Za-z0-9_\-]{16,}")]
|
||||
private static partial Regex ApiKey();
|
||||
|
||||
[GeneratedRegex(@"Bearer\s+[A-Za-z0-9._~+/\-]{16,}=*", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex Bearer();
|
||||
|
||||
[GeneratedRegex(@"(?<key>\b(?:password|passwort|pwd|secret|api[_-]?key|token)\b)(?<sep>\s*[=:]\s*""?)[^;\s""',}]+", RegexOptions.IgnoreCase)]
|
||||
private static partial Regex PasswordAssignment();
|
||||
|
||||
[GeneratedRegex(@"\b[A-Z0-9]{5}(?:-[A-Z0-9]{5}){4}\b")]
|
||||
private static partial Regex LicenseKey();
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace ClawdDotNet.Core.Transcripts;
|
||||
|
||||
public interface IRunTranscriptStore
|
||||
{
|
||||
Task SaveAsync(RunTranscript transcript, string? systemPrompt, CancellationToken ct);
|
||||
Task<RunTranscript?> GetAsync(string runId, CancellationToken ct);
|
||||
Task<string?> GetSystemPromptAsync(string hash, CancellationToken ct);
|
||||
Task<(IReadOnlyList<RunTranscriptSummary> Items, int Total)> ListAsync(TranscriptQuery query, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// Deletes what is older than each agent's retention allows. <paramref name="retentionFor"/>
|
||||
/// returns the resolved retention in days for an agent id (0 = delete all, -1 = keep).
|
||||
/// </summary>
|
||||
Task<int> PurgeAsync(Func<string, int> retentionFor, DateTime now, CancellationToken ct);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Run transcripts in the instance database (#660). Entries are one JSON column per run:
|
||||
/// a transcript is always read and written as a whole, and the full-text search over
|
||||
/// it is a plain LIKE — enough for the volumes of a single instance.
|
||||
/// </summary>
|
||||
public sealed class SqliteRunTranscriptStore(SqliteStorage storage) : IRunTranscriptStore
|
||||
{
|
||||
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
// Unescaped, so the LIKE search finds "Grüße" as typed. Never rendered as HTML.
|
||||
Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping
|
||||
};
|
||||
|
||||
private const string SummaryColumns =
|
||||
"RunId, AgentId, Model, Source, TaskId, StartedAt, CompletedAt, Status, Prompt, Response, SizeChars, Truncated";
|
||||
|
||||
public Task SaveAsync(RunTranscript t, string? systemPrompt, CancellationToken ct)
|
||||
=> storage.WriteAsync(async conn =>
|
||||
{
|
||||
if (t.SystemPromptHash is not null && systemPrompt is not null)
|
||||
{
|
||||
using var sp = conn.CreateCommand();
|
||||
sp.CommandText = """
|
||||
INSERT OR IGNORE INTO SystemPrompts (Hash, Text, FirstSeenAt) VALUES (@hash, @text, @at)
|
||||
""";
|
||||
sp.Parameters.AddWithValue("@hash", t.SystemPromptHash);
|
||||
sp.Parameters.AddWithValue("@text", systemPrompt);
|
||||
sp.Parameters.AddWithValue("@at", Format(t.StartedAt));
|
||||
await sp.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT OR REPLACE INTO RunTranscripts
|
||||
(RunId, AgentId, Model, Source, TaskId, StartedAt, CompletedAt, Status,
|
||||
SystemPromptHash, Prompt, Response, EntriesJson, SizeChars, Truncated)
|
||||
VALUES
|
||||
(@runId, @agent, @model, @source, @task, @started, @completed, @status,
|
||||
@hash, @prompt, @response, @entries, @size, @truncated)
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@runId", t.RunId);
|
||||
cmd.Parameters.AddWithValue("@agent", t.AgentId);
|
||||
cmd.Parameters.AddWithValue("@model", t.Model);
|
||||
cmd.Parameters.AddWithValue("@source", t.Source);
|
||||
cmd.Parameters.AddWithValue("@task", (object?)t.TaskId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@started", Format(t.StartedAt));
|
||||
cmd.Parameters.AddWithValue("@completed", t.CompletedAt is { } c ? Format(c) : DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@status", t.Status);
|
||||
cmd.Parameters.AddWithValue("@hash", (object?)t.SystemPromptHash ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@prompt", Snippet(t.Entries.FirstOrDefault(e => e.Kind == TranscriptEntryKind.User)?.Text));
|
||||
cmd.Parameters.AddWithValue("@response", Snippet(t.Entries.LastOrDefault(e => e.Kind == TranscriptEntryKind.Assistant && !string.IsNullOrWhiteSpace(e.Text))?.Text));
|
||||
cmd.Parameters.AddWithValue("@entries", JsonSerializer.Serialize(t.Entries, Json));
|
||||
cmd.Parameters.AddWithValue("@size", t.SizeChars);
|
||||
cmd.Parameters.AddWithValue("@truncated", t.Truncated ? 1 : 0);
|
||||
await cmd.ExecuteNonQueryAsync(ct);
|
||||
return true;
|
||||
}, ct);
|
||||
|
||||
public async Task<RunTranscript?> GetAsync(string runId, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
SELECT RunId, AgentId, Model, Source, TaskId, StartedAt, CompletedAt, Status,
|
||||
SystemPromptHash, EntriesJson, SizeChars, Truncated
|
||||
FROM RunTranscripts WHERE RunId = @runId
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@runId", runId);
|
||||
await using var r = await cmd.ExecuteReaderAsync(ct);
|
||||
if (!await r.ReadAsync(ct)) return null;
|
||||
|
||||
return new RunTranscript
|
||||
{
|
||||
RunId = r.GetString(0),
|
||||
AgentId = r.GetString(1),
|
||||
Model = r.GetString(2),
|
||||
Source = r.GetString(3),
|
||||
TaskId = r.IsDBNull(4) ? null : r.GetString(4),
|
||||
StartedAt = Parse(r.GetString(5)),
|
||||
CompletedAt = r.IsDBNull(6) ? null : Parse(r.GetString(6)),
|
||||
Status = r.GetString(7),
|
||||
SystemPromptHash = r.IsDBNull(8) ? null : r.GetString(8),
|
||||
Entries = JsonSerializer.Deserialize<List<TranscriptEntry>>(r.GetString(9), Json) ?? [],
|
||||
SizeChars = r.GetInt32(10),
|
||||
Truncated = r.GetInt32(11) != 0
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<string?> GetSystemPromptAsync(string hash, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT Text FROM SystemPrompts WHERE Hash = @hash";
|
||||
cmd.Parameters.AddWithValue("@hash", hash);
|
||||
return await cmd.ExecuteScalarAsync(ct) as string;
|
||||
}
|
||||
|
||||
public async Task<(IReadOnlyList<RunTranscriptSummary> Items, int Total)> ListAsync(TranscriptQuery q, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await storage.OpenConnectionAsync(ct);
|
||||
|
||||
var where = new List<string>();
|
||||
void Bind(SqliteCommand c)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(q.AgentId)) c.Parameters.AddWithValue("@agent", q.AgentId);
|
||||
if (q.From is { } f) c.Parameters.AddWithValue("@from", Format(f));
|
||||
if (q.To is { } t) c.Parameters.AddWithValue("@to", Format(t));
|
||||
if (!string.IsNullOrWhiteSpace(q.Search)) c.Parameters.AddWithValue("@q", "%" + Escape(q.Search.Trim()) + "%");
|
||||
}
|
||||
if (!string.IsNullOrWhiteSpace(q.AgentId)) where.Add("AgentId = @agent");
|
||||
if (q.From is not null) where.Add("StartedAt >= @from");
|
||||
if (q.To is not null) where.Add("StartedAt < @to");
|
||||
if (!string.IsNullOrWhiteSpace(q.Search)) where.Add("EntriesJson LIKE @q ESCAPE '\\'");
|
||||
var filter = where.Count == 0 ? "" : " WHERE " + string.Join(" AND ", where);
|
||||
|
||||
using var count = conn.CreateCommand();
|
||||
count.CommandText = "SELECT COUNT(*) FROM RunTranscripts" + filter;
|
||||
Bind(count);
|
||||
var total = Convert.ToInt32(await count.ExecuteScalarAsync(ct));
|
||||
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = $"SELECT {SummaryColumns} FROM RunTranscripts{filter} ORDER BY StartedAt DESC LIMIT @limit OFFSET @offset";
|
||||
Bind(cmd);
|
||||
cmd.Parameters.AddWithValue("@limit", Math.Clamp(q.Limit, 1, 500));
|
||||
cmd.Parameters.AddWithValue("@offset", Math.Max(0, q.Offset));
|
||||
|
||||
var items = new List<RunTranscriptSummary>();
|
||||
await using var r = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await r.ReadAsync(ct))
|
||||
{
|
||||
items.Add(new RunTranscriptSummary(
|
||||
r.GetString(0), r.GetString(1), r.GetString(2), r.GetString(3),
|
||||
r.IsDBNull(4) ? null : r.GetString(4),
|
||||
Parse(r.GetString(5)),
|
||||
r.IsDBNull(6) ? null : Parse(r.GetString(6)),
|
||||
r.GetString(7), r.GetString(8), r.GetString(9), r.GetInt32(10), r.GetInt32(11) != 0));
|
||||
}
|
||||
return (items, total);
|
||||
}
|
||||
|
||||
public Task<int> PurgeAsync(Func<string, int> retentionFor, DateTime now, CancellationToken ct)
|
||||
=> storage.WriteAsync(async conn =>
|
||||
{
|
||||
var agents = new List<string>();
|
||||
using (var list = conn.CreateCommand())
|
||||
{
|
||||
list.CommandText = "SELECT DISTINCT AgentId FROM RunTranscripts";
|
||||
await using var r = await list.ExecuteReaderAsync(ct);
|
||||
while (await r.ReadAsync(ct)) agents.Add(r.GetString(0));
|
||||
}
|
||||
|
||||
var deleted = 0;
|
||||
foreach (var agent in agents)
|
||||
{
|
||||
var days = PromptHistorySettings.Normalize(retentionFor(agent));
|
||||
if (days == PromptHistorySettings.Forever) continue;
|
||||
|
||||
using var del = conn.CreateCommand();
|
||||
del.CommandText = "DELETE FROM RunTranscripts WHERE AgentId = @agent AND StartedAt < @cutoff";
|
||||
del.Parameters.AddWithValue("@agent", agent);
|
||||
// Never (0): everything goes, including what was recorded before the switch.
|
||||
del.Parameters.AddWithValue("@cutoff", Format(days == PromptHistorySettings.Never ? DateTime.MaxValue : now.AddDays(-days)));
|
||||
deleted += await del.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
using var orphans = conn.CreateCommand();
|
||||
orphans.CommandText = """
|
||||
DELETE FROM SystemPrompts
|
||||
WHERE Hash NOT IN (SELECT SystemPromptHash FROM RunTranscripts WHERE SystemPromptHash IS NOT NULL)
|
||||
""";
|
||||
await orphans.ExecuteNonQueryAsync(ct);
|
||||
return deleted;
|
||||
}, ct);
|
||||
|
||||
private static string Snippet(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text)) return "";
|
||||
var t = text.Trim();
|
||||
return t.Length <= 300 ? t : t[..300] + "…";
|
||||
}
|
||||
|
||||
private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("%", "\\%").Replace("_", "\\_");
|
||||
|
||||
private static string Format(DateTime value)
|
||||
=> (value == DateTime.MaxValue ? value : value.ToUniversalTime()).ToString("O", CultureInfo.InvariantCulture);
|
||||
|
||||
private static DateTime Parse(string value)
|
||||
=> DateTime.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind).ToUniversalTime();
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Core.Transcripts;
|
||||
|
||||
/// <summary>
|
||||
/// Enforces the prompt history retention (#660): once at start and then every hour,
|
||||
/// transcripts older than their agent's retention are deleted. An agent switched to
|
||||
/// "never" loses everything recorded before the switch — that is the point of the switch.
|
||||
/// </summary>
|
||||
public sealed class TranscriptJanitor(
|
||||
IRunTranscriptStore store,
|
||||
Func<string, int> retentionFor,
|
||||
ILogger logger,
|
||||
TimeProvider? clock = null,
|
||||
TimeSpan? interval = null) : IAsyncDisposable
|
||||
{
|
||||
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task? _loop;
|
||||
|
||||
public void Start() => _loop ??= RunAsync(_cts.Token);
|
||||
|
||||
public async Task<int> PurgeOnceAsync(CancellationToken ct)
|
||||
{
|
||||
var deleted = await store.PurgeAsync(retentionFor, _clock.GetUtcNow().UtcDateTime, ct);
|
||||
if (deleted > 0)
|
||||
logger.LogInformation("Prompt history: {Count} transcript(s) past their retention deleted", deleted);
|
||||
return deleted;
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
using var timer = new PeriodicTimer(interval ?? TimeSpan.FromHours(1), _clock);
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await PurgeOnceAsync(ct);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(ex, "Prompt history cleanup failed");
|
||||
}
|
||||
}
|
||||
while (await timer.WaitForNextTickAsync(ct));
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
if (_loop is not null)
|
||||
{
|
||||
try { await _loop; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user