Audit und Laufbelege: Tageswerte je Agent und Einheit, Monatsarchiv statt Loeschen (#700)

Nach der Vorgabe vom 2026-09-22 (so wenig loeschen wie moeglich): Laeufe
aelter als hotDays (Vorgabe 180, 0 = alles bleibt) wandern aus AuditLog und
RunReceipts in Archive/audit-JJJJ-MM.db - kopiert und entfernt in einer
Transaktion, Zeilen-IDs bleiben, spaeter dazugekommene Spalten werden im
Archiv ergaenzt. Vorher wird jeder Tag in RunDaily verdichtet (Laeufe,
Fehlschlaege, Schrittgrenze, Tokens, Kosten, Werkzeugaufrufe, Fehler,
Verweigert, Freigaben) mit der Einheit des Agenten; die Tageswerte bleiben.
Nachschlagen eines Laufs findet ihn auch im Archiv. API /usage/days,
/usage/archive; Karte 'Tageswerte' in der Verbrauchsansicht.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
2026-10-02 11:44:50 +02:00
co-authored by Claude Opus 5.5
parent 0dd8a761b8
commit 095722ae9c
11 changed files with 761 additions and 13 deletions
+12
View File
@@ -141,6 +141,9 @@ public sealed class AppHost : IAsyncDisposable
/// <summary>#725: Inbound triggers — n8n or any other system hands an agent a task.</summary> /// <summary>#725: Inbound triggers — n8n or any other system hands an agent a task.</summary>
public Core.Triggers.InboundTriggers? Triggers { get; private set; } public Core.Triggers.InboundTriggers? Triggers { get; private set; }
/// <summary>#700: Day values and the month archive of audit and receipts.</summary>
public Core.Audit.AuditArchive? AuditArchive { get; private set; }
/// <summary>#197: The external MCP servers, each registered as a tool.</summary> /// <summary>#197: The external MCP servers, each registered as a tool.</summary>
public Core.Mcp.McpConnections? McpServers { get; private set; } public Core.Mcp.McpConnections? McpServers { get; private set; }
@@ -724,6 +727,15 @@ public sealed class AppHost : IAsyncDisposable
historyKeeper.Start(); historyKeeper.Start();
host._shutdown.Add(historyKeeper.DisposeAsync); host._shutdown.Add(historyKeeper.DisposeAsync);
// #700: day values, and runs older than the hot window move to the month archive.
var auditArchive = new Core.Audit.AuditArchive(
storage, () => instance.AuditArchive,
agentId => instance.Agents.FirstOrDefault(a => string.Equals(a.AgentId, agentId, StringComparison.OrdinalIgnoreCase))?.UnitId,
loggerFactory.CreateLogger("ClawdDotNet.Core.Audit"));
auditArchive.Start();
host.AuditArchive = auditArchive;
host._shutdown.Add(auditArchive.DisposeAsync);
// #698: what holds in a unit, set by its lead. // #698: what holds in a unit, set by its lead.
var doctrines = new Core.Organisation.SqliteDoctrineStore(storage); var doctrines = new Core.Organisation.SqliteDoctrineStore(storage);
engine.Doctrines = doctrines; engine.Doctrines = doctrines;
+388
View File
@@ -0,0 +1,388 @@
using System.Globalization;
using System.Text.Json.Serialization;
using ClawdDotNet.Core.Storage;
using Microsoft.Data.Sqlite;
using Microsoft.Extensions.Logging;
namespace ClawdDotNet.Core.Audit;
/// <summary>#700: How long audit and receipts stay in the working database (<c>auditArchive</c> in the instance settings).</summary>
public sealed class AuditArchiveSettings
{
/// <summary>
/// Days a run stays in the working database; older ones move to the monthly archive.
/// 0 keeps everything in the working database. Nothing is ever deleted either way.
/// </summary>
[JsonPropertyName("hotDays")]
public int HotDays { get; set; } = 180;
}
/// <summary>#700: One day of one agent, condensed — stays in the working database for good.</summary>
public sealed record RunDay(
string Day, string AgentId, string UnitId, int Runs, int Failed, int LoopLimit,
long PromptTokens, long CompletionTokens, decimal CostUsd, int ToolCalls, int ToolErrors, int Denied, int Staged);
/// <summary>An archive file as the administration shows it.</summary>
public sealed record AuditArchiveFile(string Month, string FileName, long Bytes);
/// <summary>
/// #700: The audit is evidence, and evidence is kept — the user's decision of 2026-09-22: as
/// little deleted as possible. Five years after a failed project, how it came about can be read:
/// which task was worded how, what the agent read, who approved what. That is the one real
/// advantage an organisation of agents has over one of people.
///
/// <para><b>What grows is the working table, not the disk.</b> A hundred agents write some
/// 7,400 audit rows a day; every supervision and every view reads those tables. So rows older
/// than <see cref="AuditArchiveSettings.HotDays"/> move — never get deleted — into one SQLite
/// file per month under <c>Archive/</c>, which the backup takes along like any other file.
/// Looking up a run finds them there.</para>
///
/// <para><b>Day values stay.</b> Before rows move, each day is condensed per agent and unit
/// (<see cref="RunDay"/>): runs, failures, tokens, costs, tool calls. Most questions over months
/// are answered from those, without opening an archive. The unit is taken when the day is
/// condensed — later "what did marketing cost in March" cannot be answered otherwise.</para>
/// </summary>
public sealed class AuditArchive(
SqliteStorage storage,
Func<AuditArchiveSettings> settings,
Func<string, string?> unitOf,
ILogger logger,
TimeProvider? clock = null) : IAsyncDisposable
{
public const string FolderName = "Archive";
private readonly TimeProvider _clock = clock ?? TimeProvider.System;
public string Directory => ArchiveDirectoryOf(storage);
public static string ArchiveDirectoryOf(SqliteStorage storage)
=> Path.Combine(Path.GetDirectoryName(Path.GetFullPath(storage.DatabasePath))!, FolderName);
public static string FileNameOf(string month) => $"audit-{month}.db";
private DateTime Now => _clock.GetUtcNow().UtcDateTime;
// ─── Day values ───
/// <summary>
/// Condenses every day before today that is not condensed yet — and the last two again,
/// because a run can end after midnight. Days are UTC days.
/// </summary>
public async Task<int> CondenseAsync(CancellationToken ct)
{
var today = Now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
return await storage.WriteAsync(async conn =>
{
string? last;
using (var max = conn.CreateCommand())
{
max.CommandText = "SELECT MAX(Day) FROM RunDaily";
last = await max.ExecuteScalarAsync(ct) as string;
}
var from = last is null ? "" : DateOnly.ParseExact(last, "yyyy-MM-dd", CultureInfo.InvariantCulture).AddDays(-1).ToString("yyyy-MM-dd", CultureInfo.InvariantCulture);
var days = new Dictionary<(string Day, string Agent), RunDayBuilder>();
using (var receipts = conn.CreateCommand())
{
receipts.CommandText = """
SELECT substr(OccurredAt, 1, 10) AS Day, AgentId,
SUM(CASE WHEN Status IN ('BudgetExceeded', 'TargetBusy', 'ShuttingDown') THEN 0 ELSE 1 END),
SUM(CASE WHEN Status IN ('Failed', 'Cancelled', 'Incomplete') THEN 1 ELSE 0 END),
SUM(CASE WHEN Status = 'LoopLimitExceeded' THEN 1 ELSE 0 END),
SUM(PromptTokens), SUM(CompletionTokens), GROUP_CONCAT(CostUsd, ';')
FROM RunReceipts
WHERE substr(OccurredAt, 1, 10) >= @from AND substr(OccurredAt, 1, 10) < @today
GROUP BY Day, AgentId
""";
receipts.Parameters.AddWithValue("@from", from);
receipts.Parameters.AddWithValue("@today", today);
await using var r = await receipts.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
{
var b = Builder(days, r.GetString(0), r.GetString(1));
b.Runs = r.GetInt32(2);
b.Failed = r.GetInt32(3);
b.LoopLimit = r.GetInt32(4);
b.PromptTokens = r.GetInt64(5);
b.CompletionTokens = r.GetInt64(6);
b.CostUsd = r.IsDBNull(7) ? 0m : r.GetString(7).Split(';').Sum(c =>
decimal.TryParse(c, NumberStyles.Any, CultureInfo.InvariantCulture, out var v) ? v : 0m);
}
}
using (var audit = conn.CreateCommand())
{
audit.CommandText = """
SELECT substr(OccurredAt, 1, 10) AS Day, AgentId,
SUM(CASE WHEN Status NOT IN ('Started', 'Staged', 'Denied') THEN 1 ELSE 0 END),
SUM(CASE WHEN Status = 'Error' THEN 1 ELSE 0 END),
SUM(CASE WHEN Status = 'Denied' THEN 1 ELSE 0 END),
SUM(CASE WHEN Status = 'Staged' THEN 1 ELSE 0 END)
FROM AuditLog
WHERE substr(OccurredAt, 1, 10) >= @from AND substr(OccurredAt, 1, 10) < @today AND AgentId <> ''
GROUP BY Day, AgentId
""";
audit.Parameters.AddWithValue("@from", from);
audit.Parameters.AddWithValue("@today", today);
await using var r = await audit.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
{
var b = Builder(days, r.GetString(0), r.GetString(1));
b.ToolCalls = r.GetInt32(2);
b.ToolErrors = r.GetInt32(3);
b.Denied = r.GetInt32(4);
b.Staged = r.GetInt32(5);
}
}
foreach (var ((day, agent), b) in days)
{
using var upsert = conn.CreateCommand();
upsert.CommandText = """
INSERT INTO RunDaily (Day, AgentId, UnitId, Runs, Failed, LoopLimit, PromptTokens, CompletionTokens,
CostUsd, ToolCalls, ToolErrors, Denied, Staged)
VALUES (@day, @agent, @unit, @runs, @failed, @loop, @prompt, @completion, @cost, @calls, @errors, @denied, @staged)
ON CONFLICT (Day, AgentId) DO UPDATE SET
Runs = excluded.Runs, Failed = excluded.Failed, LoopLimit = excluded.LoopLimit,
PromptTokens = excluded.PromptTokens, CompletionTokens = excluded.CompletionTokens,
CostUsd = excluded.CostUsd, ToolCalls = excluded.ToolCalls, ToolErrors = excluded.ToolErrors,
Denied = excluded.Denied, Staged = excluded.Staged
""";
// The unit stays what it was when the day was first condensed.
upsert.Parameters.AddWithValue("@day", day);
upsert.Parameters.AddWithValue("@agent", agent);
upsert.Parameters.AddWithValue("@unit", unitOf(agent) ?? "");
upsert.Parameters.AddWithValue("@runs", b.Runs);
upsert.Parameters.AddWithValue("@failed", b.Failed);
upsert.Parameters.AddWithValue("@loop", b.LoopLimit);
upsert.Parameters.AddWithValue("@prompt", b.PromptTokens);
upsert.Parameters.AddWithValue("@completion", b.CompletionTokens);
upsert.Parameters.AddWithValue("@cost", b.CostUsd.ToString(CultureInfo.InvariantCulture));
upsert.Parameters.AddWithValue("@calls", b.ToolCalls);
upsert.Parameters.AddWithValue("@errors", b.ToolErrors);
upsert.Parameters.AddWithValue("@denied", b.Denied);
upsert.Parameters.AddWithValue("@staged", b.Staged);
await upsert.ExecuteNonQueryAsync(ct);
}
return days.Count;
}, ct);
}
private sealed class RunDayBuilder
{
public int Runs, Failed, LoopLimit, ToolCalls, ToolErrors, Denied, Staged;
public long PromptTokens, CompletionTokens;
public decimal CostUsd;
}
private static RunDayBuilder Builder(Dictionary<(string, string), RunDayBuilder> days, string day, string agent)
{
if (!days.TryGetValue((day, agent), out var b))
days[(day, agent)] = b = new RunDayBuilder();
return b;
}
/// <summary>Day values in a range (inclusive), optionally for one agent or one unit.</summary>
public async Task<IReadOnlyList<RunDay>> DaysAsync(DateOnly from, DateOnly to, string? agentId, string? unitId, CancellationToken ct)
{
await using var conn = await storage.OpenConnectionAsync(ct);
using var cmd = conn.CreateCommand();
cmd.CommandText = """
SELECT Day, AgentId, UnitId, Runs, Failed, LoopLimit, PromptTokens, CompletionTokens, CostUsd,
ToolCalls, ToolErrors, Denied, Staged
FROM RunDaily
WHERE Day >= @from AND Day <= @to
AND (@agent IS NULL OR AgentId = @agent COLLATE NOCASE)
AND (@unit IS NULL OR UnitId = @unit COLLATE NOCASE)
ORDER BY Day, AgentId
""";
cmd.Parameters.AddWithValue("@from", from.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
cmd.Parameters.AddWithValue("@to", to.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));
cmd.Parameters.AddWithValue("@agent", (object?)agentId ?? DBNull.Value);
cmd.Parameters.AddWithValue("@unit", (object?)unitId ?? DBNull.Value);
var result = new List<RunDay>();
await using var r = await cmd.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
result.Add(new RunDay(r.GetString(0), r.GetString(1), r.GetString(2), r.GetInt32(3), r.GetInt32(4), r.GetInt32(5),
r.GetInt64(6), r.GetInt64(7),
decimal.TryParse(r.GetString(8), NumberStyles.Any, CultureInfo.InvariantCulture, out var c) ? c : 0m,
r.GetInt32(9), r.GetInt32(10), r.GetInt32(11), r.GetInt32(12)));
return result;
}
// ─── Moving to the archive ───
/// <summary>
/// Moves audit rows and receipts older than the hot window into their month's archive file.
/// Days are condensed first, so no day value is lost with the move. Copy and delete happen
/// in one transaction: a row is in one place or the other, never in none.
/// </summary>
public async Task<int> ArchiveAsync(CancellationToken ct)
{
var hotDays = settings().HotDays;
if (hotDays <= 0)
return 0;
await CondenseAsync(ct);
// Whole days only, and never today's or yesterday's: those are not condensed for good yet.
var cutoff = Now.Date.AddDays(-Math.Max(2, hotDays)).ToString("O");
System.IO.Directory.CreateDirectory(Directory);
return await storage.WriteAsync(async conn =>
{
var months = new SortedSet<string>(StringComparer.Ordinal);
foreach (var table in new[] { "AuditLog", "RunReceipts" })
{
using var cmd = conn.CreateCommand();
cmd.CommandText = $"SELECT DISTINCT substr(OccurredAt, 1, 7) FROM {table} WHERE OccurredAt < @cutoff";
cmd.Parameters.AddWithValue("@cutoff", cutoff);
await using var r = await cmd.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
months.Add(r.GetString(0));
}
var moved = 0;
foreach (var month in months)
moved += await MoveMonthAsync(conn, month, cutoff, ct);
if (moved > 0)
logger.LogInformation("Audit archive: {Count} row(s) older than {Days} days moved to {Folder}", moved, hotDays, Directory);
return moved;
}, ct);
}
private async Task<int> MoveMonthAsync(SqliteConnection conn, string month, string cutoff, CancellationToken ct)
{
var file = Path.Combine(Directory, FileNameOf(month));
using (var attach = conn.CreateCommand())
{
attach.CommandText = "ATTACH DATABASE @file AS arch";
attach.Parameters.AddWithValue("@file", file);
await attach.ExecuteNonQueryAsync(ct);
}
try
{
using (var create = conn.CreateCommand())
{
// Same columns as the working tables, row ids kept — an id in a report still finds its row.
create.CommandText = """
CREATE TABLE IF NOT EXISTS arch.AuditLog AS SELECT * FROM main.AuditLog WHERE 0;
CREATE TABLE IF NOT EXISTS arch.RunReceipts AS SELECT * FROM main.RunReceipts WHERE 0;
CREATE INDEX IF NOT EXISTS arch.IX_Archive_AuditLog_Run ON AuditLog (RunId);
CREATE INDEX IF NOT EXISTS arch.IX_Archive_RunReceipts_Run ON RunReceipts (RunId);
CREATE INDEX IF NOT EXISTS arch.IX_Archive_RunReceipts_Task ON RunReceipts (TaskId);
""";
await create.ExecuteNonQueryAsync(ct);
}
// A column the working table gained after the archive file was made is added to it —
// otherwise the month's next move would fail on a column count.
var columns = new Dictionary<string, IReadOnlyList<string>>();
foreach (var table in new[] { "AuditLog", "RunReceipts" })
columns[table] = await AlignColumnsAsync(conn, table, ct);
await using var tx = await conn.BeginTransactionAsync(ct);
var moved = 0;
foreach (var table in new[] { "AuditLog", "RunReceipts" })
{
var list = string.Join(", ", columns[table]);
using var copy = conn.CreateCommand();
copy.Transaction = (SqliteTransaction)tx;
copy.CommandText = $"""
INSERT INTO arch.{table} ({list}) SELECT {list} FROM main.{table}
WHERE OccurredAt < @cutoff AND substr(OccurredAt, 1, 7) = @month
AND Id NOT IN (SELECT Id FROM arch.{table});
DELETE FROM main.{table} WHERE OccurredAt < @cutoff AND substr(OccurredAt, 1, 7) = @month
AND Id IN (SELECT Id FROM arch.{table});
""";
copy.Parameters.AddWithValue("@cutoff", cutoff);
copy.Parameters.AddWithValue("@month", month);
moved += await copy.ExecuteNonQueryAsync(ct) / 2;
}
await tx.CommitAsync(ct);
return moved;
}
finally
{
using var detach = conn.CreateCommand();
detach.CommandText = "DETACH DATABASE arch";
await detach.ExecuteNonQueryAsync(CancellationToken.None);
}
}
/// <summary>The working table's columns, each present in the archive table too.</summary>
private static async Task<IReadOnlyList<string>> AlignColumnsAsync(SqliteConnection conn, string table, CancellationToken ct)
{
async Task<List<(string Name, string Type)>> Columns(string schema)
{
using var cmd = conn.CreateCommand();
cmd.CommandText = $"SELECT name, type FROM pragma_table_info('{table}', '{schema}')";
var list = new List<(string, string)>();
await using var r = await cmd.ExecuteReaderAsync(ct);
while (await r.ReadAsync(ct))
list.Add((r.GetString(0), r.GetString(1)));
return list;
}
var main = await Columns("main");
var archived = (await Columns("arch")).Select(c => c.Name).ToHashSet(StringComparer.OrdinalIgnoreCase);
foreach (var (name, type) in main.Where(c => !archived.Contains(c.Name)))
{
using var add = conn.CreateCommand();
add.CommandText = $"ALTER TABLE arch.{table} ADD COLUMN {name} {type}";
await add.ExecuteNonQueryAsync(ct);
}
return main.Select(c => c.Name).ToList();
}
/// <summary>The archive files, newest month first.</summary>
public IReadOnlyList<AuditArchiveFile> Files() => FilesIn(Directory);
private readonly CancellationTokenSource _cts = new();
private Task? _loop;
/// <summary>Condenses and archives once after the start, then every six hours.</summary>
public void Start() => _loop ??= LoopAsync(_cts.Token);
private async Task LoopAsync(CancellationToken ct)
{
try { await Task.Delay(TimeSpan.FromMinutes(2), ct); }
catch (OperationCanceledException) { return; }
while (!ct.IsCancellationRequested)
{
try
{
await ArchiveAsync(ct);
if (settings().HotDays <= 0)
await CondenseAsync(ct);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Audit archive: condensing or moving failed");
}
try { await Task.Delay(TimeSpan.FromHours(6), ct); }
catch (OperationCanceledException) { return; }
}
}
public async ValueTask DisposeAsync()
{
await _cts.CancelAsync();
if (_loop is not null)
{
try { await _loop; }
catch (OperationCanceledException) { }
}
_cts.Dispose();
}
public static IReadOnlyList<AuditArchiveFile> FilesIn(string directory)
{
if (!System.IO.Directory.Exists(directory))
return [];
return System.IO.Directory.EnumerateFiles(directory, "audit-????-??.db")
.Select(f => new AuditArchiveFile(Path.GetFileNameWithoutExtension(f)["audit-".Length..], Path.GetFileName(f), new FileInfo(f).Length))
.OrderByDescending(f => f.Month, StringComparer.Ordinal)
.ToList();
}
}
@@ -83,22 +83,72 @@ public sealed class SqliteAuditRepository : IAuditRepository
public async Task<IReadOnlyList<AuditEntry>> ListForRunAsync(string runId, CancellationToken ct) public async Task<IReadOnlyList<AuditEntry>> ListForRunAsync(string runId, CancellationToken ct)
{ {
await using var conn = await _storage.OpenConnectionAsync(ct); await using (var conn = await _storage.OpenConnectionAsync(ct))
using var cmd = conn.CreateCommand(); {
cmd.CommandText = "SELECT " + AuditColumns + " FROM AuditLog WHERE RunId = @runId ORDER BY Id"; using var cmd = conn.CreateCommand();
cmd.Parameters.AddWithValue("@runId", runId); cmd.CommandText = "SELECT " + AuditColumns + " FROM AuditLog WHERE RunId = @runId ORDER BY Id";
return await ReadEntriesAsync(cmd, ct); cmd.Parameters.AddWithValue("@runId", runId);
var hot = await ReadEntriesAsync(cmd, ct);
if (hot.Count > 0)
return hot;
}
// #700: an older run moved to the archive — it is still evidence and still found.
foreach (var archive in ArchiveConnections())
await using (archive)
{
using var cmd = archive.CreateCommand();
cmd.CommandText = "SELECT " + AuditColumns + " FROM AuditLog WHERE RunId = @runId ORDER BY Id";
cmd.Parameters.AddWithValue("@runId", runId);
var found = await ReadEntriesAsync(cmd, ct);
if (found.Count > 0)
return found;
}
return [];
} }
public async Task<RunReceipt?> GetReceiptForRunAsync(string runId, CancellationToken ct) public async Task<RunReceipt?> GetReceiptForRunAsync(string runId, CancellationToken ct)
{ {
await using var conn = await _storage.OpenConnectionAsync(ct); await using (var conn = await _storage.OpenConnectionAsync(ct))
using var cmd = conn.CreateCommand(); {
cmd.CommandText = "SELECT " + ReceiptColumns + " FROM RunReceipts WHERE RunId = @runId ORDER BY Id DESC LIMIT 1"; using var cmd = conn.CreateCommand();
cmd.Parameters.AddWithValue("@runId", runId); cmd.CommandText = "SELECT " + ReceiptColumns + " FROM RunReceipts WHERE RunId = @runId ORDER BY Id DESC LIMIT 1";
cmd.Parameters.AddWithValue("@runId", runId);
await using var reader = await cmd.ExecuteReaderAsync(ct);
if (await reader.ReadAsync(ct))
return ReadReceipt(reader);
}
await using var reader = await cmd.ExecuteReaderAsync(ct); foreach (var archive in ArchiveConnections())
return await reader.ReadAsync(ct) ? ReadReceipt(reader) : null; await using (archive)
{
using var cmd = archive.CreateCommand();
cmd.CommandText = "SELECT " + ReceiptColumns + " FROM RunReceipts WHERE RunId = @runId ORDER BY Id DESC LIMIT 1";
cmd.Parameters.AddWithValue("@runId", runId);
await using var reader = await cmd.ExecuteReaderAsync(ct);
if (await reader.ReadAsync(ct))
return ReadReceipt(reader);
}
return null;
}
/// <summary>
/// #700: The month archives, newest first, opened read-only one at a time. A lookup by run
/// is rare and asks for one run — opening a few small files for it is cheap.
/// </summary>
private IEnumerable<SqliteConnection> ArchiveConnections()
{
foreach (var file in AuditArchive.FilesIn(AuditArchive.ArchiveDirectoryOf(_storage)))
{
var connection = new SqliteConnection(new SqliteConnectionStringBuilder
{
DataSource = Path.Combine(AuditArchive.ArchiveDirectoryOf(_storage), file.FileName),
Mode = SqliteOpenMode.ReadOnly,
Pooling = false
}.ToString());
connection.Open();
yield return connection;
}
} }
public async Task<IReadOnlyList<RunReceipt>> ListReceiptsAsync(DateTime fromUtc, DateTime toUtc, int limit, CancellationToken ct) public async Task<IReadOnlyList<RunReceipt>> ListReceiptsAsync(DateTime fromUtc, DateTime toUtc, int limit, CancellationToken ct)
@@ -155,6 +155,10 @@ public sealed class InstanceConfig
[JsonPropertyName("claudeCode")] [JsonPropertyName("claudeCode")]
public Runtimes.ClaudeCodeSettings ClaudeCode { get; set; } = new(); public Runtimes.ClaudeCodeSettings ClaudeCode { get; set; } = new();
/// <summary>#700: How long runs stay in the working database before they move to the month archive — nothing is deleted.</summary>
[JsonPropertyName("auditArchive")]
public Audit.AuditArchiveSettings AuditArchive { get; set; } = new();
/// <summary>#719: The history of the workspaces — on by default, 90 days.</summary> /// <summary>#719: The history of the workspaces — on by default, 90 days.</summary>
[JsonPropertyName("fileHistory")] [JsonPropertyName("fileHistory")]
public Files.FileHistorySettings FileHistory { get; set; } = new(); public Files.FileHistorySettings FileHistory { get; set; } = new();
@@ -282,6 +282,26 @@ public sealed class SqliteStorage
CREATE INDEX IF NOT EXISTS IX_RunReceipts_Run CREATE INDEX IF NOT EXISTS IX_RunReceipts_Run
ON RunReceipts (RunId); ON RunReceipts (RunId);
-- #700: one day of one agent, condensed; stays when the rows move to the archive.
CREATE TABLE IF NOT EXISTS RunDaily (
Day TEXT NOT NULL,
AgentId TEXT NOT NULL,
UnitId TEXT NOT NULL DEFAULT '',
Runs INTEGER NOT NULL DEFAULT 0,
Failed INTEGER NOT NULL DEFAULT 0,
LoopLimit INTEGER NOT NULL DEFAULT 0,
PromptTokens INTEGER NOT NULL DEFAULT 0,
CompletionTokens INTEGER NOT NULL DEFAULT 0,
CostUsd TEXT NOT NULL DEFAULT '0',
ToolCalls INTEGER NOT NULL DEFAULT 0,
ToolErrors INTEGER NOT NULL DEFAULT 0,
Denied INTEGER NOT NULL DEFAULT 0,
Staged INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (Day, AgentId)
);
CREATE INDEX IF NOT EXISTS IX_RunDaily_Unit ON RunDaily (UnitId, Day);
-- Staging (A2): eingefrorene, freigabepflichtige Tool-Aufrufe. Ausgeführt wird -- Staging (A2): eingefrorene, freigabepflichtige Tool-Aufrufe. Ausgeführt wird
-- genau der gespeicherte Argument-JSON (Plan-Freeze). -- genau der gespeicherte Argument-JSON (Plan-Freeze).
-- #698: a unit's doctrine, every version kept — who, when, which run, why. -- #698: a unit's doctrine, every version kept — who, when, which run, why.
@@ -796,6 +796,22 @@
"usage.units.kind.Deviation": "Ungewöhnlicher Tag", "usage.units.kind.Deviation": "Ungewöhnlicher Tag",
"usage.units.kind.NearLimit": "Nahe an den Grenzen", "usage.units.kind.NearLimit": "Nahe an den Grenzen",
"usage.units.kind.MonthlyThreshold": "Monatssumme überschritten", "usage.units.kind.MonthlyThreshold": "Monatssumme überschritten",
"usage.days.title": "Tageswerte",
"usage.days.hint": "Die letzten 30 Tage, summiert über alle Agenten (UTC-Tage). Jeder Tag wird je Agent und Einheit verdichtet und bleibt für immer — auch wenn seine Läufe ins Archiv wandern. Heute erscheint morgen.",
"usage.days.none": "Noch kein Tag verdichtet — die ersten Werte erscheinen wenige Minuten nach dem Start, ab gestern.",
"usage.days.day": "Tag",
"usage.days.runs": "Läufe",
"usage.days.failed": "Gescheitert",
"usage.days.loopLimit": "Schrittgrenze",
"usage.days.cost": "Kosten",
"usage.days.calls": "Werkzeugaufrufe",
"usage.days.errors": "Werkzeugfehler",
"usage.days.staged": "Zur Freigabe",
"usage.archive.title": "Archiv",
"usage.archive.hint": "Audit-Einträge und Laufbelege sind Belege und werden nie gelöscht. Nach so vielen Tagen wandern sie aus der Arbeitsdatenbank in eine Archivdatei je Monat; ein Lauf wird dort weiterhin gefunden. 0 behält alles in der Arbeitsdatenbank.",
"usage.archive.hotDays": "Tage in der Arbeitsdatenbank",
"usage.archive.none": "Noch keine Archivdatei.",
"usage.archive.files": "Archivdateien: {list}",
"usage.forecastTitle": "Kostenprognose", "usage.forecastTitle": "Kostenprognose",
"usage.forecastDay": "Prognose je Tag", "usage.forecastDay": "Prognose je Tag",
"usage.forecastMonth": "Prognose je Monat (30 Tage)", "usage.forecastMonth": "Prognose je Monat (30 Tage)",
@@ -796,6 +796,22 @@
"usage.units.kind.Deviation": "Unusual day", "usage.units.kind.Deviation": "Unusual day",
"usage.units.kind.NearLimit": "Near the limits", "usage.units.kind.NearLimit": "Near the limits",
"usage.units.kind.MonthlyThreshold": "Monthly total crossed", "usage.units.kind.MonthlyThreshold": "Monthly total crossed",
"usage.days.title": "Day values",
"usage.days.hint": "The last 30 days, summed over all agents (UTC days). Each day is condensed per agent and unit and stays for good — also when its runs move to the archive. Today appears tomorrow.",
"usage.days.none": "No day condensed yet — the first values appear a few minutes after the start, from yesterday on.",
"usage.days.day": "Day",
"usage.days.runs": "Runs",
"usage.days.failed": "Failed",
"usage.days.loopLimit": "Step limit",
"usage.days.cost": "Cost",
"usage.days.calls": "Tool calls",
"usage.days.errors": "Tool errors",
"usage.days.staged": "For approval",
"usage.archive.title": "Archive",
"usage.archive.hint": "Audit entries and receipts are evidence and are never deleted. After this many days they move from the working database into one archive file per month; a run is still found there. 0 keeps everything in the working database.",
"usage.archive.hotDays": "Days in the working database",
"usage.archive.none": "No archive file yet.",
"usage.archive.files": "Archive files: {list}",
"usage.forecastTitle": "Cost Forecast", "usage.forecastTitle": "Cost Forecast",
"usage.forecastDay": "Forecast per day", "usage.forecastDay": "Forecast per day",
"usage.forecastMonth": "Forecast per month (30 days)", "usage.forecastMonth": "Forecast per month (30 days)",
@@ -13,7 +13,8 @@ import {
erstelleKarte, erstelleKarte,
formatiereDatumZeit, formatiereDatumZeit,
formatiereTokens, formatiereTokens,
formatiereKosten formatiereKosten,
zeigeMeldung
} from "../modules/dom.js"; } from "../modules/dom.js";
export async function rendereVerbrauch() { export async function rendereVerbrauch() {
@@ -44,10 +45,11 @@ async function ladeVerbrauch() {
); );
try { try {
const [laeufe, prognose, einheiten] = await Promise.all([api("/usage?limit=100"), erstellePrognoseKarte(), erstelleEinheitenKarte()]); const [laeufe, prognose, einheiten, tage] = await Promise.all([api("/usage?limit=100"), erstellePrognoseKarte(), erstelleEinheitenKarte(), erstelleTageswerteKarte()]);
container.replaceChildren(); container.replaceChildren();
if (einheiten) container.append(einheiten); if (einheiten) container.append(einheiten);
if (prognose) container.append(prognose); if (prognose) container.append(prognose);
if (tage) container.append(tage);
if (!laeufe || laeufe.length === 0) { if (!laeufe || laeufe.length === 0) {
container.append( container.append(
@@ -218,6 +220,81 @@ async function erstelleEinheitenKarte() {
return erstelleKarte(t("usage.units.title"), inhalt); return erstelleKarte(t("usage.units.title"), inhalt);
} }
// ─── Day values and the month archive (#700) ───
/**
* The last 30 days, summed per day over all agents — from the condensed day values, which stay
* when old runs move to the archive. Below: how long runs stay in the working database, and the
* archive files. Nothing is ever deleted.
*/
async function erstelleTageswerteKarte() {
let tage, archiv;
try {
[tage, archiv] = await Promise.all([api("/usage/days"), api("/usage/archive")]);
} catch {
return null;
}
const summe = new Map();
for (const d of tage ?? []) {
const s = summe.get(d.day) ?? { runs: 0, failed: 0, loopLimit: 0, cost: 0, calls: 0, errors: 0, staged: 0 };
s.runs += d.runs; s.failed += d.failed; s.loopLimit += d.loopLimit; s.cost += Number(d.costUsd);
s.calls += d.toolCalls; s.errors += d.toolErrors; s.staged += d.staged;
summe.set(d.day, s);
}
const inhalt = [erstelleElement("p", { klasse: "gedaempft text-klein", text: t("usage.days.hint") })];
if (summe.size === 0) {
inhalt.push(erstelleElement("p", { klasse: "gedaempft text-klein", text: t("usage.days.none") }));
} else {
const tabelle = erstelleElement("table", { klasse: "daten-tabelle" });
tabelle.append(erstelleElement("thead", {}, erstelleElement("tr", {},
...["usage.days.day", "usage.days.runs", "usage.days.failed", "usage.days.loopLimit", "usage.days.cost", "usage.days.calls", "usage.days.errors", "usage.days.staged"]
.map((k) => erstelleElement("th", { klasse: k === "usage.days.day" ? "" : "zahl", text: t(k) })))));
const koerper = erstelleElement("tbody");
for (const [tag, s] of [...summe.entries()].sort((a, b) => b[0].localeCompare(a[0]))) {
koerper.append(erstelleElement("tr", { klasse: s.failed + s.loopLimit > 0 ? "zeile-warnung" : undefined },
erstelleElement("td", { text: tag }),
erstelleElement("td", { klasse: "zahl", text: String(s.runs) }),
erstelleElement("td", { klasse: "zahl", text: String(s.failed) }),
erstelleElement("td", { klasse: "zahl", text: String(s.loopLimit) }),
erstelleElement("td", { klasse: "zahl", text: `${s.cost.toFixed(2)} $` }),
erstelleElement("td", { klasse: "zahl", text: String(s.calls) }),
erstelleElement("td", { klasse: "zahl", text: String(s.errors) }),
erstelleElement("td", { klasse: "zahl", text: String(s.staged) })));
}
tabelle.append(koerper);
inhalt.push(erstelleElement("div", { klasse: "tabelle-scroll" }, tabelle));
}
// How long runs stay in the working database — set by a human; 0 keeps everything there.
const tageFeld = erstelleElement("input", { type: "number", min: "0", max: "36500", step: "1", value: String(archiv?.hotDays ?? 180), "aria-label": t("usage.archive.hotDays") });
const sichern = erstelleElement("button", {
type: "button",
klasse: "schaltflaeche flach klein",
text: t("common.save"),
onclick: async () => {
try {
const r = await api("/usage/archive", { method: "PUT", body: { hotDays: Number(tageFeld.value) } });
zeigeMeldung(r?.message ?? t("common.saved"), "erfolg", 6000);
} catch (err) {
zeigeMeldung(err.message, "gefahr", 8000);
}
}
});
inhalt.push(
erstelleElement("h3", { klasse: "fall-verlauf-titel", text: t("usage.archive.title") }),
erstelleElement("p", { klasse: "gedaempft text-klein", text: t("usage.archive.hint") }),
erstelleElement("div", { klasse: "mcp-token-formular" },
erstelleElement("label", { text: t("usage.archive.hotDays") }), tageFeld, sichern));
const dateien = archiv?.files ?? [];
inhalt.push(erstelleElement("p", { klasse: "gedaempft text-klein", text: dateien.length === 0
? t("usage.archive.none")
: t("usage.archive.files", { list: dateien.map((f) => `${f.month} (${(f.bytes / 1048576).toFixed(1)} MB)`).join(", ") }) }));
return erstelleKarte(t("usage.days.title"), inhalt);
}
// ─── Kostenprognose (#595) ─── // ─── Kostenprognose (#595) ───
function getGrundlageText(basis, sample) { function getGrundlageText(basis, sample) {
@@ -0,0 +1,13 @@
namespace ClawdDotNet.Web.Contracts;
/// <summary>#700: One day of one agent, condensed.</summary>
public sealed record RunDayDto(
string Day, string AgentId, string UnitId, int Runs, int Failed, int LoopLimit,
long PromptTokens, long CompletionTokens, decimal CostUsd, int ToolCalls, int ToolErrors, int Denied, int Staged);
public sealed record AuditArchiveFileDto(string Month, string FileName, long Bytes);
/// <summary>#700: How long runs stay in the working database, and the month archives.</summary>
public sealed record AuditArchiveDto(int HotDays, IReadOnlyList<AuditArchiveFileDto> Files);
public sealed record AuditArchiveInputDto(int HotDays);
@@ -111,6 +111,34 @@ public static class AnalyticsEndpoints
return Results.Ok(new UsageFacetsDto(f.Agents, f.Models, f.Sources, f.Statuses)); return Results.Ok(new UsageFacetsDto(f.Agents, f.Models, f.Sources, f.Statuses));
}); });
// #700: day values per agent and unit — they stay when old runs move to the archive.
api.MapGet("/usage/days", async (AppHost host, CancellationToken ct, string? from = null, string? to = null, string? agent = null, string? unit = null) =>
{
if (host.AuditArchive is not { } archive)
return ApiEndpoints.Unavailable("No day values in this instance.");
var today = DateOnly.FromDateTime(DateTime.UtcNow);
var end = DateOnly.TryParse(to, System.Globalization.CultureInfo.InvariantCulture, out var t) ? t : today;
var start = DateOnly.TryParse(from, System.Globalization.CultureInfo.InvariantCulture, out var f) ? f : end.AddDays(-29);
var days = await archive.DaysAsync(start, end, agent, unit, ct);
return Results.Ok(days.Select(d => new RunDayDto(d.Day, d.AgentId, d.UnitId, d.Runs, d.Failed, d.LoopLimit,
d.PromptTokens, d.CompletionTokens, d.CostUsd, d.ToolCalls, d.ToolErrors, d.Denied, d.Staged)).ToList());
});
api.MapGet("/usage/archive", (AppHost host) => Results.Ok(new AuditArchiveDto(
host.Instance.AuditArchive.HotDays,
(host.AuditArchive?.Files() ?? []).Select(f => new AuditArchiveFileDto(f.Month, f.FileName, f.Bytes)).ToList())));
api.MapPut("/usage/archive", (AppHost host, AuditArchiveInputDto input) =>
{
if (input.HotDays is < 0 or > 36500)
return Results.BadRequest(new OutcomeDto(false, "Days between 0 (keep everything in the working database) and 36500."));
host.Instance.AuditArchive.HotDays = input.HotDays;
host.SaveInstance(new InstanceChange(InstanceChangeKind.Settings));
return Results.Ok(new OutcomeDto(true, input.HotDays == 0
? "Runs stay in the working database."
: $"Runs older than {input.HotDays} days move to the month archive. Nothing is deleted."));
});
api.MapGet("/runs/{runId}", async (AppHost host, string runId, CancellationToken ct) => api.MapGet("/runs/{runId}", async (AppHost host, string runId, CancellationToken ct) =>
{ {
if (host.Insights is not { } insights) if (host.Insights is not { } insights)
@@ -0,0 +1,124 @@
using ClawdDotNet.Core.Audit;
using ClawdDotNet.Core.Storage;
using ClawdDotNet.Core.Tests.Infrastructure;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
namespace ClawdDotNet.Core.Tests.Audit;
/// <summary>
/// #700: Audit and receipts are evidence — nothing is deleted. Old runs move from the working
/// database into one archive file per month and are still found; each day is condensed per
/// agent and unit first, and those values stay.
/// </summary>
public sealed class AuditArchiveTests : IDisposable
{
private readonly string _directory = Path.Combine(Path.GetTempPath(), "clawd-tests", "arc-" + Guid.NewGuid().ToString("N")[..8]);
private readonly SqliteStorage _storage;
private readonly SqliteAuditRepository _audit;
private readonly FakeTimeProvider _clock = new(new DateTimeOffset(2026, 10, 2, 9, 0, 0, TimeSpan.Zero));
private readonly AuditArchiveSettings _settings = new() { HotDays = 30 };
private readonly AuditArchive _archive;
public AuditArchiveTests()
{
_storage = new SqliteStorage(Path.Combine(_directory, "state.db"));
_audit = new SqliteAuditRepository(_storage);
_archive = new AuditArchive(_storage, () => _settings, agent => agent == "desk" ? "sales" : null, NullLogger.Instance, _clock);
}
public void Dispose()
{
SqliteStorage.ReleasePooledConnections(_directory);
try { Directory.Delete(_directory, recursive: true); }
catch { /* cleanup is secondary */ }
}
private async Task Run(string runId, string agent, DateTime at, string status = "Completed", decimal cost = 0.02m, params string[] callStatus)
{
foreach (var s in callStatus)
await _audit.AppendAsync(new AuditEntry
{
RunId = runId, AgentId = agent, Tool = "FileRW", Arguments = "{\"action\":\"read\"}",
Status = Enum.Parse<AuditStatus>(s), OccurredAt = at
}, default);
await _audit.RecordReceiptAsync(new RunReceipt
{
RunId = runId, AgentId = agent, Status = status, CostUsd = cost, CostIsKnown = true,
PromptTokens = 1000, CompletionTokens = 100, OccurredAt = at
}, default);
}
[Fact]
public async Task Each_past_day_is_condensed_per_agent_with_its_unit_and_today_waits()
{
var yesterday = new DateTime(2026, 10, 1, 12, 0, 0, DateTimeKind.Utc);
await Run("r1", "desk", yesterday, callStatus: ["Ok", "Error", "Staged"]);
await Run("r2", "desk", yesterday.AddHours(1), "LoopLimitExceeded", 0.05m, "Ok");
await Run("r3", "desk", yesterday.AddHours(2), "Failed");
await Run("r4", "desk", yesterday.AddHours(3), "BudgetExceeded", 0m);
await Run("r5", "other", yesterday.AddDays(-1), callStatus: ["Denied"]);
await Run("r6", "desk", new DateTime(2026, 10, 2, 8, 0, 0, DateTimeKind.Utc));
await _archive.CondenseAsync(default);
var days = await _archive.DaysAsync(new DateOnly(2026, 9, 1), new DateOnly(2026, 10, 2), null, null, default);
days.Select(d => (d.Day, d.AgentId)).ShouldBe([("2026-09-30", "other"), ("2026-10-01", "desk")]);
var desk = days.Single(d => d.AgentId == "desk");
(desk.UnitId, desk.Runs, desk.Failed, desk.LoopLimit).ShouldBe(("sales", 3, 1, 1), "a budget refusal is no run");
desk.CostUsd.ShouldBe(0.09m);
(desk.ToolCalls, desk.ToolErrors, desk.Staged).ShouldBe((3, 1, 1));
days.Single(d => d.AgentId == "other").Denied.ShouldBe(1);
(await _archive.DaysAsync(new DateOnly(2026, 9, 1), new DateOnly(2026, 10, 2), null, "sales", default)).Count.ShouldBe(1);
}
[Fact]
public async Task Old_runs_move_to_their_month_archive_are_still_found_and_nothing_is_lost()
{
await Run("aug", "desk", new DateTime(2026, 8, 20, 10, 0, 0, DateTimeKind.Utc), callStatus: ["Started", "Ok"]);
await Run("jul", "desk", new DateTime(2026, 7, 31, 10, 0, 0, DateTimeKind.Utc), callStatus: ["Ok"]);
await Run("now", "desk", new DateTime(2026, 9, 28, 10, 0, 0, DateTimeKind.Utc), callStatus: ["Ok"]);
var moved = await _archive.ArchiveAsync(default);
moved.ShouldBe(5, "three audit rows and two receipts");
_archive.Files().Select(f => f.Month).ShouldBe(["2026-08", "2026-07"]);
(await _audit.CountAsync(default)).ShouldBe(1, "only the recent run stays in the working table");
(await _audit.ListForRunAsync("aug", default)).Count.ShouldBe(2);
(await _audit.GetReceiptForRunAsync("jul", default))!.AgentId.ShouldBe("desk");
(await _audit.GetReceiptForRunAsync("now", default)).ShouldNotBeNull();
(await _archive.DaysAsync(new DateOnly(2026, 7, 1), new DateOnly(2026, 8, 31), null, null, default))
.Select(d => d.Day).ShouldBe(["2026-07-31", "2026-08-20"], "the day values stay after the move");
(await _archive.ArchiveAsync(default)).ShouldBe(0, "a second pass moves nothing twice");
}
[Fact]
public async Task A_column_added_later_is_added_to_an_existing_archive()
{
await Run("a1", "desk", new DateTime(2026, 8, 1, 10, 0, 0, DateTimeKind.Utc));
await _archive.ArchiveAsync(default);
await _storage.WriteAsync(async conn =>
{
using var cmd = conn.CreateCommand();
cmd.CommandText = "ALTER TABLE RunReceipts ADD COLUMN LaterField TEXT NULL";
await cmd.ExecuteNonQueryAsync();
}, default);
await Run("a2", "desk", new DateTime(2026, 8, 2, 10, 0, 0, DateTimeKind.Utc));
(await _archive.ArchiveAsync(default)).ShouldBe(1);
(await _audit.GetReceiptForRunAsync("a2", default)).ShouldNotBeNull();
}
[Fact]
public async Task Zero_days_keeps_everything_in_the_working_database()
{
_settings.HotDays = 0;
await Run("old", "desk", new DateTime(2025, 1, 1, 10, 0, 0, DateTimeKind.Utc), callStatus: ["Ok"]);
(await _archive.ArchiveAsync(default)).ShouldBe(0);
_archive.Files().ShouldBeEmpty();
(await _audit.CountAsync(default)).ShouldBe(1);
}
}