Versionierung der Arbeitsbereiche: Verlauf je Datei mit Agent, Lauf und Vorfassung (#719)
Eigene inhaltsadressierte Ablage (Agents/.history/objects, SHA-256) statt Git, Journal FileHistory in state.db. Jeder Schreibweg - FileRW write/append/copy/ delete/stock_add, FTP- und RocketChat-Downloads - legt die alte Fassung ab und verbucht die Aenderung mit RunId. Externe Aenderungen findet ein Abgleich beim Start und stuendlich. Aufbewahrung nach Tagen und Fassungen, verwaiste Objekte werden entfernt. WebUI: Dateiverlauf mit Filter, Diff und Wiederherstellen nur durch den Menschen mit Grund. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Richard
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
621a83de43
commit
e9c39c3c96
@@ -155,6 +155,10 @@ public sealed class InstanceConfig
|
||||
[JsonPropertyName("claudeCode")]
|
||||
public Runtimes.ClaudeCodeSettings ClaudeCode { get; set; } = new();
|
||||
|
||||
/// <summary>#719: The history of the workspaces — on by default, 90 days.</summary>
|
||||
[JsonPropertyName("fileHistory")]
|
||||
public Files.FileHistorySettings FileHistory { get; set; } = new();
|
||||
|
||||
[JsonPropertyName("maxDelegationDepth")]
|
||||
public int MaxDelegationDepth { get; set; } = 3;
|
||||
|
||||
|
||||
@@ -63,6 +63,9 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
|
||||
/// <summary>#720: Which version of a shared file each agent read — so a write cannot silently lose another.</summary>
|
||||
public Files.FileLedger? Files { get; set; }
|
||||
|
||||
/// <summary>#719: The history of the workspaces — who changed which file in which run, and what was in it before.</summary>
|
||||
public Files.FileHistory? History { get; set; }
|
||||
|
||||
/// <summary>#654: The sandbox tools run code in.</summary>
|
||||
public Sandbox.ISandboxRunner? Sandbox { get; set; }
|
||||
|
||||
@@ -1860,6 +1863,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut
|
||||
SpecialBudgets = SpecialBudgets,
|
||||
Sandbox = Sandbox,
|
||||
Files = Files,
|
||||
History = History,
|
||||
// #697: Where something an agent hands on came from — known to the engine, not claimed by the agent.
|
||||
RunId = runId,
|
||||
RunTaskId = runId is null ? null : Activity.Find(runId)?.TaskId
|
||||
|
||||
@@ -0,0 +1,513 @@
|
||||
using System.Globalization;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json.Serialization;
|
||||
using ClawdDotNet.Core.Storage;
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace ClawdDotNet.Core.Files;
|
||||
|
||||
/// <summary>#719: How long the history of the workspaces is kept (<c>fileHistory</c> in <c>instance.json</c>).</summary>
|
||||
public sealed class FileHistorySettings
|
||||
{
|
||||
[JsonPropertyName("enabled")]
|
||||
public bool Enabled { get; set; } = true;
|
||||
|
||||
/// <summary>Entries older than this are removed — the newest one of each file always stays.</summary>
|
||||
[JsonPropertyName("retentionDays")]
|
||||
public int RetentionDays { get; set; } = 90;
|
||||
|
||||
/// <summary>At most this many versions per file.</summary>
|
||||
[JsonPropertyName("maxVersionsPerFile")]
|
||||
public int MaxVersionsPerFile { get; set; } = 50;
|
||||
|
||||
/// <summary>Larger files are journaled with their hash, but their content is not kept.</summary>
|
||||
[JsonPropertyName("maxObjectMegabytes")]
|
||||
public int MaxObjectMegabytes { get; set; } = 20;
|
||||
|
||||
/// <summary>How often changes made outside the tools are looked for.</summary>
|
||||
[JsonPropertyName("reconcileMinutes")]
|
||||
public int ReconcileMinutes { get; set; } = 60;
|
||||
}
|
||||
|
||||
/// <summary>#719: What happened to a file.</summary>
|
||||
public static class FileAction
|
||||
{
|
||||
public const string Create = "create";
|
||||
public const string Replace = "replace";
|
||||
public const string Append = "append";
|
||||
public const string Copy = "copy";
|
||||
public const string Delete = "delete";
|
||||
public const string Restore = "restore";
|
||||
|
||||
/// <summary>Changed outside every tool — a human at the file system, found by the reconciliation.</summary>
|
||||
public const string External = "external";
|
||||
}
|
||||
|
||||
/// <summary>#719: One line of the history. <see cref="Stored"/>: the version after it can be shown and restored.</summary>
|
||||
public sealed record FileHistoryEntry(
|
||||
long Id,
|
||||
string Workspace,
|
||||
string Path,
|
||||
string Action,
|
||||
string? AgentId,
|
||||
string? RunId,
|
||||
string? HashBefore,
|
||||
string? HashAfter,
|
||||
long? SizeAfter,
|
||||
bool Stored,
|
||||
DateTime OccurredAtUtc,
|
||||
string? Note);
|
||||
|
||||
/// <summary>#719: A change in progress: the version before, captured before the tool touches the file.</summary>
|
||||
public sealed record FileChange(string FullPath, string Workspace, string? HashBefore);
|
||||
|
||||
/// <summary>
|
||||
/// #719: The history of the workspaces — who created, replaced, appended to or deleted which
|
||||
/// file, in which run, and what was in it before. For post-mortems of failed work and for
|
||||
/// data that does not add up.
|
||||
///
|
||||
/// <para><b>Our own store, not Git.</b> Git would bring diff and history ready-made, but a
|
||||
/// commit per write with many agents writing in parallel means locks and waiting; the link
|
||||
/// to the run and the audit would be extra work; binary files would bloat it for good; and
|
||||
/// removing what is past its retention means rewriting history. This store fits the pattern
|
||||
/// the instance already has: SQLite as the truth, append-only, everything in the backup.</para>
|
||||
///
|
||||
/// <para><b>Each content once.</b> Versions are kept under their SHA-256 in
|
||||
/// <c>Agents/.history/objects/</c>, so an unchanged file costs nothing and an append costs
|
||||
/// one version. The journal in <c>state.db</c> links every change to its run — and through
|
||||
/// it to the receipt and the transcript. No <c>.bak</c> or <c>.tmp</c> names: the backup
|
||||
/// leaves those out.</para>
|
||||
///
|
||||
/// <para><b>One way in.</b> Every tool that changes a file calls <see cref="BeforeAsync"/>
|
||||
/// before and <see cref="RecordAsync"/> after, under the file's lock. What changes outside
|
||||
/// the tools is found by <see cref="ReconcileAsync"/> and recorded as external.</para>
|
||||
/// </summary>
|
||||
public sealed class FileHistory
|
||||
{
|
||||
public const string FolderName = ".history";
|
||||
|
||||
private readonly string _agentsDirectory;
|
||||
private readonly SqliteStorage _storage;
|
||||
private readonly Func<FileHistorySettings> _settings;
|
||||
private readonly TimeProvider _clock;
|
||||
|
||||
public FileHistory(string agentsDirectory, SqliteStorage storage, Func<FileHistorySettings>? settings = null, TimeProvider? clock = null)
|
||||
{
|
||||
_agentsDirectory = Path.GetFullPath(agentsDirectory);
|
||||
_storage = storage;
|
||||
_settings = settings ?? (() => new FileHistorySettings());
|
||||
_clock = clock ?? TimeProvider.System;
|
||||
}
|
||||
|
||||
private string ObjectsDirectory => Path.Combine(_agentsDirectory, FolderName, "objects");
|
||||
|
||||
private bool Enabled => _settings().Enabled;
|
||||
|
||||
/// <summary>The workspace's own files — never the history itself.</summary>
|
||||
public bool Covers(string fullPath)
|
||||
=> !IsInside(Path.GetFullPath(fullPath), Path.Combine(_agentsDirectory, FolderName));
|
||||
|
||||
// ─── Recording ───
|
||||
|
||||
/// <summary>Keeps the version the file has now, before a tool changes it.</summary>
|
||||
public Task<FileChange> BeforeAsync(string fullPath, string workspace, CancellationToken ct)
|
||||
{
|
||||
fullPath = Path.GetFullPath(fullPath);
|
||||
string? hash = null;
|
||||
if (Enabled && Covers(fullPath) && File.Exists(fullPath))
|
||||
hash = Store(fullPath).Hash;
|
||||
return Task.FromResult(new FileChange(fullPath, workspace, hash));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Journals what the tool did. A write that left the content as it was is not a change and
|
||||
/// is not recorded. The action is read from the file: gone is a delete, new is a create.
|
||||
/// </summary>
|
||||
public async Task<FileHistoryEntry?> RecordAsync(
|
||||
FileChange change, string action, string? agentId, string? runId, CancellationToken ct, string? note = null)
|
||||
{
|
||||
if (!Enabled || !Covers(change.FullPath))
|
||||
return null;
|
||||
|
||||
string? after = null;
|
||||
long? size = null;
|
||||
var stored = false;
|
||||
if (File.Exists(change.FullPath))
|
||||
{
|
||||
(after, stored) = Store(change.FullPath);
|
||||
size = new FileInfo(change.FullPath).Length;
|
||||
}
|
||||
|
||||
if (after == change.HashBefore && action != FileAction.Restore)
|
||||
return null;
|
||||
if (after is null)
|
||||
action = FileAction.Delete;
|
||||
else if (change.HashBefore is null && action is FileAction.Replace or FileAction.Append)
|
||||
action = FileAction.Create;
|
||||
|
||||
var entry = new FileHistoryEntry(0, change.Workspace, Relative(change.FullPath), action, agentId, runId,
|
||||
change.HashBefore, after, size, stored, Now, note);
|
||||
return await AppendAsync(entry, change.FullPath, ct);
|
||||
}
|
||||
|
||||
private async Task<FileHistoryEntry> AppendAsync(FileHistoryEntry entry, string fullPath, CancellationToken ct)
|
||||
{
|
||||
var id = await _storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
INSERT INTO FileHistory (Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note)
|
||||
VALUES (@ws, @path, @action, @agent, @run, @before, @after, @size, @stored, @at, @note);
|
||||
SELECT last_insert_rowid();
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@ws", entry.Workspace);
|
||||
cmd.Parameters.AddWithValue("@path", entry.Path);
|
||||
cmd.Parameters.AddWithValue("@action", entry.Action);
|
||||
cmd.Parameters.AddWithValue("@agent", (object?)entry.AgentId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@run", (object?)entry.RunId ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@before", (object?)entry.HashBefore ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@after", (object?)entry.HashAfter ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@size", (object?)entry.SizeAfter ?? DBNull.Value);
|
||||
cmd.Parameters.AddWithValue("@stored", entry.Stored ? 1 : 0);
|
||||
cmd.Parameters.AddWithValue("@at", Format(entry.OccurredAtUtc));
|
||||
cmd.Parameters.AddWithValue("@note", (object?)entry.Note ?? DBNull.Value);
|
||||
var rowId = (long)(await cmd.ExecuteScalarAsync(ct))!;
|
||||
await SetStateAsync(conn, entry.Path, entry.Workspace, fullPath, entry.HashAfter, ct);
|
||||
return rowId;
|
||||
}, ct);
|
||||
return entry with { Id = id };
|
||||
}
|
||||
|
||||
/// <summary>What the history knows the file to be now — the reconciliation compares against it.</summary>
|
||||
private static async Task SetStateAsync(SqliteConnection conn, string path, string workspace, string fullPath, string? hash, CancellationToken ct)
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
if (hash is null)
|
||||
{
|
||||
cmd.CommandText = "DELETE FROM FileHistoryState WHERE Path = @path";
|
||||
cmd.Parameters.AddWithValue("@path", path);
|
||||
}
|
||||
else
|
||||
{
|
||||
var info = new FileInfo(fullPath);
|
||||
cmd.CommandText = """
|
||||
INSERT INTO FileHistoryState (Path, Workspace, Hash, Size, Modified) VALUES (@path, @ws, @hash, @size, @mod)
|
||||
ON CONFLICT(Path) DO UPDATE SET Workspace = excluded.Workspace, Hash = excluded.Hash, Size = excluded.Size, Modified = excluded.Modified
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@path", path);
|
||||
cmd.Parameters.AddWithValue("@ws", workspace);
|
||||
cmd.Parameters.AddWithValue("@hash", hash);
|
||||
cmd.Parameters.AddWithValue("@size", info.Exists ? info.Length : 0);
|
||||
cmd.Parameters.AddWithValue("@mod", info.Exists ? info.LastWriteTimeUtc.Ticks : 0);
|
||||
}
|
||||
await cmd.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
/// <summary>Stores the file's content under its hash, once. Too large: the hash only.</summary>
|
||||
private (string Hash, bool Stored) Store(string fullPath)
|
||||
{
|
||||
string hash;
|
||||
using (var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
|
||||
hash = Convert.ToHexStringLower(SHA256.HashData(stream));
|
||||
|
||||
var target = ObjectPath(hash);
|
||||
if (File.Exists(target))
|
||||
return (hash, true);
|
||||
if (new FileInfo(fullPath).Length > (long)_settings().MaxObjectMegabytes * 1024 * 1024)
|
||||
return (hash, false);
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
var temp = target + "." + Guid.NewGuid().ToString("N")[..8] + ".part";
|
||||
File.Copy(fullPath, temp, overwrite: true);
|
||||
try
|
||||
{
|
||||
File.Move(temp, target, overwrite: false);
|
||||
}
|
||||
catch (IOException) when (File.Exists(target))
|
||||
{
|
||||
File.Delete(temp); // someone else stored the same content meanwhile
|
||||
}
|
||||
return (hash, true);
|
||||
}
|
||||
|
||||
private string ObjectPath(string hash) => Path.Combine(ObjectsDirectory, hash[..2], hash);
|
||||
|
||||
// ─── Reading ───
|
||||
|
||||
/// <summary>The newest entries, filtered — for the history view.</summary>
|
||||
public async Task<IReadOnlyList<FileHistoryEntry>> ListAsync(
|
||||
string? workspace, string? path, string? agentId, string? runId, int limit, CancellationToken ct, long? beforeId = null)
|
||||
{
|
||||
await using var conn = await _storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
var where = new List<string>();
|
||||
if (!string.IsNullOrWhiteSpace(workspace)) { where.Add("Workspace = @ws COLLATE NOCASE"); cmd.Parameters.AddWithValue("@ws", workspace); }
|
||||
if (!string.IsNullOrWhiteSpace(path)) { where.Add("Path = @path"); cmd.Parameters.AddWithValue("@path", path); }
|
||||
if (!string.IsNullOrWhiteSpace(agentId)) { where.Add("AgentId = @agent COLLATE NOCASE"); cmd.Parameters.AddWithValue("@agent", agentId); }
|
||||
if (!string.IsNullOrWhiteSpace(runId)) { where.Add("RunId = @run"); cmd.Parameters.AddWithValue("@run", runId); }
|
||||
if (beforeId is { } before) { where.Add("Id < @beforeId"); cmd.Parameters.AddWithValue("@beforeId", before); }
|
||||
cmd.CommandText = $"""
|
||||
SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note
|
||||
FROM FileHistory
|
||||
{(where.Count > 0 ? "WHERE " + string.Join(" AND ", where) : "")}
|
||||
ORDER BY Id DESC
|
||||
LIMIT @limit
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@limit", Math.Clamp(limit, 1, 1000));
|
||||
|
||||
var result = new List<FileHistoryEntry>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await reader.ReadAsync(ct))
|
||||
result.Add(Read(reader));
|
||||
return result;
|
||||
}
|
||||
|
||||
public async Task<FileHistoryEntry?> FindAsync(long id, CancellationToken ct)
|
||||
{
|
||||
await using var conn = await _storage.OpenConnectionAsync(ct);
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note
|
||||
FROM FileHistory WHERE Id = @id
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@id", id);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
return await reader.ReadAsync(ct) ? Read(reader) : null;
|
||||
}
|
||||
|
||||
/// <summary>A stored version's bytes, or <c>null</c> if it was not kept (too large, or pruned).</summary>
|
||||
public byte[]? Content(string? hash)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hash) || hash.Length != 64 || !hash.All(Uri.IsHexDigit))
|
||||
return null;
|
||||
var path = ObjectPath(hash.ToLowerInvariant());
|
||||
return File.Exists(path) ? File.ReadAllBytes(path) : null;
|
||||
}
|
||||
|
||||
/// <summary>The file on disk that a journal path stands for.</summary>
|
||||
public string FullPathOf(string relativePath)
|
||||
=> Path.GetFullPath(Path.IsPathRooted(relativePath) ? relativePath : Path.Combine(_agentsDirectory, relativePath));
|
||||
|
||||
// ─── Restoring ───
|
||||
|
||||
/// <summary>
|
||||
/// Puts a version back — by a human only, with a reason, as a journal line of its own. The
|
||||
/// target must lie in a workspace the caller names; the history's own folder never is one.
|
||||
/// </summary>
|
||||
public async Task<FileHistoryEntry> RestoreAsync(
|
||||
FileHistoryEntry source, bool versionBefore, string by, string reason,
|
||||
IReadOnlyList<string> workspaceRoots, CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reason))
|
||||
throw new ArgumentException("A restore needs a reason.");
|
||||
var hash = versionBefore ? source.HashBefore : source.HashAfter;
|
||||
var content = Content(hash)
|
||||
?? throw new InvalidOperationException(hash is null
|
||||
? "That version is the file's absence — there is nothing to restore."
|
||||
: "That version was not kept (too large, or past its retention).");
|
||||
|
||||
var target = FullPathOf(source.Path);
|
||||
if (!Covers(target) || !workspaceRoots.Any(root => IsInside(target, Path.GetFullPath(root))))
|
||||
throw new InvalidOperationException("The file does not lie in a workspace.");
|
||||
|
||||
var change = await BeforeAsync(target, source.Workspace, ct);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
await AtomicFile.WriteAllBytesAsync(target, content, ct);
|
||||
return (await RecordAsync(change, FileAction.Restore, by, null, ct,
|
||||
$"Restored the version of entry {source.Id} ({(versionBefore ? "before" : "after")}): {reason.Trim()}"))!;
|
||||
}
|
||||
|
||||
// ─── Outside the tools ───
|
||||
|
||||
/// <summary>
|
||||
/// Looks for changes made outside the tools and journals them as external. The first time
|
||||
/// a file is seen, it is only noted — the whole existing workspace is no change. Size and
|
||||
/// modification time decide whether a file is hashed at all, so an unchanged tree costs a
|
||||
/// directory walk.
|
||||
/// </summary>
|
||||
public async Task<int> ReconcileAsync(IReadOnlyList<(string Workspace, string Root)> workspaces, CancellationToken ct)
|
||||
{
|
||||
if (!Enabled)
|
||||
return 0;
|
||||
|
||||
var known = new Dictionary<string, (string Workspace, string Hash, long Size, long Modified)>(StringComparer.Ordinal);
|
||||
await using (var conn = await _storage.OpenConnectionAsync(ct))
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT Path, Workspace, Hash, Size, Modified FROM FileHistoryState";
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await reader.ReadAsync(ct))
|
||||
known[reader.GetString(0)] = (reader.GetString(1), reader.GetString(2), reader.GetInt64(3), reader.GetInt64(4));
|
||||
}
|
||||
|
||||
var changes = 0;
|
||||
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||
var coveredRoots = new List<string>();
|
||||
foreach (var (workspace, root) in workspaces)
|
||||
{
|
||||
if (!Directory.Exists(root))
|
||||
continue;
|
||||
coveredRoots.Add(Path.GetFullPath(root));
|
||||
foreach (var file in RegularFiles(root))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (!Covers(file))
|
||||
continue;
|
||||
var relative = Relative(file);
|
||||
if (!seen.Add(relative))
|
||||
continue; // a workspace inside another — counted once
|
||||
|
||||
var info = new FileInfo(file);
|
||||
if (known.TryGetValue(relative, out var state)
|
||||
&& state.Size == info.Length && state.Modified == info.LastWriteTimeUtc.Ticks)
|
||||
continue;
|
||||
|
||||
if (state.Hash is null)
|
||||
{
|
||||
// Seen for the first time: noted, not a change.
|
||||
await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, HashOnly(file), ct), ct);
|
||||
continue;
|
||||
}
|
||||
|
||||
var (hash, stored) = Store(file);
|
||||
if (hash == state.Hash)
|
||||
{
|
||||
await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, hash, ct), ct);
|
||||
continue;
|
||||
}
|
||||
await AppendAsync(new FileHistoryEntry(0, workspace, relative, FileAction.External, null, null,
|
||||
state.Hash, hash, info.Length, stored, Now, null), file, ct);
|
||||
changes++;
|
||||
}
|
||||
}
|
||||
|
||||
// Known files that are gone — deleted outside the tools.
|
||||
foreach (var (relative, state) in known)
|
||||
{
|
||||
if (seen.Contains(relative))
|
||||
continue;
|
||||
var full = FullPathOf(relative);
|
||||
if (File.Exists(full) || !coveredRoots.Any(root => IsInside(full, root)))
|
||||
continue;
|
||||
await AppendAsync(new FileHistoryEntry(0, state.Workspace, relative, FileAction.External, null, null,
|
||||
state.Hash, null, null, false, Now, "deleted outside the tools"), full, ct);
|
||||
changes++;
|
||||
}
|
||||
return changes;
|
||||
}
|
||||
|
||||
private static string HashOnly(string file)
|
||||
{
|
||||
using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
|
||||
return Convert.ToHexStringLower(SHA256.HashData(stream));
|
||||
}
|
||||
|
||||
/// <summary>Regular files only — no links out of the workspace, no pipes that would block.</summary>
|
||||
private static IEnumerable<string> RegularFiles(string root)
|
||||
{
|
||||
var pending = new Stack<string>([Path.GetFullPath(root)]);
|
||||
while (pending.Count > 0)
|
||||
{
|
||||
var dir = pending.Pop();
|
||||
IEnumerable<FileSystemInfo> entries;
|
||||
try { entries = new DirectoryInfo(dir).EnumerateFileSystemInfos().ToList(); }
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; }
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
switch (Runtimes.FileKind.Of(entry))
|
||||
{
|
||||
case Runtimes.FileKind.Kind.Directory: pending.Push(entry.FullName); break;
|
||||
case Runtimes.FileKind.Kind.Regular: yield return entry.FullName; break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Retention ───
|
||||
|
||||
/// <summary>
|
||||
/// Removes entries past their retention and beyond the version limit — never a file's
|
||||
/// newest — and then every stored version no entry points to any more.
|
||||
/// </summary>
|
||||
public async Task<int> PurgeAsync(CancellationToken ct)
|
||||
{
|
||||
var settings = _settings();
|
||||
var cutoff = Format(Now.AddDays(-Math.Max(1, settings.RetentionDays)));
|
||||
var keep = Math.Max(1, settings.MaxVersionsPerFile);
|
||||
|
||||
var removed = await _storage.WriteAsync(async conn =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = """
|
||||
DELETE FROM FileHistory WHERE Id IN (
|
||||
SELECT Id FROM (
|
||||
SELECT Id, OccurredAt,
|
||||
ROW_NUMBER() OVER (PARTITION BY Path ORDER BY Id DESC) AS Newest
|
||||
FROM FileHistory)
|
||||
WHERE Newest > 1 AND (OccurredAt < @cutoff OR Newest > @keep))
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("@cutoff", cutoff);
|
||||
cmd.Parameters.AddWithValue("@keep", keep);
|
||||
return await cmd.ExecuteNonQueryAsync(ct);
|
||||
}, ct);
|
||||
|
||||
// Objects nobody points to. Collected after the deletion: a version stored meanwhile is
|
||||
// either referenced or younger than the walk, and the walk skips unfinished ".part" files.
|
||||
var referenced = new HashSet<string>(StringComparer.Ordinal);
|
||||
await using (var conn = await _storage.OpenConnectionAsync(ct))
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT HashBefore FROM FileHistory UNION SELECT HashAfter FROM FileHistory UNION SELECT Hash FROM FileHistoryState";
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct);
|
||||
while (await reader.ReadAsync(ct))
|
||||
if (!reader.IsDBNull(0))
|
||||
referenced.Add(reader.GetString(0));
|
||||
}
|
||||
if (Directory.Exists(ObjectsDirectory))
|
||||
{
|
||||
var threshold = Now.AddMinutes(-10);
|
||||
foreach (var file in Directory.EnumerateFiles(ObjectsDirectory, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var name = Path.GetFileName(file);
|
||||
if (name.Length != 64 || referenced.Contains(name) || File.GetLastWriteTimeUtc(file) > threshold)
|
||||
continue;
|
||||
try { File.Delete(file); }
|
||||
catch (IOException) { }
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
// ─── Helpers ───
|
||||
|
||||
private DateTime Now => _clock.GetUtcNow().UtcDateTime;
|
||||
|
||||
private static string Format(DateTime utc) => utc.ToString("O", CultureInfo.InvariantCulture);
|
||||
|
||||
/// <summary>Relative to the Agents folder with '/', or the full path for a workspace elsewhere.</summary>
|
||||
private string Relative(string fullPath)
|
||||
{
|
||||
var relative = Path.GetRelativePath(_agentsDirectory, fullPath);
|
||||
return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)
|
||||
? fullPath
|
||||
: relative.Replace(Path.DirectorySeparatorChar, '/');
|
||||
}
|
||||
|
||||
private static bool IsInside(string path, string root)
|
||||
{
|
||||
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
root = Path.TrimEndingDirectorySeparator(root);
|
||||
return path.Equals(root, comparison) || path.StartsWith(root + Path.DirectorySeparatorChar, comparison);
|
||||
}
|
||||
|
||||
private static FileHistoryEntry Read(SqliteDataReader r) => new(
|
||||
r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3),
|
||||
r.IsDBNull(4) ? null : r.GetString(4),
|
||||
r.IsDBNull(5) ? null : r.GetString(5),
|
||||
r.IsDBNull(6) ? null : r.GetString(6),
|
||||
r.IsDBNull(7) ? null : r.GetString(7),
|
||||
r.IsDBNull(8) ? null : r.GetInt64(8),
|
||||
r.GetInt32(9) != 0,
|
||||
DateTime.Parse(r.GetString(10), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind),
|
||||
r.IsDBNull(11) ? null : r.GetString(11));
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using ClawdDotNet.Core.Config;
|
||||
using ClawdDotNet.Core.Organisation;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Core.Files;
|
||||
|
||||
/// <summary>
|
||||
/// #719: Keeps the history in order — at start and then at the configured interval it looks
|
||||
/// for changes made outside the tools and removes what is past its retention.
|
||||
/// </summary>
|
||||
public sealed class FileHistoryKeeper(
|
||||
FileHistory history,
|
||||
Func<IReadOnlyList<(string Workspace, string Root)>> workspaces,
|
||||
Func<FileHistorySettings> settings,
|
||||
ILogger logger) : IAsyncDisposable
|
||||
{
|
||||
private readonly CancellationTokenSource _cts = new();
|
||||
private Task? _loop;
|
||||
|
||||
/// <summary>Every workspace of the instance: each agent's own, the shared one, and those with member lists.</summary>
|
||||
public static IReadOnlyList<(string Workspace, string Root)> WorkspacesOf(IReadOnlyList<AgentConfig> agents, OrgChart organisation)
|
||||
{
|
||||
var result = new List<(string, string)>();
|
||||
foreach (var agent in agents.Where(a => !string.IsNullOrWhiteSpace(a.WorkspacePath)))
|
||||
result.Add(($"{WorkspaceLayout.Personal}:{agent.AgentId}", agent.WorkspacePath));
|
||||
|
||||
var shared = agents.Select(a => a.SharedWorkspacePath).FirstOrDefault(p => !string.IsNullOrWhiteSpace(p));
|
||||
if (shared is not null)
|
||||
{
|
||||
result.Add((WorkspaceLayout.Shared, shared));
|
||||
foreach (var workspace in organisation.Workspaces)
|
||||
if (WorkspaceLayout.PathOf(shared, workspace.Id) is { } root)
|
||||
result.Add((workspace.Id, root));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public void Start() => _loop ??= RunAsync(_cts.Token);
|
||||
|
||||
public async Task RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
var external = await history.ReconcileAsync(workspaces(), ct);
|
||||
if (external > 0)
|
||||
logger.LogInformation("File history: {Count} change(s) made outside the tools recorded", external);
|
||||
var removed = await history.PurgeAsync(ct);
|
||||
if (removed > 0)
|
||||
logger.LogInformation("File history: {Count} entr(y/ies) past their retention removed", removed);
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
// Not in the way of the start: the first walk of a large tree may take a while.
|
||||
try { await Task.Delay(TimeSpan.FromSeconds(30), ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await RunOnceAsync(ct);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(ex, "File history: reconciliation or cleanup failed");
|
||||
}
|
||||
|
||||
try { await Task.Delay(TimeSpan.FromMinutes(Math.Clamp(settings().ReconcileMinutes, 5, 24 * 60)), ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
if (_loop is not null)
|
||||
{
|
||||
try { await _loop; }
|
||||
catch (OperationCanceledException) { }
|
||||
}
|
||||
_cts.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -374,6 +374,34 @@ public sealed class SqliteStorage
|
||||
Json TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- #719: The history of the workspaces. Append-only; only the retention removes lines,
|
||||
-- never a file's newest. Path is relative to the Agents folder.
|
||||
CREATE TABLE IF NOT EXISTS FileHistory (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
Workspace TEXT NOT NULL,
|
||||
Path TEXT NOT NULL,
|
||||
Action TEXT NOT NULL,
|
||||
AgentId TEXT NULL,
|
||||
RunId TEXT NULL,
|
||||
HashBefore TEXT NULL,
|
||||
HashAfter TEXT NULL,
|
||||
SizeAfter INTEGER NULL,
|
||||
Stored INTEGER NOT NULL DEFAULT 0,
|
||||
OccurredAt TEXT NOT NULL,
|
||||
Note TEXT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS IX_FileHistory_Path ON FileHistory (Path, Id);
|
||||
CREATE INDEX IF NOT EXISTS IX_FileHistory_Run ON FileHistory (RunId);
|
||||
|
||||
-- #719: What the history knows each file to be now — the reconciliation compares against it.
|
||||
CREATE TABLE IF NOT EXISTS FileHistoryState (
|
||||
Path TEXT PRIMARY KEY,
|
||||
Workspace TEXT NOT NULL,
|
||||
Hash TEXT NOT NULL,
|
||||
Size INTEGER NOT NULL,
|
||||
Modified INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- #720: Writes refused because they would have lost someone's change.
|
||||
CREATE TABLE IF NOT EXISTS FileConflicts (
|
||||
Id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
|
||||
@@ -61,6 +61,12 @@ public sealed record AgentToolContext(
|
||||
/// </summary>
|
||||
public Files.FileLedger? Files { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// #719: The history of the workspaces. Tools that change a file keep the version before
|
||||
/// and journal the change, with this call's agent and run. <c>null</c>: no history.
|
||||
/// </summary>
|
||||
public Files.FileHistory? History { get; init; }
|
||||
|
||||
/// <summary>#697: The run this call belongs to — for stamping where something came from.</summary>
|
||||
public string? RunId { get; init; }
|
||||
|
||||
|
||||
Reference in New Issue
Block a user