using System.Globalization; using System.Security.Cryptography; using System.Text.Json.Serialization; using ClawdDotNet.Core.Storage; using Microsoft.Data.Sqlite; namespace ClawdDotNet.Core.Files; /// #719: How long the history of the workspaces is kept (fileHistory in instance.json). public sealed class FileHistorySettings { [JsonPropertyName("enabled")] public bool Enabled { get; set; } = true; /// Entries older than this are removed — the newest one of each file always stays. [JsonPropertyName("retentionDays")] public int RetentionDays { get; set; } = 90; /// At most this many versions per file. [JsonPropertyName("maxVersionsPerFile")] public int MaxVersionsPerFile { get; set; } = 50; /// Larger files are journaled with their hash, but their content is not kept. [JsonPropertyName("maxObjectMegabytes")] public int MaxObjectMegabytes { get; set; } = 20; /// How often changes made outside the tools are looked for. [JsonPropertyName("reconcileMinutes")] public int ReconcileMinutes { get; set; } = 60; } /// #719: What happened to a file. public static class FileAction { public const string Create = "create"; public const string Replace = "replace"; public const string Append = "append"; public const string Copy = "copy"; public const string Delete = "delete"; public const string Restore = "restore"; /// Changed outside every tool — a human at the file system, found by the reconciliation. public const string External = "external"; } /// #719: One line of the history. : the version after it can be shown and restored. public sealed record FileHistoryEntry( long Id, string Workspace, string Path, string Action, string? AgentId, string? RunId, string? HashBefore, string? HashAfter, long? SizeAfter, bool Stored, DateTime OccurredAtUtc, string? Note); /// #719: A change in progress: the version before, captured before the tool touches the file. public sealed record FileChange(string FullPath, string Workspace, string? HashBefore); /// /// #719: The history of the workspaces — who created, replaced, appended to or deleted which /// file, in which run, and what was in it before. For post-mortems of failed work and for /// data that does not add up. /// /// Our own store, not Git. Git would bring diff and history ready-made, but a /// commit per write with many agents writing in parallel means locks and waiting; the link /// to the run and the audit would be extra work; binary files would bloat it for good; and /// removing what is past its retention means rewriting history. This store fits the pattern /// the instance already has: SQLite as the truth, append-only, everything in the backup. /// /// Each content once. Versions are kept under their SHA-256 in /// Agents/.history/objects/, so an unchanged file costs nothing and an append costs /// one version. The journal in state.db links every change to its run — and through /// it to the receipt and the transcript. No .bak or .tmp names: the backup /// leaves those out. /// /// One way in. Every tool that changes a file calls /// before and after, under the file's lock. What changes outside /// the tools is found by and recorded as external. /// public sealed class FileHistory { public const string FolderName = ".history"; private readonly string _agentsDirectory; private readonly SqliteStorage _storage; private readonly Func _settings; private readonly TimeProvider _clock; public FileHistory(string agentsDirectory, SqliteStorage storage, Func? settings = null, TimeProvider? clock = null) { _agentsDirectory = Path.GetFullPath(agentsDirectory); _storage = storage; _settings = settings ?? (() => new FileHistorySettings()); _clock = clock ?? TimeProvider.System; } private string ObjectsDirectory => Path.Combine(_agentsDirectory, FolderName, "objects"); private bool Enabled => _settings().Enabled; /// The workspace's own files — never the history itself. public bool Covers(string fullPath) => !IsInside(Path.GetFullPath(fullPath), Path.Combine(_agentsDirectory, FolderName)); /// /// #195: Whether a delete of this file can be undone — the history is on, covers the file /// and keeps content of its size. Only then is deleting it no irreversible act. /// public bool CanKeep(string fullPath) { fullPath = Path.GetFullPath(fullPath); return Enabled && Covers(fullPath) && File.Exists(fullPath) && new FileInfo(fullPath).Length <= (long)_settings().MaxObjectMegabytes * 1024 * 1024; } /// #195: Whether the content of a version is still kept. public bool CanKeepContent(string? hash) => !string.IsNullOrWhiteSpace(hash) && hash.Length == 64 && hash.All(Uri.IsHexDigit) && File.Exists(ObjectPath(hash.ToLowerInvariant())); /// #195: How long a deleted file's content stays restorable. public int RetentionDays => Math.Max(1, _settings().RetentionDays); // ─── Recording ─── /// Keeps the version the file has now, before a tool changes it. public Task BeforeAsync(string fullPath, string workspace, CancellationToken ct) { fullPath = Path.GetFullPath(fullPath); string? hash = null; if (Enabled && Covers(fullPath) && File.Exists(fullPath)) hash = Store(fullPath).Hash; return Task.FromResult(new FileChange(fullPath, workspace, hash)); } /// /// Journals what the tool did. A write that left the content as it was is not a change and /// is not recorded. The action is read from the file: gone is a delete, new is a create. /// public async Task RecordAsync( FileChange change, string action, string? agentId, string? runId, CancellationToken ct, string? note = null) { if (!Enabled || !Covers(change.FullPath)) return null; string? after = null; long? size = null; var stored = false; if (File.Exists(change.FullPath)) { (after, stored) = Store(change.FullPath); size = new FileInfo(change.FullPath).Length; } if (after == change.HashBefore && action != FileAction.Restore) return null; if (after is null) action = FileAction.Delete; else if (change.HashBefore is null && action is FileAction.Replace or FileAction.Append) action = FileAction.Create; var entry = new FileHistoryEntry(0, change.Workspace, Relative(change.FullPath), action, agentId, runId, change.HashBefore, after, size, stored, Now, note); return await AppendAsync(entry, change.FullPath, ct); } private async Task AppendAsync(FileHistoryEntry entry, string fullPath, CancellationToken ct) { var id = await _storage.WriteAsync(async conn => { using var cmd = conn.CreateCommand(); cmd.CommandText = """ INSERT INTO FileHistory (Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note) VALUES (@ws, @path, @action, @agent, @run, @before, @after, @size, @stored, @at, @note); SELECT last_insert_rowid(); """; cmd.Parameters.AddWithValue("@ws", entry.Workspace); cmd.Parameters.AddWithValue("@path", entry.Path); cmd.Parameters.AddWithValue("@action", entry.Action); cmd.Parameters.AddWithValue("@agent", (object?)entry.AgentId ?? DBNull.Value); cmd.Parameters.AddWithValue("@run", (object?)entry.RunId ?? DBNull.Value); cmd.Parameters.AddWithValue("@before", (object?)entry.HashBefore ?? DBNull.Value); cmd.Parameters.AddWithValue("@after", (object?)entry.HashAfter ?? DBNull.Value); cmd.Parameters.AddWithValue("@size", (object?)entry.SizeAfter ?? DBNull.Value); cmd.Parameters.AddWithValue("@stored", entry.Stored ? 1 : 0); cmd.Parameters.AddWithValue("@at", Format(entry.OccurredAtUtc)); cmd.Parameters.AddWithValue("@note", (object?)entry.Note ?? DBNull.Value); var rowId = (long)(await cmd.ExecuteScalarAsync(ct))!; await SetStateAsync(conn, entry.Path, entry.Workspace, fullPath, entry.HashAfter, ct); return rowId; }, ct); return entry with { Id = id }; } /// What the history knows the file to be now — the reconciliation compares against it. private static async Task SetStateAsync(SqliteConnection conn, string path, string workspace, string fullPath, string? hash, CancellationToken ct) { using var cmd = conn.CreateCommand(); if (hash is null) { cmd.CommandText = "DELETE FROM FileHistoryState WHERE Path = @path"; cmd.Parameters.AddWithValue("@path", path); } else { var info = new FileInfo(fullPath); cmd.CommandText = """ INSERT INTO FileHistoryState (Path, Workspace, Hash, Size, Modified) VALUES (@path, @ws, @hash, @size, @mod) ON CONFLICT(Path) DO UPDATE SET Workspace = excluded.Workspace, Hash = excluded.Hash, Size = excluded.Size, Modified = excluded.Modified """; cmd.Parameters.AddWithValue("@path", path); cmd.Parameters.AddWithValue("@ws", workspace); cmd.Parameters.AddWithValue("@hash", hash); cmd.Parameters.AddWithValue("@size", info.Exists ? info.Length : 0); cmd.Parameters.AddWithValue("@mod", info.Exists ? info.LastWriteTimeUtc.Ticks : 0); } await cmd.ExecuteNonQueryAsync(ct); } /// Stores the file's content under its hash, once. Too large: the hash only. private (string Hash, bool Stored) Store(string fullPath) { string hash; using (var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) hash = Convert.ToHexStringLower(SHA256.HashData(stream)); var target = ObjectPath(hash); if (File.Exists(target)) return (hash, true); if (new FileInfo(fullPath).Length > (long)_settings().MaxObjectMegabytes * 1024 * 1024) return (hash, false); Directory.CreateDirectory(Path.GetDirectoryName(target)!); var temp = target + "." + Guid.NewGuid().ToString("N")[..8] + ".part"; File.Copy(fullPath, temp, overwrite: true); try { File.Move(temp, target, overwrite: false); } catch (IOException) when (File.Exists(target)) { File.Delete(temp); // someone else stored the same content meanwhile } return (hash, true); } private string ObjectPath(string hash) => Path.Combine(ObjectsDirectory, hash[..2], hash); // ─── Reading ─── /// The newest entries, filtered — for the history view. public async Task> ListAsync( string? workspace, string? path, string? agentId, string? runId, int limit, CancellationToken ct, long? beforeId = null) { await using var conn = await _storage.OpenConnectionAsync(ct); using var cmd = conn.CreateCommand(); var where = new List(); if (!string.IsNullOrWhiteSpace(workspace)) { where.Add("Workspace = @ws COLLATE NOCASE"); cmd.Parameters.AddWithValue("@ws", workspace); } if (!string.IsNullOrWhiteSpace(path)) { where.Add("Path = @path"); cmd.Parameters.AddWithValue("@path", path); } if (!string.IsNullOrWhiteSpace(agentId)) { where.Add("AgentId = @agent COLLATE NOCASE"); cmd.Parameters.AddWithValue("@agent", agentId); } if (!string.IsNullOrWhiteSpace(runId)) { where.Add("RunId = @run"); cmd.Parameters.AddWithValue("@run", runId); } if (beforeId is { } before) { where.Add("Id < @beforeId"); cmd.Parameters.AddWithValue("@beforeId", before); } cmd.CommandText = $""" SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note FROM FileHistory {(where.Count > 0 ? "WHERE " + string.Join(" AND ", where) : "")} ORDER BY Id DESC LIMIT @limit """; cmd.Parameters.AddWithValue("@limit", Math.Clamp(limit, 1, 1000)); var result = new List(); await using var reader = await cmd.ExecuteReaderAsync(ct); while (await reader.ReadAsync(ct)) result.Add(Read(reader)); return result; } public async Task FindAsync(long id, CancellationToken ct) { await using var conn = await _storage.OpenConnectionAsync(ct); using var cmd = conn.CreateCommand(); cmd.CommandText = """ SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note FROM FileHistory WHERE Id = @id """; cmd.Parameters.AddWithValue("@id", id); await using var reader = await cmd.ExecuteReaderAsync(ct); return await reader.ReadAsync(ct) ? Read(reader) : null; } /// A stored version's bytes, or null if it was not kept (too large, or pruned). public byte[]? Content(string? hash) { if (string.IsNullOrWhiteSpace(hash) || hash.Length != 64 || !hash.All(Uri.IsHexDigit)) return null; var path = ObjectPath(hash.ToLowerInvariant()); return File.Exists(path) ? File.ReadAllBytes(path) : null; } /// The file on disk that a journal path stands for. public string FullPathOf(string relativePath) => Path.GetFullPath(Path.IsPathRooted(relativePath) ? relativePath : Path.Combine(_agentsDirectory, relativePath)); // ─── Restoring ─── /// /// Puts a version back — by a human only, with a reason, as a journal line of its own. The /// target must lie in a workspace the caller names; the history's own folder never is one. /// public async Task RestoreAsync( FileHistoryEntry source, bool versionBefore, string by, string reason, IReadOnlyList workspaceRoots, CancellationToken ct) { if (string.IsNullOrWhiteSpace(reason)) throw new ArgumentException("A restore needs a reason."); var hash = versionBefore ? source.HashBefore : source.HashAfter; var content = Content(hash) ?? throw new InvalidOperationException(hash is null ? "That version is the file's absence — there is nothing to restore." : "That version was not kept (too large, or past its retention)."); var target = FullPathOf(source.Path); if (!Covers(target) || !workspaceRoots.Any(root => IsInside(target, Path.GetFullPath(root)))) throw new InvalidOperationException("The file does not lie in a workspace."); var change = await BeforeAsync(target, source.Workspace, ct); Directory.CreateDirectory(Path.GetDirectoryName(target)!); await AtomicFile.WriteAllBytesAsync(target, content, ct); return (await RecordAsync(change, FileAction.Restore, by, null, ct, $"Restored the version of entry {source.Id} ({(versionBefore ? "before" : "after")}): {reason.Trim()}"))!; } // ─── Outside the tools ─── /// /// Looks for changes made outside the tools and journals them as external. The first time /// a file is seen, it is only noted — the whole existing workspace is no change. Size and /// modification time decide whether a file is hashed at all, so an unchanged tree costs a /// directory walk. /// public async Task ReconcileAsync(IReadOnlyList<(string Workspace, string Root)> workspaces, CancellationToken ct) { if (!Enabled) return 0; var known = new Dictionary(StringComparer.Ordinal); await using (var conn = await _storage.OpenConnectionAsync(ct)) { using var cmd = conn.CreateCommand(); cmd.CommandText = "SELECT Path, Workspace, Hash, Size, Modified FROM FileHistoryState"; await using var reader = await cmd.ExecuteReaderAsync(ct); while (await reader.ReadAsync(ct)) known[reader.GetString(0)] = (reader.GetString(1), reader.GetString(2), reader.GetInt64(3), reader.GetInt64(4)); } var changes = 0; var seen = new HashSet(StringComparer.Ordinal); var coveredRoots = new List(); foreach (var (workspace, root) in workspaces) { if (!Directory.Exists(root)) continue; coveredRoots.Add(Path.GetFullPath(root)); foreach (var file in RegularFiles(root)) { ct.ThrowIfCancellationRequested(); if (!Covers(file)) continue; var relative = Relative(file); if (!seen.Add(relative)) continue; // a workspace inside another — counted once var info = new FileInfo(file); if (known.TryGetValue(relative, out var state) && state.Size == info.Length && state.Modified == info.LastWriteTimeUtc.Ticks) continue; if (state.Hash is null) { // Seen for the first time: noted, not a change. await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, HashOnly(file), ct), ct); continue; } var (hash, stored) = Store(file); if (hash == state.Hash) { await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, hash, ct), ct); continue; } await AppendAsync(new FileHistoryEntry(0, workspace, relative, FileAction.External, null, null, state.Hash, hash, info.Length, stored, Now, null), file, ct); changes++; } } // Known files that are gone — deleted outside the tools. foreach (var (relative, state) in known) { if (seen.Contains(relative)) continue; var full = FullPathOf(relative); if (File.Exists(full) || !coveredRoots.Any(root => IsInside(full, root))) continue; await AppendAsync(new FileHistoryEntry(0, state.Workspace, relative, FileAction.External, null, null, state.Hash, null, null, false, Now, "deleted outside the tools"), full, ct); changes++; } return changes; } private static string HashOnly(string file) { using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); return Convert.ToHexStringLower(SHA256.HashData(stream)); } /// Regular files only — no links out of the workspace, no pipes that would block. private static IEnumerable RegularFiles(string root) { var pending = new Stack([Path.GetFullPath(root)]); while (pending.Count > 0) { var dir = pending.Pop(); IEnumerable entries; try { entries = new DirectoryInfo(dir).EnumerateFileSystemInfos().ToList(); } catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; } foreach (var entry in entries) { switch (Runtimes.FileKind.Of(entry)) { case Runtimes.FileKind.Kind.Directory: pending.Push(entry.FullName); break; case Runtimes.FileKind.Kind.Regular: yield return entry.FullName; break; } } } } // ─── Retention ─── /// /// Removes entries past their retention and beyond the version limit — never a file's /// newest — and then every stored version no entry points to any more. /// public async Task PurgeAsync(CancellationToken ct) { var settings = _settings(); var cutoff = Format(Now.AddDays(-Math.Max(1, settings.RetentionDays))); var keep = Math.Max(1, settings.MaxVersionsPerFile); var removed = await _storage.WriteAsync(async conn => { using var cmd = conn.CreateCommand(); cmd.CommandText = """ DELETE FROM FileHistory WHERE Id IN ( SELECT Id FROM ( SELECT Id, OccurredAt, ROW_NUMBER() OVER (PARTITION BY Path ORDER BY Id DESC) AS Newest FROM FileHistory) WHERE Newest > 1 AND (OccurredAt < @cutoff OR Newest > @keep)) """; cmd.Parameters.AddWithValue("@cutoff", cutoff); cmd.Parameters.AddWithValue("@keep", keep); return await cmd.ExecuteNonQueryAsync(ct); }, ct); // Objects nobody points to. Collected after the deletion: a version stored meanwhile is // either referenced or younger than the walk, and the walk skips unfinished ".part" files. var referenced = new HashSet(StringComparer.Ordinal); await using (var conn = await _storage.OpenConnectionAsync(ct)) { using var cmd = conn.CreateCommand(); cmd.CommandText = "SELECT HashBefore FROM FileHistory UNION SELECT HashAfter FROM FileHistory UNION SELECT Hash FROM FileHistoryState"; await using var reader = await cmd.ExecuteReaderAsync(ct); while (await reader.ReadAsync(ct)) if (!reader.IsDBNull(0)) referenced.Add(reader.GetString(0)); } if (Directory.Exists(ObjectsDirectory)) { var threshold = Now.AddMinutes(-10); foreach (var file in Directory.EnumerateFiles(ObjectsDirectory, "*", SearchOption.AllDirectories)) { var name = Path.GetFileName(file); if (name.Length != 64 || referenced.Contains(name) || File.GetLastWriteTimeUtc(file) > threshold) continue; try { File.Delete(file); } catch (IOException) { } } } return removed; } // ─── Helpers ─── private DateTime Now => _clock.GetUtcNow().UtcDateTime; private static string Format(DateTime utc) => utc.ToString("O", CultureInfo.InvariantCulture); /// Relative to the Agents folder with '/', or the full path for a workspace elsewhere. private string Relative(string fullPath) { var relative = Path.GetRelativePath(_agentsDirectory, fullPath); return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) ? fullPath : relative.Replace(Path.DirectorySeparatorChar, '/'); } private static bool IsInside(string path, string root) { var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; root = Path.TrimEndingDirectorySeparator(root); return path.Equals(root, comparison) || path.StartsWith(root + Path.DirectorySeparatorChar, comparison); } private static FileHistoryEntry Read(SqliteDataReader r) => new( r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3), r.IsDBNull(4) ? null : r.GetString(4), r.IsDBNull(5) ? null : r.GetString(5), r.IsDBNull(6) ? null : r.GetString(6), r.IsDBNull(7) ? null : r.GetString(7), r.IsDBNull(8) ? null : r.GetInt64(8), r.GetInt32(9) != 0, DateTime.Parse(r.GetString(10), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), r.IsDBNull(11) ? null : r.GetString(11)); }