From e9c39c3c96f85764eec7e4bfd53e18f3f4d068de Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 28 Sep 2026 11:12:13 +0200 Subject: [PATCH] Versionierung der Arbeitsbereiche: Verlauf je Datei mit Agent, Lauf und Vorfassung (#719) Eigene inhaltsadressierte Ablage (Agents/.history/objects, SHA-256) statt Git, Journal FileHistory in state.db. Jeder Schreibweg - FileRW write/append/copy/ delete/stock_add, FTP- und RocketChat-Downloads - legt die alte Fassung ab und verbucht die Aenderung mit RunId. Externe Aenderungen findet ein Abgleich beim Start und stuendlich. Aufbewahrung nach Tagen und Fassungen, verwaiste Objekte werden entfernt. WebUI: Dateiverlauf mit Filter, Diff und Wiederherstellen nur durch den Menschen mit Grund. Co-Authored-By: Claude Opus 5.5 --- src/ClawdDotNet.App/AppHost.cs | 13 + src/ClawdDotNet.Core/Config/InstanceConfig.cs | 4 + src/ClawdDotNet.Core/Engine/AgentEngine.cs | 4 + src/ClawdDotNet.Core/Files/FileHistory.cs | 513 ++++++++++++++++++ .../Files/FileHistoryKeeper.cs | 82 +++ src/ClawdDotNet.Core/Storage/SqliteStorage.cs | 28 + .../Tools/AgentToolContext.cs | 6 + src/ClawdDotNet.Manager.Web/wwwroot/app.css | 97 ++++ src/ClawdDotNet.Manager.Web/wwwroot/app.js | 2 + .../wwwroot/i18n/de.json | 46 +- .../wwwroot/i18n/en.json | 46 +- .../wwwroot/index.html | 6 + .../wwwroot/views/file-history.js | 258 +++++++++ src/ClawdDotNet.Tools.FTP/FtpTool.cs | 6 + src/ClawdDotNet.Tools.FileRW/FileRWTool.cs | 78 +++ .../RocketChatTool.cs | 6 + src/ClawdDotNet.Web/Contracts/Dtos.cs | 10 + src/ClawdDotNet.Web/Endpoints/ApiEndpoints.cs | 1 + .../Endpoints/FileHistoryEndpoints.cs | 87 +++ .../Files/FileHistoryTests.cs | 253 +++++++++ .../FileRW/FileHistoryRecordingTests.cs | 119 ++++ 21 files changed, 1663 insertions(+), 2 deletions(-) create mode 100644 src/ClawdDotNet.Core/Files/FileHistory.cs create mode 100644 src/ClawdDotNet.Core/Files/FileHistoryKeeper.cs create mode 100644 src/ClawdDotNet.Manager.Web/wwwroot/views/file-history.js create mode 100644 src/ClawdDotNet.Web/Endpoints/FileHistoryEndpoints.cs create mode 100644 tests/ClawdDotNet.Core.Tests/Files/FileHistoryTests.cs create mode 100644 tests/ClawdDotNet.Tools.Tests/FileRW/FileHistoryRecordingTests.cs diff --git a/src/ClawdDotNet.App/AppHost.cs b/src/ClawdDotNet.App/AppHost.cs index af0d59c..df74f20 100644 --- a/src/ClawdDotNet.App/AppHost.cs +++ b/src/ClawdDotNet.App/AppHost.cs @@ -601,6 +601,10 @@ public sealed class AppHost : IAsyncDisposable // #720: which version of a shared file each agent read — a write cannot silently lose another. engine.Files = new Core.Files.FileLedger(storage); + // #719: the history of the workspaces, next to them in Agents/.history. + var fileHistory = new Core.Files.FileHistory(Path.Combine(instancePath, "Agents"), storage, () => instance.FileHistory); + engine.History = fileHistory; + // #712: Foreign runtimes. The application itself is the launcher inside the sandbox // (network through the proxy, secrets from a file) and the MCP bridge back to us. var mcpGrants = new Core.Mcp.McpGrants(storage); @@ -690,6 +694,15 @@ public sealed class AppHost : IAsyncDisposable janitor.Start(); host._shutdown.Add(janitor.DisposeAsync); + // #719: changes made outside the tools, and the retention of the history. + var historyKeeper = new Core.Files.FileHistoryKeeper( + fileHistory, + () => Core.Files.FileHistoryKeeper.WorkspacesOf(instance.Agents, Core.Organisation.OrgChart.From(instance)), + () => instance.FileHistory, + loggerFactory.CreateLogger("ClawdDotNet.Core.Files")); + historyKeeper.Start(); + host._shutdown.Add(historyKeeper.DisposeAsync); + // #593: Angelegt wurde der Dienst schon immer, gestartet nie — weder kopflos noch // mit Fenster. Die Kopfzeile der Weboberflaeche stand deshalb dauerhaft auf // "OpenRouter: —" und /api/status auf "Prüfe…". diff --git a/src/ClawdDotNet.Core/Config/InstanceConfig.cs b/src/ClawdDotNet.Core/Config/InstanceConfig.cs index 2d85a94..d682d58 100644 --- a/src/ClawdDotNet.Core/Config/InstanceConfig.cs +++ b/src/ClawdDotNet.Core/Config/InstanceConfig.cs @@ -155,6 +155,10 @@ public sealed class InstanceConfig [JsonPropertyName("claudeCode")] public Runtimes.ClaudeCodeSettings ClaudeCode { get; set; } = new(); + /// #719: The history of the workspaces — on by default, 90 days. + [JsonPropertyName("fileHistory")] + public Files.FileHistorySettings FileHistory { get; set; } = new(); + [JsonPropertyName("maxDelegationDepth")] public int MaxDelegationDepth { get; set; } = 3; diff --git a/src/ClawdDotNet.Core/Engine/AgentEngine.cs b/src/ClawdDotNet.Core/Engine/AgentEngine.cs index aca11d7..a70b7a9 100644 --- a/src/ClawdDotNet.Core/Engine/AgentEngine.cs +++ b/src/ClawdDotNet.Core/Engine/AgentEngine.cs @@ -63,6 +63,9 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut /// #720: Which version of a shared file each agent read — so a write cannot silently lose another. public Files.FileLedger? Files { get; set; } + /// #719: The history of the workspaces — who changed which file in which run, and what was in it before. + public Files.FileHistory? History { get; set; } + /// #654: The sandbox tools run code in. public Sandbox.ISandboxRunner? Sandbox { get; set; } @@ -1860,6 +1863,7 @@ public sealed class AgentEngine : IAgentMessageRouter, Staging.IFrozenCallExecut SpecialBudgets = SpecialBudgets, Sandbox = Sandbox, Files = Files, + History = History, // #697: Where something an agent hands on came from — known to the engine, not claimed by the agent. RunId = runId, RunTaskId = runId is null ? null : Activity.Find(runId)?.TaskId diff --git a/src/ClawdDotNet.Core/Files/FileHistory.cs b/src/ClawdDotNet.Core/Files/FileHistory.cs new file mode 100644 index 0000000..43190fa --- /dev/null +++ b/src/ClawdDotNet.Core/Files/FileHistory.cs @@ -0,0 +1,513 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text.Json.Serialization; +using ClawdDotNet.Core.Storage; +using Microsoft.Data.Sqlite; + +namespace ClawdDotNet.Core.Files; + +/// #719: How long the history of the workspaces is kept (fileHistory in instance.json). +public sealed class FileHistorySettings +{ + [JsonPropertyName("enabled")] + public bool Enabled { get; set; } = true; + + /// Entries older than this are removed — the newest one of each file always stays. + [JsonPropertyName("retentionDays")] + public int RetentionDays { get; set; } = 90; + + /// At most this many versions per file. + [JsonPropertyName("maxVersionsPerFile")] + public int MaxVersionsPerFile { get; set; } = 50; + + /// Larger files are journaled with their hash, but their content is not kept. + [JsonPropertyName("maxObjectMegabytes")] + public int MaxObjectMegabytes { get; set; } = 20; + + /// How often changes made outside the tools are looked for. + [JsonPropertyName("reconcileMinutes")] + public int ReconcileMinutes { get; set; } = 60; +} + +/// #719: What happened to a file. +public static class FileAction +{ + public const string Create = "create"; + public const string Replace = "replace"; + public const string Append = "append"; + public const string Copy = "copy"; + public const string Delete = "delete"; + public const string Restore = "restore"; + + /// Changed outside every tool — a human at the file system, found by the reconciliation. + public const string External = "external"; +} + +/// #719: One line of the history. : the version after it can be shown and restored. +public sealed record FileHistoryEntry( + long Id, + string Workspace, + string Path, + string Action, + string? AgentId, + string? RunId, + string? HashBefore, + string? HashAfter, + long? SizeAfter, + bool Stored, + DateTime OccurredAtUtc, + string? Note); + +/// #719: A change in progress: the version before, captured before the tool touches the file. +public sealed record FileChange(string FullPath, string Workspace, string? HashBefore); + +/// +/// #719: The history of the workspaces — who created, replaced, appended to or deleted which +/// file, in which run, and what was in it before. For post-mortems of failed work and for +/// data that does not add up. +/// +/// Our own store, not Git. Git would bring diff and history ready-made, but a +/// commit per write with many agents writing in parallel means locks and waiting; the link +/// to the run and the audit would be extra work; binary files would bloat it for good; and +/// removing what is past its retention means rewriting history. This store fits the pattern +/// the instance already has: SQLite as the truth, append-only, everything in the backup. +/// +/// Each content once. Versions are kept under their SHA-256 in +/// Agents/.history/objects/, so an unchanged file costs nothing and an append costs +/// one version. The journal in state.db links every change to its run — and through +/// it to the receipt and the transcript. No .bak or .tmp names: the backup +/// leaves those out. +/// +/// One way in. Every tool that changes a file calls +/// before and after, under the file's lock. What changes outside +/// the tools is found by and recorded as external. +/// +public sealed class FileHistory +{ + public const string FolderName = ".history"; + + private readonly string _agentsDirectory; + private readonly SqliteStorage _storage; + private readonly Func _settings; + private readonly TimeProvider _clock; + + public FileHistory(string agentsDirectory, SqliteStorage storage, Func? settings = null, TimeProvider? clock = null) + { + _agentsDirectory = Path.GetFullPath(agentsDirectory); + _storage = storage; + _settings = settings ?? (() => new FileHistorySettings()); + _clock = clock ?? TimeProvider.System; + } + + private string ObjectsDirectory => Path.Combine(_agentsDirectory, FolderName, "objects"); + + private bool Enabled => _settings().Enabled; + + /// The workspace's own files — never the history itself. + public bool Covers(string fullPath) + => !IsInside(Path.GetFullPath(fullPath), Path.Combine(_agentsDirectory, FolderName)); + + // ─── Recording ─── + + /// Keeps the version the file has now, before a tool changes it. + public Task BeforeAsync(string fullPath, string workspace, CancellationToken ct) + { + fullPath = Path.GetFullPath(fullPath); + string? hash = null; + if (Enabled && Covers(fullPath) && File.Exists(fullPath)) + hash = Store(fullPath).Hash; + return Task.FromResult(new FileChange(fullPath, workspace, hash)); + } + + /// + /// Journals what the tool did. A write that left the content as it was is not a change and + /// is not recorded. The action is read from the file: gone is a delete, new is a create. + /// + public async Task RecordAsync( + FileChange change, string action, string? agentId, string? runId, CancellationToken ct, string? note = null) + { + if (!Enabled || !Covers(change.FullPath)) + return null; + + string? after = null; + long? size = null; + var stored = false; + if (File.Exists(change.FullPath)) + { + (after, stored) = Store(change.FullPath); + size = new FileInfo(change.FullPath).Length; + } + + if (after == change.HashBefore && action != FileAction.Restore) + return null; + if (after is null) + action = FileAction.Delete; + else if (change.HashBefore is null && action is FileAction.Replace or FileAction.Append) + action = FileAction.Create; + + var entry = new FileHistoryEntry(0, change.Workspace, Relative(change.FullPath), action, agentId, runId, + change.HashBefore, after, size, stored, Now, note); + return await AppendAsync(entry, change.FullPath, ct); + } + + private async Task AppendAsync(FileHistoryEntry entry, string fullPath, CancellationToken ct) + { + var id = await _storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + INSERT INTO FileHistory (Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note) + VALUES (@ws, @path, @action, @agent, @run, @before, @after, @size, @stored, @at, @note); + SELECT last_insert_rowid(); + """; + cmd.Parameters.AddWithValue("@ws", entry.Workspace); + cmd.Parameters.AddWithValue("@path", entry.Path); + cmd.Parameters.AddWithValue("@action", entry.Action); + cmd.Parameters.AddWithValue("@agent", (object?)entry.AgentId ?? DBNull.Value); + cmd.Parameters.AddWithValue("@run", (object?)entry.RunId ?? DBNull.Value); + cmd.Parameters.AddWithValue("@before", (object?)entry.HashBefore ?? DBNull.Value); + cmd.Parameters.AddWithValue("@after", (object?)entry.HashAfter ?? DBNull.Value); + cmd.Parameters.AddWithValue("@size", (object?)entry.SizeAfter ?? DBNull.Value); + cmd.Parameters.AddWithValue("@stored", entry.Stored ? 1 : 0); + cmd.Parameters.AddWithValue("@at", Format(entry.OccurredAtUtc)); + cmd.Parameters.AddWithValue("@note", (object?)entry.Note ?? DBNull.Value); + var rowId = (long)(await cmd.ExecuteScalarAsync(ct))!; + await SetStateAsync(conn, entry.Path, entry.Workspace, fullPath, entry.HashAfter, ct); + return rowId; + }, ct); + return entry with { Id = id }; + } + + /// What the history knows the file to be now — the reconciliation compares against it. + private static async Task SetStateAsync(SqliteConnection conn, string path, string workspace, string fullPath, string? hash, CancellationToken ct) + { + using var cmd = conn.CreateCommand(); + if (hash is null) + { + cmd.CommandText = "DELETE FROM FileHistoryState WHERE Path = @path"; + cmd.Parameters.AddWithValue("@path", path); + } + else + { + var info = new FileInfo(fullPath); + cmd.CommandText = """ + INSERT INTO FileHistoryState (Path, Workspace, Hash, Size, Modified) VALUES (@path, @ws, @hash, @size, @mod) + ON CONFLICT(Path) DO UPDATE SET Workspace = excluded.Workspace, Hash = excluded.Hash, Size = excluded.Size, Modified = excluded.Modified + """; + cmd.Parameters.AddWithValue("@path", path); + cmd.Parameters.AddWithValue("@ws", workspace); + cmd.Parameters.AddWithValue("@hash", hash); + cmd.Parameters.AddWithValue("@size", info.Exists ? info.Length : 0); + cmd.Parameters.AddWithValue("@mod", info.Exists ? info.LastWriteTimeUtc.Ticks : 0); + } + await cmd.ExecuteNonQueryAsync(ct); + } + + /// Stores the file's content under its hash, once. Too large: the hash only. + private (string Hash, bool Stored) Store(string fullPath) + { + string hash; + using (var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + hash = Convert.ToHexStringLower(SHA256.HashData(stream)); + + var target = ObjectPath(hash); + if (File.Exists(target)) + return (hash, true); + if (new FileInfo(fullPath).Length > (long)_settings().MaxObjectMegabytes * 1024 * 1024) + return (hash, false); + + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + var temp = target + "." + Guid.NewGuid().ToString("N")[..8] + ".part"; + File.Copy(fullPath, temp, overwrite: true); + try + { + File.Move(temp, target, overwrite: false); + } + catch (IOException) when (File.Exists(target)) + { + File.Delete(temp); // someone else stored the same content meanwhile + } + return (hash, true); + } + + private string ObjectPath(string hash) => Path.Combine(ObjectsDirectory, hash[..2], hash); + + // ─── Reading ─── + + /// The newest entries, filtered — for the history view. + public async Task> ListAsync( + string? workspace, string? path, string? agentId, string? runId, int limit, CancellationToken ct, long? beforeId = null) + { + await using var conn = await _storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + var where = new List(); + if (!string.IsNullOrWhiteSpace(workspace)) { where.Add("Workspace = @ws COLLATE NOCASE"); cmd.Parameters.AddWithValue("@ws", workspace); } + if (!string.IsNullOrWhiteSpace(path)) { where.Add("Path = @path"); cmd.Parameters.AddWithValue("@path", path); } + if (!string.IsNullOrWhiteSpace(agentId)) { where.Add("AgentId = @agent COLLATE NOCASE"); cmd.Parameters.AddWithValue("@agent", agentId); } + if (!string.IsNullOrWhiteSpace(runId)) { where.Add("RunId = @run"); cmd.Parameters.AddWithValue("@run", runId); } + if (beforeId is { } before) { where.Add("Id < @beforeId"); cmd.Parameters.AddWithValue("@beforeId", before); } + cmd.CommandText = $""" + SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note + FROM FileHistory + {(where.Count > 0 ? "WHERE " + string.Join(" AND ", where) : "")} + ORDER BY Id DESC + LIMIT @limit + """; + cmd.Parameters.AddWithValue("@limit", Math.Clamp(limit, 1, 1000)); + + var result = new List(); + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + result.Add(Read(reader)); + return result; + } + + public async Task FindAsync(long id, CancellationToken ct) + { + await using var conn = await _storage.OpenConnectionAsync(ct); + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note + FROM FileHistory WHERE Id = @id + """; + cmd.Parameters.AddWithValue("@id", id); + await using var reader = await cmd.ExecuteReaderAsync(ct); + return await reader.ReadAsync(ct) ? Read(reader) : null; + } + + /// A stored version's bytes, or null if it was not kept (too large, or pruned). + public byte[]? Content(string? hash) + { + if (string.IsNullOrWhiteSpace(hash) || hash.Length != 64 || !hash.All(Uri.IsHexDigit)) + return null; + var path = ObjectPath(hash.ToLowerInvariant()); + return File.Exists(path) ? File.ReadAllBytes(path) : null; + } + + /// The file on disk that a journal path stands for. + public string FullPathOf(string relativePath) + => Path.GetFullPath(Path.IsPathRooted(relativePath) ? relativePath : Path.Combine(_agentsDirectory, relativePath)); + + // ─── Restoring ─── + + /// + /// Puts a version back — by a human only, with a reason, as a journal line of its own. The + /// target must lie in a workspace the caller names; the history's own folder never is one. + /// + public async Task RestoreAsync( + FileHistoryEntry source, bool versionBefore, string by, string reason, + IReadOnlyList workspaceRoots, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(reason)) + throw new ArgumentException("A restore needs a reason."); + var hash = versionBefore ? source.HashBefore : source.HashAfter; + var content = Content(hash) + ?? throw new InvalidOperationException(hash is null + ? "That version is the file's absence — there is nothing to restore." + : "That version was not kept (too large, or past its retention)."); + + var target = FullPathOf(source.Path); + if (!Covers(target) || !workspaceRoots.Any(root => IsInside(target, Path.GetFullPath(root)))) + throw new InvalidOperationException("The file does not lie in a workspace."); + + var change = await BeforeAsync(target, source.Workspace, ct); + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + await AtomicFile.WriteAllBytesAsync(target, content, ct); + return (await RecordAsync(change, FileAction.Restore, by, null, ct, + $"Restored the version of entry {source.Id} ({(versionBefore ? "before" : "after")}): {reason.Trim()}"))!; + } + + // ─── Outside the tools ─── + + /// + /// Looks for changes made outside the tools and journals them as external. The first time + /// a file is seen, it is only noted — the whole existing workspace is no change. Size and + /// modification time decide whether a file is hashed at all, so an unchanged tree costs a + /// directory walk. + /// + public async Task ReconcileAsync(IReadOnlyList<(string Workspace, string Root)> workspaces, CancellationToken ct) + { + if (!Enabled) + return 0; + + var known = new Dictionary(StringComparer.Ordinal); + await using (var conn = await _storage.OpenConnectionAsync(ct)) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT Path, Workspace, Hash, Size, Modified FROM FileHistoryState"; + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + known[reader.GetString(0)] = (reader.GetString(1), reader.GetString(2), reader.GetInt64(3), reader.GetInt64(4)); + } + + var changes = 0; + var seen = new HashSet(StringComparer.Ordinal); + var coveredRoots = new List(); + foreach (var (workspace, root) in workspaces) + { + if (!Directory.Exists(root)) + continue; + coveredRoots.Add(Path.GetFullPath(root)); + foreach (var file in RegularFiles(root)) + { + ct.ThrowIfCancellationRequested(); + if (!Covers(file)) + continue; + var relative = Relative(file); + if (!seen.Add(relative)) + continue; // a workspace inside another — counted once + + var info = new FileInfo(file); + if (known.TryGetValue(relative, out var state) + && state.Size == info.Length && state.Modified == info.LastWriteTimeUtc.Ticks) + continue; + + if (state.Hash is null) + { + // Seen for the first time: noted, not a change. + await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, HashOnly(file), ct), ct); + continue; + } + + var (hash, stored) = Store(file); + if (hash == state.Hash) + { + await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, hash, ct), ct); + continue; + } + await AppendAsync(new FileHistoryEntry(0, workspace, relative, FileAction.External, null, null, + state.Hash, hash, info.Length, stored, Now, null), file, ct); + changes++; + } + } + + // Known files that are gone — deleted outside the tools. + foreach (var (relative, state) in known) + { + if (seen.Contains(relative)) + continue; + var full = FullPathOf(relative); + if (File.Exists(full) || !coveredRoots.Any(root => IsInside(full, root))) + continue; + await AppendAsync(new FileHistoryEntry(0, state.Workspace, relative, FileAction.External, null, null, + state.Hash, null, null, false, Now, "deleted outside the tools"), full, ct); + changes++; + } + return changes; + } + + private static string HashOnly(string file) + { + using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete); + return Convert.ToHexStringLower(SHA256.HashData(stream)); + } + + /// Regular files only — no links out of the workspace, no pipes that would block. + private static IEnumerable RegularFiles(string root) + { + var pending = new Stack([Path.GetFullPath(root)]); + while (pending.Count > 0) + { + var dir = pending.Pop(); + IEnumerable entries; + try { entries = new DirectoryInfo(dir).EnumerateFileSystemInfos().ToList(); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; } + foreach (var entry in entries) + { + switch (Runtimes.FileKind.Of(entry)) + { + case Runtimes.FileKind.Kind.Directory: pending.Push(entry.FullName); break; + case Runtimes.FileKind.Kind.Regular: yield return entry.FullName; break; + } + } + } + } + + // ─── Retention ─── + + /// + /// Removes entries past their retention and beyond the version limit — never a file's + /// newest — and then every stored version no entry points to any more. + /// + public async Task PurgeAsync(CancellationToken ct) + { + var settings = _settings(); + var cutoff = Format(Now.AddDays(-Math.Max(1, settings.RetentionDays))); + var keep = Math.Max(1, settings.MaxVersionsPerFile); + + var removed = await _storage.WriteAsync(async conn => + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = """ + DELETE FROM FileHistory WHERE Id IN ( + SELECT Id FROM ( + SELECT Id, OccurredAt, + ROW_NUMBER() OVER (PARTITION BY Path ORDER BY Id DESC) AS Newest + FROM FileHistory) + WHERE Newest > 1 AND (OccurredAt < @cutoff OR Newest > @keep)) + """; + cmd.Parameters.AddWithValue("@cutoff", cutoff); + cmd.Parameters.AddWithValue("@keep", keep); + return await cmd.ExecuteNonQueryAsync(ct); + }, ct); + + // Objects nobody points to. Collected after the deletion: a version stored meanwhile is + // either referenced or younger than the walk, and the walk skips unfinished ".part" files. + var referenced = new HashSet(StringComparer.Ordinal); + await using (var conn = await _storage.OpenConnectionAsync(ct)) + { + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT HashBefore FROM FileHistory UNION SELECT HashAfter FROM FileHistory UNION SELECT Hash FROM FileHistoryState"; + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + if (!reader.IsDBNull(0)) + referenced.Add(reader.GetString(0)); + } + if (Directory.Exists(ObjectsDirectory)) + { + var threshold = Now.AddMinutes(-10); + foreach (var file in Directory.EnumerateFiles(ObjectsDirectory, "*", SearchOption.AllDirectories)) + { + var name = Path.GetFileName(file); + if (name.Length != 64 || referenced.Contains(name) || File.GetLastWriteTimeUtc(file) > threshold) + continue; + try { File.Delete(file); } + catch (IOException) { } + } + } + return removed; + } + + // ─── Helpers ─── + + private DateTime Now => _clock.GetUtcNow().UtcDateTime; + + private static string Format(DateTime utc) => utc.ToString("O", CultureInfo.InvariantCulture); + + /// Relative to the Agents folder with '/', or the full path for a workspace elsewhere. + private string Relative(string fullPath) + { + var relative = Path.GetRelativePath(_agentsDirectory, fullPath); + return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative) + ? fullPath + : relative.Replace(Path.DirectorySeparatorChar, '/'); + } + + private static bool IsInside(string path, string root) + { + var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + root = Path.TrimEndingDirectorySeparator(root); + return path.Equals(root, comparison) || path.StartsWith(root + Path.DirectorySeparatorChar, comparison); + } + + private static FileHistoryEntry Read(SqliteDataReader r) => new( + r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3), + r.IsDBNull(4) ? null : r.GetString(4), + r.IsDBNull(5) ? null : r.GetString(5), + r.IsDBNull(6) ? null : r.GetString(6), + r.IsDBNull(7) ? null : r.GetString(7), + r.IsDBNull(8) ? null : r.GetInt64(8), + r.GetInt32(9) != 0, + DateTime.Parse(r.GetString(10), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + r.IsDBNull(11) ? null : r.GetString(11)); +} diff --git a/src/ClawdDotNet.Core/Files/FileHistoryKeeper.cs b/src/ClawdDotNet.Core/Files/FileHistoryKeeper.cs new file mode 100644 index 0000000..88918de --- /dev/null +++ b/src/ClawdDotNet.Core/Files/FileHistoryKeeper.cs @@ -0,0 +1,82 @@ +using ClawdDotNet.Core.Config; +using ClawdDotNet.Core.Organisation; +using Microsoft.Extensions.Logging; + +namespace ClawdDotNet.Core.Files; + +/// +/// #719: Keeps the history in order — at start and then at the configured interval it looks +/// for changes made outside the tools and removes what is past its retention. +/// +public sealed class FileHistoryKeeper( + FileHistory history, + Func> workspaces, + Func settings, + ILogger logger) : IAsyncDisposable +{ + private readonly CancellationTokenSource _cts = new(); + private Task? _loop; + + /// Every workspace of the instance: each agent's own, the shared one, and those with member lists. + public static IReadOnlyList<(string Workspace, string Root)> WorkspacesOf(IReadOnlyList agents, OrgChart organisation) + { + var result = new List<(string, string)>(); + foreach (var agent in agents.Where(a => !string.IsNullOrWhiteSpace(a.WorkspacePath))) + result.Add(($"{WorkspaceLayout.Personal}:{agent.AgentId}", agent.WorkspacePath)); + + var shared = agents.Select(a => a.SharedWorkspacePath).FirstOrDefault(p => !string.IsNullOrWhiteSpace(p)); + if (shared is not null) + { + result.Add((WorkspaceLayout.Shared, shared)); + foreach (var workspace in organisation.Workspaces) + if (WorkspaceLayout.PathOf(shared, workspace.Id) is { } root) + result.Add((workspace.Id, root)); + } + return result; + } + + public void Start() => _loop ??= RunAsync(_cts.Token); + + public async Task RunOnceAsync(CancellationToken ct) + { + var external = await history.ReconcileAsync(workspaces(), ct); + if (external > 0) + logger.LogInformation("File history: {Count} change(s) made outside the tools recorded", external); + var removed = await history.PurgeAsync(ct); + if (removed > 0) + logger.LogInformation("File history: {Count} entr(y/ies) past their retention removed", removed); + } + + private async Task RunAsync(CancellationToken ct) + { + // Not in the way of the start: the first walk of a large tree may take a while. + try { await Task.Delay(TimeSpan.FromSeconds(30), ct); } + catch (OperationCanceledException) { return; } + + while (!ct.IsCancellationRequested) + { + try + { + await RunOnceAsync(ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "File history: reconciliation or cleanup failed"); + } + + try { await Task.Delay(TimeSpan.FromMinutes(Math.Clamp(settings().ReconcileMinutes, 5, 24 * 60)), ct); } + catch (OperationCanceledException) { return; } + } + } + + public async ValueTask DisposeAsync() + { + await _cts.CancelAsync(); + if (_loop is not null) + { + try { await _loop; } + catch (OperationCanceledException) { } + } + _cts.Dispose(); + } +} diff --git a/src/ClawdDotNet.Core/Storage/SqliteStorage.cs b/src/ClawdDotNet.Core/Storage/SqliteStorage.cs index eaab74d..ed64939 100644 --- a/src/ClawdDotNet.Core/Storage/SqliteStorage.cs +++ b/src/ClawdDotNet.Core/Storage/SqliteStorage.cs @@ -374,6 +374,34 @@ public sealed class SqliteStorage Json TEXT NOT NULL ); + -- #719: The history of the workspaces. Append-only; only the retention removes lines, + -- never a file's newest. Path is relative to the Agents folder. + CREATE TABLE IF NOT EXISTS FileHistory ( + Id INTEGER PRIMARY KEY AUTOINCREMENT, + Workspace TEXT NOT NULL, + Path TEXT NOT NULL, + Action TEXT NOT NULL, + AgentId TEXT NULL, + RunId TEXT NULL, + HashBefore TEXT NULL, + HashAfter TEXT NULL, + SizeAfter INTEGER NULL, + Stored INTEGER NOT NULL DEFAULT 0, + OccurredAt TEXT NOT NULL, + Note TEXT NULL + ); + CREATE INDEX IF NOT EXISTS IX_FileHistory_Path ON FileHistory (Path, Id); + CREATE INDEX IF NOT EXISTS IX_FileHistory_Run ON FileHistory (RunId); + + -- #719: What the history knows each file to be now — the reconciliation compares against it. + CREATE TABLE IF NOT EXISTS FileHistoryState ( + Path TEXT PRIMARY KEY, + Workspace TEXT NOT NULL, + Hash TEXT NOT NULL, + Size INTEGER NOT NULL, + Modified INTEGER NOT NULL + ); + -- #720: Writes refused because they would have lost someone's change. CREATE TABLE IF NOT EXISTS FileConflicts ( Id INTEGER PRIMARY KEY AUTOINCREMENT, diff --git a/src/ClawdDotNet.Core/Tools/AgentToolContext.cs b/src/ClawdDotNet.Core/Tools/AgentToolContext.cs index 0a3617b..476d833 100644 --- a/src/ClawdDotNet.Core/Tools/AgentToolContext.cs +++ b/src/ClawdDotNet.Core/Tools/AgentToolContext.cs @@ -61,6 +61,12 @@ public sealed record AgentToolContext( /// public Files.FileLedger? Files { get; init; } + /// + /// #719: The history of the workspaces. Tools that change a file keep the version before + /// and journal the change, with this call's agent and run. null: no history. + /// + public Files.FileHistory? History { get; init; } + /// #697: The run this call belongs to — for stamping where something came from. public string? RunId { get; init; } diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/app.css b/src/ClawdDotNet.Manager.Web/wwwroot/app.css index 1169094..9e421d9 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/app.css +++ b/src/ClawdDotNet.Manager.Web/wwwroot/app.css @@ -869,6 +869,103 @@ button.klein { min-width: 0; } +/* #719: the file history */ +.dateiverlauf-filter { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-bottom: 14px; +} + +.dateiverlauf-filter input, +.dateiverlauf-filter select { + flex: 1 1 160px; + min-width: 0; +} + +.dateiverlauf-liste { + display: flex; + flex-direction: column; +} + +.dateiverlauf-pfad { + font-family: var(--mono, monospace); + word-break: break-all; + text-align: left; +} + +.dateiverlauf-aktion { + font-size: 11px; + font-weight: 700; + text-transform: uppercase; + letter-spacing: 0.04em; + padding: 1px 6px; + border: 1px solid var(--divider); + white-space: nowrap; +} + +.dateiverlauf-aktion.aktion-delete, +.dateiverlauf-aktion.aktion-external { + border-color: var(--gefahr); + color: var(--gefahr); +} + +.dateiverlauf-aktion.aktion-restore { + border-color: var(--accent); + color: var(--accent); +} + +.verknuepfung { + background: none; + border: 0; + padding: 0; + color: var(--accent); + cursor: pointer; + font: inherit; +} + +.verknuepfung:hover, +.verknuepfung:focus-visible { + text-decoration: underline; +} + +.modal-box.modal-breit { + max-width: min(1100px, 100%); +} + +.dateiverlauf-diff, +.dateiverlauf-text { + font-family: var(--mono, monospace); + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; + max-height: 60vh; + overflow: auto; + border: 1px solid var(--divider); + padding: 8px; + margin: 0; +} + +.dateiverlauf-seiten { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 8px; +} + +/* One block per line, so the colour covers the whole line, not just its text. */ +.dateiverlauf-diff span { + display: block; + min-height: 1.3em; +} + +.diff-removed { + background: color-mix(in srgb, var(--gefahr) 16%, transparent); +} + +.diff-added { + background: color-mix(in srgb, var(--erfolg, #2e7d32) 18%, transparent); +} + /* #720: files with repeated write conflicts, above the team canvas */ .team-konflikte { border: 1px solid var(--divider); diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/app.js b/src/ClawdDotNet.Manager.Web/wwwroot/app.js index ca41b77..0bc4119 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/app.js +++ b/src/ClawdDotNet.Manager.Web/wwwroot/app.js @@ -35,6 +35,7 @@ import { renderUsageRuns } from "./views/usage-runs.js"; import { renderUsageTimeline } from "./views/usage-timeline.js"; import { renderRunDetail } from "./views/run-detail.js"; import { renderPromptHistory } from "./views/prompt-history.js"; +import { renderFileHistory } from "./views/file-history.js"; import { renderTeam, leaveTeam } from "./views/team.js"; import { renderOrganisation } from "./views/organisation.js"; import { renderCases, holeMenschenfaelle } from "./views/cases.js"; @@ -65,6 +66,7 @@ registriereRoute("/verbrauch/runs", () => { verlasseAnsicht(); return renderUsag registriereRoute("/verbrauch/runs/:runId", (p) => { verlasseAnsicht(); return renderRunDetail(p.runId, "/verbrauch/runs"); }); registriereRoute("/verbrauch/timeline", () => { verlasseAnsicht(); return renderUsageTimeline(); }); registriereRoute("/prompt-history", () => { verlasseAnsicht(); return renderPromptHistory(); }); +registriereRoute("/file-history", () => { verlasseAnsicht(); return renderFileHistory(); }); registriereRoute("/prompt-history/:runId", (p) => { verlasseAnsicht(); return renderRunDetail(p.runId, "/prompt-history"); }); registriereRoute("/agenten-chats", () => { verlasseAnsicht(); return rendereAgentenChats(); }); registriereRoute("/protokoll", () => { verlasseAnsicht(); return rendereProtokoll(); }); diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json index f8a4ee8..fed7e2f 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json +++ b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/de.json @@ -986,5 +986,49 @@ "team.conflicts.title": "Dateien, in denen sich Agenten gegenseitig überschreiben", "team.conflicts.hint": "Hier wurden Schreibvorgänge abgelehnt, weil sie die Änderung eines anderen Agenten verloren hätten. Passiert das oft, sollte die Ablage anders werden: eine Datei je Agent oder Tag, Anhängen statt Ersetzen oder eine Tabelle.", - "team.conflicts.count": "{n} Konflikte in 7 Tagen, zwischen {agents}; zuletzt {time}" + "team.conflicts.count": "{n} Konflikte in 7 Tagen, zwischen {agents}; zuletzt {time}", + + "nav.fileHistory": "Dateiverlauf", + "fileHistory.title": "Dateiverlauf", + "fileHistory.intro": "Jede Änderung in den Arbeitsbereichen: wer welche Datei in welchem Lauf angelegt, ersetzt, ergänzt oder gelöscht hat, und was vorher darin stand. Änderungen außerhalb der Werkzeuge werden gefunden und als extern geführt. Eine Fassung lässt sich zurückholen – mit Grund, als eigener Eintrag.", + "fileHistory.workspace": "Arbeitsbereich", + "fileHistory.allWorkspaces": "Alle Arbeitsbereiche", + "fileHistory.agent": "Agent", + "fileHistory.path": "Dateipfad (genau)", + "fileHistory.run": "Lauf", + "fileHistory.apply": "Filtern", + "fileHistory.reset": "Zurücksetzen", + "fileHistory.more": "Ältere Einträge", + "fileHistory.unavailable": "Dateiverlauf nicht verfügbar", + "fileHistory.empty.title": "Keine Änderungen", + "fileHistory.empty.text": "Nichts passt – oder seit Beginn des Verlaufs wurde nichts geändert.", + "fileHistory.external": "außerhalb der Werkzeuge", + "fileHistory.toRun": "Lauf", + "fileHistory.onlyThisFile": "Nur diese Datei zeigen", + "fileHistory.show": "Anzeigen", + "fileHistory.action.create": "angelegt", + "fileHistory.action.replace": "ersetzt", + "fileHistory.action.append": "ergänzt", + "fileHistory.action.copy": "kopiert", + "fileHistory.action.delete": "gelöscht", + "fileHistory.action.restore": "wiederhergestellt", + "fileHistory.action.external": "extern", + "fileHistory.before": "Vorher", + "fileHistory.after": "Nachher", + "fileHistory.text": "Text", + "fileHistory.absent": "keine Datei", + "fileHistory.notKept": "nicht aufbewahrt", + "fileHistory.tooLarge": "zu groß zum Anzeigen ({bytes} Bytes)", + "fileHistory.binary": "binär ({bytes} Bytes)", + "fileHistory.restoreBefore": "Fassung davor wiederherstellen", + "fileHistory.restoreAfter": "Fassung danach wiederherstellen", + "fileHistory.restore": "Wiederherstellen", + "fileHistory.restoreTitle": "{path} wiederherstellen", + "fileHistory.restoreBeforeHint": "Die Datei bekommt den Inhalt, den sie vor dieser Änderung hatte. Was jetzt darin steht, bleibt im Verlauf erhalten.", + "fileHistory.restoreAfterHint": "Die Datei bekommt den Inhalt, den ihr diese Änderung gegeben hat. Was jetzt darin steht, bleibt im Verlauf erhalten.", + "fileHistory.reason": "Grund", + "fileHistory.reasonPlaceholder": "Warum wird diese Fassung zurückgeholt?", + "fileHistory.reasonMissing": "Eine Wiederherstellung braucht einen Grund.", + "fileHistory.restored": "{path} wiederhergestellt.", + "fileHistory.failed": "Fehlgeschlagen: {error}" } diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json index 7602425..a3e51c3 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json +++ b/src/ClawdDotNet.Manager.Web/wwwroot/i18n/en.json @@ -986,5 +986,49 @@ "team.conflicts.title": "Files agents keep overwriting each other in", "team.conflicts.hint": "Writes were refused here because they would have lost another agent's change. If it keeps happening, change how the file is kept: one file per agent or day, appending instead of replacing, or a table.", - "team.conflicts.count": "{n} conflicts in 7 days, between {agents}; last {time}" + "team.conflicts.count": "{n} conflicts in 7 days, between {agents}; last {time}", + + "nav.fileHistory": "File History", + "fileHistory.title": "File History", + "fileHistory.intro": "Every change in the workspaces: who created, replaced, appended to or deleted which file, in which run, and what was in it before. Changes made outside the tools are found and listed as external. A version can be put back — with a reason, as an entry of its own.", + "fileHistory.workspace": "Workspace", + "fileHistory.allWorkspaces": "All workspaces", + "fileHistory.agent": "Agent", + "fileHistory.path": "File path (exact)", + "fileHistory.run": "Run", + "fileHistory.apply": "Filter", + "fileHistory.reset": "Reset", + "fileHistory.more": "Older entries", + "fileHistory.unavailable": "File history unavailable", + "fileHistory.empty.title": "No changes", + "fileHistory.empty.text": "Nothing matches — or nothing has been changed since the history started.", + "fileHistory.external": "outside the tools", + "fileHistory.toRun": "run", + "fileHistory.onlyThisFile": "Show only this file", + "fileHistory.show": "Show", + "fileHistory.action.create": "created", + "fileHistory.action.replace": "replaced", + "fileHistory.action.append": "appended", + "fileHistory.action.copy": "copied", + "fileHistory.action.delete": "deleted", + "fileHistory.action.restore": "restored", + "fileHistory.action.external": "external", + "fileHistory.before": "Before", + "fileHistory.after": "After", + "fileHistory.text": "text", + "fileHistory.absent": "no file", + "fileHistory.notKept": "not kept", + "fileHistory.tooLarge": "too large to show ({bytes} bytes)", + "fileHistory.binary": "binary ({bytes} bytes)", + "fileHistory.restoreBefore": "Restore version before", + "fileHistory.restoreAfter": "Restore version after", + "fileHistory.restore": "Restore", + "fileHistory.restoreTitle": "Restore {path}", + "fileHistory.restoreBeforeHint": "The file gets the content it had before this change. What it holds now is kept in the history.", + "fileHistory.restoreAfterHint": "The file gets the content this change gave it. What it holds now is kept in the history.", + "fileHistory.reason": "Reason", + "fileHistory.reasonPlaceholder": "Why is this version put back?", + "fileHistory.reasonMissing": "A restore needs a reason.", + "fileHistory.restored": "{path} restored.", + "fileHistory.failed": "Failed: {error}" } diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/index.html b/src/ClawdDotNet.Manager.Web/wwwroot/index.html index 1ddccd9..ab2b620 100644 --- a/src/ClawdDotNet.Manager.Web/wwwroot/index.html +++ b/src/ClawdDotNet.Manager.Web/wwwroot/index.html @@ -101,6 +101,12 @@ Prompt History + + + + + File History + diff --git a/src/ClawdDotNet.Manager.Web/wwwroot/views/file-history.js b/src/ClawdDotNet.Manager.Web/wwwroot/views/file-history.js new file mode 100644 index 0000000..923a68c --- /dev/null +++ b/src/ClawdDotNet.Manager.Web/wwwroot/views/file-history.js @@ -0,0 +1,258 @@ +// ClawdDotNet — Analysis › File History (#719) +// +// Who created, replaced, appended to or deleted which file in the workspaces, in which run, +// and what was in it before. Every entry links to its run. A version can be put back — by a +// human only, with a reason, as an entry of its own. +// +// File contents are foreign text: only ever inserted via textContent. + +import { api } from "../modules/api.js"; +import { erstelleElement, erstelleKarte, erstelleModal, formatiereDatumZeit, zeigeMeldung } from "../modules/dom.js"; +import { t } from "../modules/i18n.js"; + +const PAGE_SIZE = 100; + +/** Diff only up to this many lines per side — beyond, both versions are shown as they are. */ +const MAX_DIFF_LINES = 3000; + +export async function renderFileHistory() { + const container = document.getElementById("ansicht-inhalt"); + if (!container) return; + + const filter = { workspace: "", path: "", agent: "", run: "" }; + const list = erstelleElement("div", { klasse: "dateiverlauf-liste" }); + const more = erstelleElement("button", { type: "button", klasse: "schaltflaeche flach", text: t("fileHistory.more") }); + more.hidden = true; + let oldest = null; + + const workspace = erstelleElement("select", { "aria-label": t("fileHistory.workspace") }); + workspace.append(erstelleElement("option", { value: "", text: t("fileHistory.allWorkspaces") })); + const agent = erstelleElement("input", { type: "text", placeholder: t("fileHistory.agent"), "aria-label": t("fileHistory.agent") }); + const path = erstelleElement("input", { type: "text", placeholder: t("fileHistory.path"), "aria-label": t("fileHistory.path") }); + const run = erstelleElement("input", { type: "text", placeholder: t("fileHistory.run"), "aria-label": t("fileHistory.run") }); + const apply = erstelleElement("button", { + type: "submit", klasse: "schaltflaeche primaer", text: t("fileHistory.apply") + }); + const reset = erstelleElement("button", { + type: "button", klasse: "schaltflaeche flach", text: t("fileHistory.reset"), + onclick: () => { + workspace.value = agent.value = path.value = run.value = ""; + load(true); + } + }); + const form = erstelleElement("form", { klasse: "dateiverlauf-filter" }, workspace, agent, path, run, apply, reset); + form.addEventListener("submit", (e) => { + e.preventDefault(); + load(true); + }); + + container.replaceChildren(erstelleElement("div", { klasse: "verbrauch-ansicht" }, + erstelleElement("h1", { text: t("fileHistory.title") }), + erstelleElement("p", { klasse: "hinweis", text: t("fileHistory.intro") }), + form, list, more)); + + try { + for (const name of (await api("/file-history/workspaces")) || []) { + workspace.append(erstelleElement("option", { value: name, text: name })); + } + } catch { + // The list works without the choices. + } + + more.addEventListener("click", () => load(false)); + + /** Filters by a value from a row — the path or the run of an entry. */ + function filterBy(field, value) { + ({ path, run, agent })[field].value = value; + load(true); + } + + async function load(fresh) { + Object.assign(filter, { + workspace: workspace.value, path: path.value.trim(), agent: agent.value.trim(), run: run.value.trim() + }); + if (fresh) { + oldest = null; + list.replaceChildren(erstelleElement("div", { klasse: "hinweis-zentriert", text: t("common.loading") })); + } + const query = new URLSearchParams({ limit: String(PAGE_SIZE) }); + for (const [key, value] of Object.entries(filter)) if (value) query.set(key, value); + if (oldest) query.set("before", String(oldest)); + + let entries; + try { + entries = (await api(`/file-history?${query}`)) || []; + } catch (err) { + list.replaceChildren(erstelleKarte(t("fileHistory.unavailable"), [erstelleElement("p", { klasse: "gedaempft", text: err.message })])); + return; + } + + if (fresh) list.replaceChildren(); + if (fresh && entries.length === 0) { + list.append(erstelleKarte(t("fileHistory.empty.title"), [erstelleElement("p", { klasse: "gedaempft", text: t("fileHistory.empty.text") })])); + } + for (const entry of entries) list.append(row(entry, filterBy, () => load(true))); + if (entries.length > 0) oldest = entries[entries.length - 1].id; + more.hidden = entries.length < PAGE_SIZE; + } + + await load(true); +} + +function row(entry, filterBy, reload) { + const who = entry.agentId + ? erstelleElement("button", { + type: "button", klasse: "verknuepfung", text: entry.agentId, + onclick: () => filterBy("agent", entry.agentId) + }) + : erstelleElement("span", { klasse: "gedaempft", text: entry.action === "external" ? t("fileHistory.external") : "—" }); + + const runLink = entry.runId + ? erstelleElement("a", { href: `#/prompt-history/${encodeURIComponent(entry.runId)}`, text: t("fileHistory.toRun") }) + : null; + + return erstelleElement("div", { klasse: "werkzeug-zeile dateiverlauf-zeile" }, + erstelleElement("div", { klasse: "werkzeug-info" }, + erstelleElement("div", { klasse: "gruppe-horizontal" }, + erstelleElement("span", { klasse: `dateiverlauf-aktion aktion-${entry.action}`, text: t(`fileHistory.action.${entry.action}`) }), + erstelleElement("button", { + type: "button", klasse: "verknuepfung dateiverlauf-pfad", text: entry.path, + title: t("fileHistory.onlyThisFile"), + onclick: () => filterBy("path", entry.path) + })), + erstelleElement("div", { klasse: "werkzeug-beschreibung gedaempft" }, + `${formatiereDatumZeit(entry.occurredAtUtc)} · ${entry.workspace} · `, who, + runLink ? " · " : null, runLink, + entry.note ? ` · ${entry.note}` : null)), + erstelleElement("div", { klasse: "werkzeug-aktionen" }, + erstelleElement("button", { + type: "button", klasse: "schaltflaeche flach klein", text: t("fileHistory.show"), + onclick: () => showEntry(entry, reload) + }))); +} + +/** Before and after of one change, as a diff where both are text, and putting either back. */ +async function showEntry(entry, reload) { + const body = erstelleElement("div", { klasse: "dateiverlauf-detail" }, + erstelleElement("div", { klasse: "hinweis-zentriert", text: t("common.loading") })); + const restoreBefore = erstelleElement("button", { + type: "button", klasse: "schaltflaeche flach", text: t("fileHistory.restoreBefore"), + onclick: () => restore(entry, "before", modal, reload) + }); + const restoreAfter = erstelleElement("button", { + type: "button", klasse: "schaltflaeche flach", text: t("fileHistory.restoreAfter"), + onclick: () => restore(entry, "after", modal, reload) + }); + restoreBefore.disabled = !entry.hasBefore; + restoreAfter.disabled = !entry.hasAfter; + const close = erstelleElement("button", { type: "button", klasse: "schaltflaeche primaer", text: t("common.close"), onclick: () => modal.schliessen() }); + const modal = erstelleModal(`${t(`fileHistory.action.${entry.action}`)}: ${entry.path}`, body, [restoreBefore, restoreAfter, close], { klasse: "modal-breit" }); + modal.oeffnen(); + + let before, after; + try { + [before, after] = await Promise.all([ + api(`/file-history/${entry.id}/content?version=before`), + api(`/file-history/${entry.id}/content?version=after`) + ]); + } catch (err) { + body.replaceChildren(erstelleElement("p", { klasse: "fehler", text: err.message })); + return; + } + // Only a version that was kept can be put back. + restoreBefore.disabled = !entry.hasBefore || !before.kept; + restoreAfter.disabled = !entry.hasAfter || !after.kept; + + const describe = (version, present) => !present ? t("fileHistory.absent") + : !version.kept ? t("fileHistory.notKept") + : version.tooLarge ? t("fileHistory.tooLarge", { bytes: version.bytes }) + : version.binary ? t("fileHistory.binary", { bytes: version.bytes }) + : null; + + const beforeNote = describe(before, entry.hasBefore); + const afterNote = describe(after, entry.hasAfter); + const parts = []; + if (beforeNote || afterNote) { + parts.push(erstelleElement("p", { klasse: "gedaempft text-klein", text: `${t("fileHistory.before")}: ${beforeNote ?? t("fileHistory.text")} · ${t("fileHistory.after")}: ${afterNote ?? t("fileHistory.text")}` })); + } + if (before.text != null || after.text != null) { + parts.push(diffView(before.text ?? "", after.text ?? "")); + } + body.replaceChildren(...parts); +} + +async function restore(entry, version, modal, reload) { + const reason = erstelleElement("textarea", { rows: "3", "aria-label": t("fileHistory.reason"), placeholder: t("fileHistory.reasonPlaceholder") }); + const confirm = erstelleElement("button", { type: "button", klasse: "schaltflaeche primaer", text: t("fileHistory.restore") }); + const cancel = erstelleElement("button", { type: "button", klasse: "schaltflaeche flach", text: t("common.cancel") }); + const ask = erstelleModal(t("fileHistory.restoreTitle", { path: entry.path }), erstelleElement("div", { klasse: "formular-raster" }, + erstelleElement("p", { klasse: "gedaempft", text: t(version === "before" ? "fileHistory.restoreBeforeHint" : "fileHistory.restoreAfterHint") }), + erstelleElement("label", { text: t("fileHistory.reason") }), reason), [cancel, confirm]); + cancel.addEventListener("click", () => ask.schliessen()); + confirm.addEventListener("click", async () => { + if (!reason.value.trim()) { + zeigeMeldung(t("fileHistory.reasonMissing"), "warnung"); + return; + } + confirm.disabled = true; + try { + await api(`/file-history/${entry.id}/restore`, { method: "POST", body: { version, reason: reason.value.trim() } }); + zeigeMeldung(t("fileHistory.restored", { path: entry.path }), "info"); + ask.schliessen(); + modal.schliessen(); + reload(); + } catch (err) { + zeigeMeldung(t("fileHistory.failed", { error: err.message }), "gefahr"); + confirm.disabled = false; + } + }); + ask.oeffnen(); +} + +// ─── Diff ─── + +/** Lines of both versions, marked as kept, removed or added — a longest-common-subsequence diff. */ +export function diffLines(beforeText, afterText) { + const a = beforeText === "" ? [] : beforeText.split("\n"); + const b = afterText === "" ? [] : afterText.split("\n"); + if (a.length > MAX_DIFF_LINES || b.length > MAX_DIFF_LINES) return null; + + // Common start and end first: most changes touch a small part of a file. + let start = 0; + while (start < a.length && start < b.length && a[start] === b[start]) start++; + let endA = a.length, endB = b.length; + while (endA > start && endB > start && a[endA - 1] === b[endB - 1]) { endA--; endB--; } + + const midA = a.slice(start, endA), midB = b.slice(start, endB); + const n = midA.length, m = midB.length; + const lcs = Array.from({ length: n + 1 }, () => new Uint16Array(m + 1)); + for (let i = n - 1; i >= 0; i--) { + for (let j = m - 1; j >= 0; j--) { + lcs[i][j] = midA[i] === midB[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]); + } + } + + const out = a.slice(0, start).map((line) => ({ kind: "same", line })); + let i = 0, j = 0; + while (i < n && j < m) { + if (midA[i] === midB[j]) { out.push({ kind: "same", line: midA[i] }); i++; j++; } + else if (lcs[i + 1][j] >= lcs[i][j + 1]) { out.push({ kind: "removed", line: midA[i] }); i++; } + else { out.push({ kind: "added", line: midB[j] }); j++; } + } + while (i < n) out.push({ kind: "removed", line: midA[i++] }); + while (j < m) out.push({ kind: "added", line: midB[j++] }); + for (const line of a.slice(endA)) out.push({ kind: "same", line }); + return out; +} + +function diffView(beforeText, afterText) { + const lines = diffLines(beforeText, afterText); + if (!lines) { + return erstelleElement("div", { klasse: "dateiverlauf-seiten" }, + erstelleElement("pre", { klasse: "dateiverlauf-text", text: beforeText }), + erstelleElement("pre", { klasse: "dateiverlauf-text", text: afterText })); + } + const sign = { same: " ", removed: "- ", added: "+ " }; + return erstelleElement("pre", { klasse: "dateiverlauf-diff" }, + ...lines.map((l) => erstelleElement("span", { klasse: `diff-${l.kind}`, text: `${sign[l.kind]}${l.line}` }))); +} diff --git a/src/ClawdDotNet.Tools.FTP/FtpTool.cs b/src/ClawdDotNet.Tools.FTP/FtpTool.cs index ecd3774..52411ae 100644 --- a/src/ClawdDotNet.Tools.FTP/FtpTool.cs +++ b/src/ClawdDotNet.Tools.FTP/FtpTool.cs @@ -328,9 +328,15 @@ public sealed class FTPTool : IAgentTool var localDir = Path.GetDirectoryName(localFull); if (localDir != null && !Directory.Exists(localDir)) Directory.CreateDirectory(localDir); + // #719: the download is a change like a write — kept and journaled. + var historyWorkspace = wsLabel == WorkspaceLayout.Personal ? $"{WorkspaceLayout.Personal}:{context.AgentId}" : wsLabel; + var change = context.History is { } history ? await history.BeforeAsync(localFull, historyWorkspace, ct) : null; + var status = await client.DownloadFile(localFull, remote, FtpLocalExists.Overwrite, FtpVerify.None, null, ct); if (guarded && status == FtpStatus.Success) context.Files!.NoteReplace(context.AgentId, context.RunId, localFull, Core.Files.FileLedger.HashOf(localFull)); + if (change is not null && status == FtpStatus.Success) + await context.History!.RecordAsync(change, Core.Files.FileAction.Replace, context.AgentId, context.RunId, ct, $"FTP download of {remote}"); return status == FtpStatus.Success ? ToolResult.Ok($"[{wsLabel} Workspace] File successfully downloaded: {remote} → {localRel}") : ToolResult.Fail($"Download failed: {status}"); diff --git a/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs b/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs index 8aeb45e..799ca57 100644 --- a/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs +++ b/src/ClawdDotNet.Tools.FileRW/FileRWTool.cs @@ -311,6 +311,45 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe || relative.StartsWith(TasksFolder + "/", StringComparison.OrdinalIgnoreCase); } + // ─── History (#719) ─── + + /// The workspace as the history names it — the personal one with its owner, since every agent has one. + private static string HistoryWorkspace(Area area, AgentToolContext context) + => area.IsPersonal ? $"{WorkspaceLayout.Personal}:{context.AgentId}" : area.Name; + + /// + /// Keeps the version before the change. Best effort: a history that cannot be written + /// must not stop the work — it is logged, and the change happens without its entry. + /// + private static async Task BeforeChangeAsync(string path, Area area, AgentToolContext context, CancellationToken ct) + { + if (context.History is not { } history) + return null; + try + { + return await history.BeforeAsync(path, HistoryWorkspace(area, context), ct); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or Microsoft.Data.Sqlite.SqliteException) + { + context.Logger.LogWarning(ex, "File history: the version before could not be kept for {Path}", path); + return null; + } + } + + private static async Task AfterChangeAsync(FileChange? change, string action, AgentToolContext context, CancellationToken ct) + { + if (change is null || context.History is not { } history) + return; + try + { + await history.RecordAsync(change, action, context.AgentId, context.RunId, ct); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or Microsoft.Data.Sqlite.SqliteException) + { + context.Logger.LogWarning(ex, "File history: the change could not be recorded for {Path}", change.FullPath); + } + } + // ─── Lost updates (#720) ─── /// Only workspaces more than one agent can write to — never the personal one. @@ -712,6 +751,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe var dir = Path.GetDirectoryName(path); if (dir != null && !Directory.Exists(dir)) Directory.CreateDirectory(dir); + var change = await BeforeChangeAsync(path, area, context, ct); // Atomar schreiben (#553): erst in eine Nebendatei, dann ersetzen. Ein Absturz // mitten im Vorgang hinterlässt damit entweder die alte oder die neue Fassung, @@ -720,6 +760,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe // Agenten aber nicht. await AtomicFile.WriteAllTextAsync(path, content, new UTF8Encoding(false), ct); NoteReplace(path, area, context); + await AfterChangeAsync(change, FileAction.Replace, context, ct); return ToolResult.Ok($"[{area.Name} Workspace] File successfully written: {area.Name}:/{Path.GetRelativePath(rootPath, path)}"); } finally @@ -744,6 +785,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe // #720: appending loses nothing and is not checked — but noted, so whoever read // the file before does not replace it without the appended part. var before = Guarded(area, context) ? FileLedger.HashOf(path) : null; + var change = await BeforeChangeAsync(path, area, context, ct); // #575: Neu entstehende Dateien gehoeren nur dem Besitzer (0600). await using (var stream = ClawdDotNet.Core.Storage.OwnerOnlyFile.OpenAppend(path)) @@ -752,6 +794,7 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe if (Guarded(area, context)) context.Files!.NoteAppend(context.AgentId, context.RunId, path, before, FileLedger.HashOf(path)); + await AfterChangeAsync(change, FileAction.Append, context, ct); return ToolResult.Ok($"[{area.Name} Workspace] Content successfully appended to: {area.Name}:/{Path.GetRelativePath(rootPath, path)}"); } finally @@ -895,12 +938,20 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe { if (File.Exists(path)) { + var change = await BeforeChangeAsync(path, area, context, ct); File.Delete(path); + await AfterChangeAsync(change, FileAction.Delete, context, ct); return ToolResult.Ok($"[{area.Name} Workspace] File deleted: {area.Name}:/{Path.GetRelativePath(rootPath, path)}"); } else if (Directory.Exists(path)) { + // #719: every file in it is a deletion of its own in the history. + var changes = new List(); + foreach (var file in Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)) + changes.Add(await BeforeChangeAsync(file, area, context, ct)); Directory.Delete(path, true); + foreach (var change in changes) + await AfterChangeAsync(change, FileAction.Delete, context, ct); return ToolResult.Ok($"[{area.Name} Workspace] Directory deleted: {area.Name}:/{Path.GetRelativePath(rootPath, path)}"); } @@ -968,9 +1019,11 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe var destDir = Path.GetDirectoryName(destPath); if (destDir != null && !Directory.Exists(destDir)) Directory.CreateDirectory(destDir); + var change = await BeforeChangeAsync(destPath, destArea, context, ct); await Task.Run(() => File.Copy(sourcePath, destPath, overwrite: !IsPathProtected(destPath, destRootPath, destArea)), ct); NoteReplace(destPath, destArea, context); + await AfterChangeAsync(change, FileAction.Copy, context, ct); var srcLabel = $"{sourceArea.Name}:/{Path.GetRelativePath(sourceRootPath, sourcePath)}"; var dstLabel = $"{destArea.Name}:/{Path.GetRelativePath(destRootPath, destPath)}"; @@ -988,8 +1041,28 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe } else if (Directory.Exists(sourcePath)) { + // #720/#719: each file the copy would replace is checked and recorded like a single copy. + var targets = Directory.EnumerateFiles(sourcePath, "*", SearchOption.AllDirectories) + .Select(f => Path.Combine(destPath, Path.GetRelativePath(sourcePath, f))) + .ToList(); + foreach (var target in targets) + { + var shown = $"{destArea.Name}:/{Path.GetRelativePath(destRootPath, target).Replace('\\', '/')}"; + if (await RefuseLostUpdateAsync(target, shown, destArea, context, ct) is { } conflict) + return conflict; + } + var changes = new List(); + foreach (var target in targets) + changes.Add(await BeforeChangeAsync(target, destArea, context, ct)); + await Task.Run(() => CopyDirectory(sourcePath, destPath), ct); + foreach (var (target, change) in targets.Zip(changes)) + { + NoteReplace(target, destArea, context); + await AfterChangeAsync(change, FileAction.Copy, context, ct); + } + var srcLabel = $"{sourceArea.Name}:/{Path.GetRelativePath(sourceRootPath, sourcePath)}"; var dstLabel = $"{destArea.Name}:/{Path.GetRelativePath(destRootPath, destPath)}"; return ToolResult.Ok($"Directory copied: {srcLabel} → {dstLabel}"); @@ -1077,7 +1150,10 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe // Datenpunkt speichern var json = JsonSerializer.Serialize(entry, StockJsonOpts); + var stockArea = new Area(WorkspaceLayout.Shared, sharedRoot, WorkspaceAccess.Write, []); + var dataChange = await BeforeChangeAsync(filePath, stockArea, context, ct); await ClawdDotNet.Core.Storage.AtomicFile.WriteAllTextAsync(filePath, json, new UTF8Encoding(false), ct); + await AfterChangeAsync(dataChange, FileAction.Create, context, ct); // _index.json aktualisieren (thread-safe via per-file SemaphoreSlim, append-only) var indexPath = Path.Combine(stockDir, "_index.json"); @@ -1120,10 +1196,12 @@ public sealed class FileRWTool : IAgentTool, IToolStateProbe } index.Add(indexEntry); + var indexChange = await BeforeChangeAsync(indexPath, stockArea, context, ct); // Atomar: Der Index ist die einzige Übersicht über die Datenpunkte — // eine halb geschriebene Datei wäre nicht wiederherstellbar. await ClawdDotNet.Core.Storage.AtomicFile.WriteAllTextAsync( indexPath, JsonSerializer.Serialize(index, StockJsonOpts), ct: ct); + await AfterChangeAsync(indexChange, FileAction.Append, context, ct); // Den Stand hier festhalten, solange die Sperre noch gilt (B9). Vorher // wurde die Datei nach dem Freigeben ein zweites Mal gelesen und geparst: diff --git a/src/ClawdDotNet.Tools.RocketChat/RocketChatTool.cs b/src/ClawdDotNet.Tools.RocketChat/RocketChatTool.cs index 0234037..6eed439 100644 --- a/src/ClawdDotNet.Tools.RocketChat/RocketChatTool.cs +++ b/src/ClawdDotNet.Tools.RocketChat/RocketChatTool.cs @@ -265,9 +265,15 @@ public sealed class RocketChatTool : IAgentTool, IToolJobProvider Directory.CreateDirectory(Path.GetDirectoryName(file.FullPath)!); + // #719: the download is a change like a write — kept and journaled. + var historyWorkspace = file.Label == WorkspaceLayout.Personal ? $"{WorkspaceLayout.Personal}:{context.AgentId}" : file.Label; + var change = context.History is { } history ? await history.BeforeAsync(file.FullPath, historyWorkspace, ct) : null; + var bytes = await client.DownloadToFileAsync(fileId, file.FullPath, config.MaxDownloadBytes, ct); if (guarded) context.Files!.NoteReplace(context.AgentId, context.RunId, file.FullPath, Core.Files.FileLedger.HashOf(file.FullPath)); + if (change is not null) + await context.History!.RecordAsync(change, Core.Files.FileAction.Replace, context.AgentId, context.RunId, ct, "RocketChat download"); return ToolResult.Ok(JsonSerializer.Serialize(new { diff --git a/src/ClawdDotNet.Web/Contracts/Dtos.cs b/src/ClawdDotNet.Web/Contracts/Dtos.cs index 109095b..1aa9223 100644 --- a/src/ClawdDotNet.Web/Contracts/Dtos.cs +++ b/src/ClawdDotNet.Web/Contracts/Dtos.cs @@ -375,3 +375,13 @@ public sealed record AgentCommunicationDto( string Content, DateTime Timestamp); +/// #719: One change in the history. Hashes are not sent — only whether a version exists. +public sealed record FileHistoryEntryDto( + long Id, string Workspace, string Path, string Action, string? AgentId, string? RunId, + bool HasBefore, bool HasAfter, long? SizeAfter, bool Stored, DateTime OccurredAtUtc, string? Note); + +/// #719: A version's content. only for text of a showable size. +public sealed record FileVersionDto(string? Text, bool Binary, bool TooLarge, bool Kept, long Bytes); + +/// #719: Putting a version back — which one, and why. Without a reason it is refused. +public sealed record FileRestoreInputDto(string? Version, string? Reason); diff --git a/src/ClawdDotNet.Web/Endpoints/ApiEndpoints.cs b/src/ClawdDotNet.Web/Endpoints/ApiEndpoints.cs index 5cde985..692bd2e 100644 --- a/src/ClawdDotNet.Web/Endpoints/ApiEndpoints.cs +++ b/src/ClawdDotNet.Web/Endpoints/ApiEndpoints.cs @@ -33,6 +33,7 @@ internal static class ApiEndpoints api.MapLogs(); api.MapOrganisation(); api.MapCases(); + api.MapFileHistory(); api.MapGet("/team", async (AppHost host, CancellationToken ct) => (await host.Team.SnapshotAsync(ct)).ToDto()); app.MapEvents(); diff --git a/src/ClawdDotNet.Web/Endpoints/FileHistoryEndpoints.cs b/src/ClawdDotNet.Web/Endpoints/FileHistoryEndpoints.cs new file mode 100644 index 0000000..e94b33f --- /dev/null +++ b/src/ClawdDotNet.Web/Endpoints/FileHistoryEndpoints.cs @@ -0,0 +1,87 @@ +using System.Text; +using ClawdDotNet.App; +using ClawdDotNet.Core.Files; +using ClawdDotNet.Web.Contracts; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Routing; + +namespace ClawdDotNet.Web.Endpoints; + +/// +/// #719: The history of the workspaces for the human — who changed what, what was in it +/// before, and putting a version back. Restoring is the human's only: agents have no way to it. +/// +internal static class FileHistoryEndpoints +{ + /// A version larger than this is offered as "too large to show", not sent. + public const int MaxShownBytes = 512 * 1024; + + public static void MapFileHistory(this RouteGroupBuilder api) + { + var history = api.MapGroup("/file-history"); + + history.MapGet("/", async (AppHost host, string? workspace, string? path, string? agent, string? run, int? limit, long? before, CancellationToken ct) => + { + if (host.Engine?.History is not { } files) + return Results.Ok(Array.Empty()); + var list = await files.ListAsync(workspace, path, agent, run, limit ?? 100, ct, before); + return Results.Ok(list.Select(ToDto).ToList()); + }); + + history.MapGet("/workspaces", (AppHost host) => + FileHistoryKeeper.WorkspacesOf(host.Instance.Agents, Core.Organisation.OrgChart.From(host.Instance)) + .Select(w => w.Workspace).ToList()); + + history.MapGet("/{id:long}/content", async (AppHost host, long id, string? version, CancellationToken ct) => + { + if (host.Engine?.History is not { } files || await files.FindAsync(id, ct) is not { } entry) + return Results.NotFound(new OutcomeDto(false, $"No history entry {id}.")); + var hash = version == "before" ? entry.HashBefore : entry.HashAfter; + if (hash is null) + return Results.Ok(new FileVersionDto(null, false, false, true, 0)); + if (files.Content(hash) is not { } bytes) + return Results.Ok(new FileVersionDto(null, false, false, false, entry.SizeAfter ?? 0)); + if (bytes.Length > MaxShownBytes) + return Results.Ok(new FileVersionDto(null, false, true, true, bytes.Length)); + return Results.Ok(AsText(bytes) is { } text + ? new FileVersionDto(text, false, false, true, bytes.Length) + : new FileVersionDto(null, true, false, true, bytes.Length)); + }); + + history.MapPost("/{id:long}/restore", async (AppHost host, long id, FileRestoreInputDto input, CancellationToken ct) => + { + if (host.Engine?.History is not { } files || await files.FindAsync(id, ct) is not { } entry) + return Results.NotFound(new OutcomeDto(false, $"No history entry {id}.")); + try + { + var roots = FileHistoryKeeper.WorkspacesOf(host.Instance.Agents, Core.Organisation.OrgChart.From(host.Instance)) + .Select(w => w.Root).ToList(); + var restored = await files.RestoreAsync(entry, input.Version == "before", "human", input.Reason ?? "", roots, ct); + return Results.Ok(ToDto(restored)); + } + catch (Exception ex) when (ex is ArgumentException or InvalidOperationException) + { + return Results.BadRequest(new OutcomeDto(false, ex.Message)); + } + }); + } + + /// The bytes as text if they are UTF-8 without control characters a text file would not have. + private static string? AsText(byte[] bytes) + { + try + { + var text = new UTF8Encoding(false, throwOnInvalidBytes: true).GetString(bytes); + return text.Any(c => char.IsControl(c) && c is not ('\n' or '\r' or '\t')) ? null : text; + } + catch (DecoderFallbackException) + { + return null; + } + } + + private static FileHistoryEntryDto ToDto(FileHistoryEntry e) => new( + e.Id, e.Workspace, e.Path, e.Action, e.AgentId, e.RunId, e.HashBefore is not null, e.HashAfter is not null, + e.SizeAfter, e.Stored, e.OccurredAtUtc, e.Note); +} diff --git a/tests/ClawdDotNet.Core.Tests/Files/FileHistoryTests.cs b/tests/ClawdDotNet.Core.Tests/Files/FileHistoryTests.cs new file mode 100644 index 0000000..8b19a2b --- /dev/null +++ b/tests/ClawdDotNet.Core.Tests/Files/FileHistoryTests.cs @@ -0,0 +1,253 @@ +using System.Text; +using ClawdDotNet.Core.Files; +using ClawdDotNet.Core.Storage; +using ClawdDotNet.Core.Tests.Infrastructure; +using Shouldly; + +namespace ClawdDotNet.Core.Tests.Files; + +/// +/// #719: The history of the workspaces — every change with its agent, its run and the version +/// before; changes made by hand found as external; a version put back only with a reason. +/// +public sealed class FileHistoryTests : IDisposable +{ + private readonly string _instance = Path.Combine(Path.GetTempPath(), "clawd-tests", "history-" + Guid.NewGuid().ToString("N")[..8]); + private readonly string _agents; + private readonly string _shared; + private readonly FakeTimeProvider _clock = new(new DateTimeOffset(2026, 9, 28, 10, 0, 0, TimeSpan.Zero)); + private readonly FileHistorySettings _settings = new(); + private readonly FileHistory _history; + + public FileHistoryTests() + { + _agents = Path.Combine(_instance, "Agents"); + _shared = Path.Combine(_agents, "SharedWorkspace"); + Directory.CreateDirectory(_shared); + _history = new FileHistory(_agents, new SqliteStorage(Path.Combine(_instance, "state.db")), () => _settings, _clock); + } + + public void Dispose() + { + SqliteStorage.ReleasePooledConnections(_instance); + try { Directory.Delete(_instance, recursive: true); } + catch { /* cleanup is secondary */ } + } + + private string File_(string name) => Path.Combine(_shared, name); + + private async Task Change(string name, string? content, string action, string agent = "a", string run = "r1") + { + var path = File_(name); + var change = await _history.BeforeAsync(path, "shared", default); + if (content is null) File.Delete(path); + else File.WriteAllText(path, content); + _clock.Advance(TimeSpan.FromMinutes(1)); + return await _history.RecordAsync(change, action, agent, run, default); + } + + private Task> All() => _history.ListAsync(null, null, null, null, 1000, default); + + // ═══════════════════ recording ═══════════════════ + + [Fact] + public async Task Every_change_is_journaled_with_agent_run_and_the_version_before() + { + await Change("plan.md", "v1", FileAction.Replace, "a", "r1"); + await Change("plan.md", "v2", FileAction.Replace, "b", "r2"); + await Change("plan.md", null, FileAction.Delete, "c", "r3"); + + var entries = (await All()).Reverse().ToList(); + + entries.Select(e => (e.Action, e.AgentId, e.RunId)).ShouldBe([ + (FileAction.Create, "a", "r1"), + (FileAction.Replace, "b", "r2"), + (FileAction.Delete, "c", "r3") + ]); + entries[0].Path.ShouldBe("SharedWorkspace/plan.md"); + Encoding.UTF8.GetString(_history.Content(entries[1].HashBefore)!).ShouldBe("v1"); + Encoding.UTF8.GetString(_history.Content(entries[2].HashBefore)!).ShouldBe("v2"); + entries[2].HashAfter.ShouldBeNull(); + } + + [Fact] + public async Task A_write_that_changes_nothing_is_not_a_change() + { + await Change("plan.md", "same", FileAction.Replace); + (await Change("plan.md", "same", FileAction.Replace)).ShouldBeNull(); + + (await All()).Count.ShouldBe(1); + } + + [Fact] + public async Task Each_content_is_stored_once() + { + await Change("a.md", "shared text", FileAction.Replace); + await Change("b.md", "shared text", FileAction.Replace); + + var objects = Directory.GetFiles(Path.Combine(_agents, FileHistory.FolderName, "objects"), "*", SearchOption.AllDirectories); + objects.Length.ShouldBe(1); + Path.GetExtension(objects[0]).ShouldBeEmpty("the backup leaves out .bak and .tmp — an object must not look like one"); + } + + [Fact] + public async Task A_file_too_large_is_journaled_but_not_kept() + { + _settings.MaxObjectMegabytes = 1; + var entry = await Change("big.bin", new string('x', 1024 * 1024 + 1), FileAction.Replace); + + entry!.Stored.ShouldBeFalse(); + entry.HashAfter.ShouldNotBeNull(); + _history.Content(entry.HashAfter).ShouldBeNull(); + } + + [Fact] + public async Task Switched_off_it_records_nothing() + { + _settings.Enabled = false; + (await Change("plan.md", "v1", FileAction.Replace)).ShouldBeNull(); + (await All()).ShouldBeEmpty(); + } + + // ═══════════════════ restoring ═══════════════════ + + [Fact] + public async Task A_version_is_put_back_with_a_reason_and_the_current_one_stays_in_the_history() + { + await Change("plan.md", "good", FileAction.Replace); + var bad = await Change("plan.md", "broken", FileAction.Replace, "b", "r2"); + + var restored = await _history.RestoreAsync(bad!, versionBefore: true, "human", "b broke the plan", [_shared], default); + + File.ReadAllText(File_("plan.md")).ShouldBe("good"); + restored.Action.ShouldBe(FileAction.Restore); + restored.AgentId.ShouldBe("human"); + restored.Note!.ShouldContain("b broke the plan"); + Encoding.UTF8.GetString(_history.Content(restored.HashBefore)!).ShouldBe("broken"); + } + + [Fact] + public async Task Without_a_reason_nothing_is_restored() + { + await Change("plan.md", "good", FileAction.Replace); + var bad = await Change("plan.md", "broken", FileAction.Replace); + + await Should.ThrowAsync(() => _history.RestoreAsync(bad!, true, "human", " ", [_shared], default)); + File.ReadAllText(File_("plan.md")).ShouldBe("broken"); + } + + [Fact] + public async Task A_file_outside_the_named_workspaces_is_not_restored() + { + await Change("plan.md", "good", FileAction.Replace); + var bad = await Change("plan.md", "broken", FileAction.Replace); + + await Should.ThrowAsync(() => + _history.RestoreAsync(bad!, true, "human", "reason", [Path.Combine(_agents, "Agent-other", "Workspace")], default)); + } + + // ═══════════════════ outside the tools ═══════════════════ + + [Fact] + public async Task The_first_look_only_notes_what_is_there() + { + File.WriteAllText(File_("old.md"), "from before the history"); + + (await _history.ReconcileAsync([("shared", _shared)], default)).ShouldBe(0); + (await All()).ShouldBeEmpty(); + } + + [Fact] + public async Task A_change_by_hand_is_recorded_as_external_with_both_versions() + { + await Change("plan.md", "by the agent", FileAction.Replace); + File.WriteAllText(File_("plan.md"), "edited by hand"); + File.SetLastWriteTimeUtc(File_("plan.md"), DateTime.UtcNow.AddMinutes(5)); + + (await _history.ReconcileAsync([("shared", _shared)], default)).ShouldBe(1); + + var external = (await All())[0]; + external.Action.ShouldBe(FileAction.External); + external.AgentId.ShouldBeNull(); + Encoding.UTF8.GetString(_history.Content(external.HashBefore)!).ShouldBe("by the agent"); + Encoding.UTF8.GetString(_history.Content(external.HashAfter)!).ShouldBe("edited by hand"); + } + + [Fact] + public async Task A_deletion_by_hand_is_recorded_too() + { + await Change("plan.md", "v1", FileAction.Replace); + File.Delete(File_("plan.md")); + + (await _history.ReconcileAsync([("shared", _shared)], default)).ShouldBe(1); + (await All())[0].HashAfter.ShouldBeNull(); + } + + [Fact] + public async Task Nothing_changed_means_nothing_recorded() + { + await Change("plan.md", "v1", FileAction.Replace); + await _history.ReconcileAsync([("shared", _shared)], default); + + (await _history.ReconcileAsync([("shared", _shared)], default)).ShouldBe(0); + (await All()).Count.ShouldBe(1); + } + + [Fact] + public async Task The_history_does_not_record_itself() + { + await Change("plan.md", "v1", FileAction.Replace); + + await _history.ReconcileAsync([("everything", _agents)], default); + await _history.ReconcileAsync([("everything", _agents)], default); + + (await All()).ShouldAllBe(e => !e.Path.Contains(FileHistory.FolderName)); + } + + // ═══════════════════ retention ═══════════════════ + + [Fact] + public async Task Old_entries_go_but_the_newest_of_each_file_stays() + { + _settings.RetentionDays = 30; + await Change("plan.md", "v1", FileAction.Replace); + await Change("plan.md", "v2", FileAction.Replace); + await Change("once.md", "only", FileAction.Replace); + _clock.Advance(TimeSpan.FromDays(40)); + + await _history.PurgeAsync(default); + + var left = await All(); + left.Select(e => e.Path).ShouldBe(["SharedWorkspace/once.md", "SharedWorkspace/plan.md"], ignoreOrder: true); + } + + [Fact] + public async Task Beyond_the_version_limit_the_oldest_go() + { + _settings.MaxVersionsPerFile = 2; + for (var i = 1; i <= 4; i++) + await Change("plan.md", $"v{i}", FileAction.Replace); + + await _history.PurgeAsync(default); + + (await All()).Count.ShouldBe(2); + } + + [Fact] + public async Task A_version_nobody_points_to_is_removed() + { + _settings.MaxVersionsPerFile = 1; + await Change("plan.md", "v1", FileAction.Replace); + await Change("plan.md", "v2", FileAction.Replace); + await Change("plan.md", "v3", FileAction.Replace); + var v1 = Convert.ToHexStringLower(System.Security.Cryptography.SHA256.HashData("v1"u8)); + _clock.Advance(TimeSpan.FromHours(1)); + foreach (var file in Directory.GetFiles(Path.Combine(_agents, FileHistory.FolderName, "objects"), "*", SearchOption.AllDirectories)) + File.SetLastWriteTimeUtc(file, _clock.GetUtcNow().UtcDateTime.AddHours(-1)); + + await _history.PurgeAsync(default); + + _history.Content(v1).ShouldBeNull("v1 is referenced by no remaining entry"); + _history.Content((await All())[0].HashAfter).ShouldNotBeNull(); + } +} diff --git a/tests/ClawdDotNet.Tools.Tests/FileRW/FileHistoryRecordingTests.cs b/tests/ClawdDotNet.Tools.Tests/FileRW/FileHistoryRecordingTests.cs new file mode 100644 index 0000000..85352c3 --- /dev/null +++ b/tests/ClawdDotNet.Tools.Tests/FileRW/FileHistoryRecordingTests.cs @@ -0,0 +1,119 @@ +using System.Text; +using System.Text.Json; +using ClawdDotNet.Core.Files; +using ClawdDotNet.Core.Storage; +using ClawdDotNet.Core.Tools; +using ClawdDotNet.Tools.FileRW; +using Microsoft.Extensions.Logging.Abstractions; +using Shouldly; + +namespace ClawdDotNet.Tools.Tests.FileRW; + +/// +/// #719: FileRW is the main way into the workspaces — every change it makes lands in the +/// history, with the agent and the run of the call. +/// +public sealed class FileHistoryRecordingTests : IDisposable +{ + private readonly string _instance = Path.Combine(Path.GetTempPath(), "clawd-tests", "fh-" + Guid.NewGuid().ToString("N")[..8]); + private readonly string _agents; + private readonly FileHistory _history; + private readonly FileRWTool _tool = new(); + + public FileHistoryRecordingTests() + { + _agents = Path.Combine(_instance, "Agents"); + Directory.CreateDirectory(Path.Combine(_agents, "SharedWorkspace")); + _history = new FileHistory(_agents, new SqliteStorage(Path.Combine(_instance, "state.db"))); + } + + public void Dispose() + { + SqliteStorage.ReleasePooledConnections(_instance); + try { Directory.Delete(_instance, recursive: true); } + catch { /* cleanup is secondary */ } + } + + private Task Run(string agentId, object input) + { + var personal = Path.Combine(_agents, "Agent-" + agentId, "Workspace"); + Directory.CreateDirectory(personal); + var context = new AgentToolContext(agentId, "inst", + new Dictionary { ["sharedAccessLevel"] = "Admin" }, + new NullStateStore(), NullLogger.Instance, CancellationToken.None, + WorkspacePath: personal, SharedWorkspacePath: Path.Combine(_agents, "SharedWorkspace")) + { + History = _history, + RunId = "run-" + agentId + }; + return _tool.ExecuteAsync(JsonSerializer.SerializeToElement(input), context, CancellationToken.None); + } + + private async Task> Journal() => (await _history.ListAsync(null, null, null, null, 100, default)).Reverse().ToList(); + + [Fact] + public async Task Write_append_and_delete_are_recorded_with_agent_and_run() + { + await Run("a", new { action = "write", workspace = "shared", path = "notes/plan.md", content = "v1" }); + await Run("b", new { action = "append", workspace = "shared", path = "notes/plan.md", content = "\n+b" }); + await Run("c", new { action = "delete", workspace = "shared", path = "notes/plan.md" }); + + (await Journal()).Select(e => (e.Action, e.AgentId, e.RunId, e.Path, e.Workspace)).ShouldBe([ + (FileAction.Create, "a", "run-a", "SharedWorkspace/notes/plan.md", "shared"), + (FileAction.Append, "b", "run-b", "SharedWorkspace/notes/plan.md", "shared"), + (FileAction.Delete, "c", "run-c", "SharedWorkspace/notes/plan.md", "shared") + ]); + } + + [Fact] + public async Task What_a_delete_removed_can_be_read_back() + { + await Run("a", new { action = "write", workspace = "shared", path = "plan.md", content = "precious" }); + await Run("b", new { action = "delete", workspace = "shared", path = "plan.md" }); + + var delete = (await Journal()).Last(); + Encoding.UTF8.GetString(_history.Content(delete.HashBefore)!).ShouldBe("precious"); + } + + [Fact] + public async Task Deleting_a_folder_records_each_file() + { + await Run("a", new { action = "write", workspace = "shared", path = "old/1.md", content = "one" }); + await Run("a", new { action = "write", workspace = "shared", path = "old/2.md", content = "two" }); + + await Run("a", new { action = "delete", workspace = "shared", path = "old" }); + + (await Journal()).Count(e => e.Action == FileAction.Delete).ShouldBe(2); + } + + [Fact] + public async Task A_copy_is_recorded_at_its_destination() + { + await Run("a", new { action = "write", workspace = "shared", path = "src.md", content = "x" }); + await Run("a", new { action = "copy", workspace = "shared", path = "src.md", destinationPath = "dst.md" }); + + (await Journal()).Last().ShouldSatisfyAllConditions( + e => e.Action.ShouldBe(FileAction.Copy), + e => e.Path.ShouldBe("SharedWorkspace/dst.md")); + } + + [Fact] + public async Task The_personal_workspace_is_recorded_under_its_owner() + { + await Run("a", new { action = "write", path = "mine.md", content = "x" }); + + var entry = (await Journal()).Single(); + entry.Workspace.ShouldBe("personal:a"); + entry.Path.ShouldBe("Agent-a/Workspace/mine.md"); + } + + [Fact] + public async Task A_stock_data_point_records_the_file_and_the_index() + { + await Run("a", new { action = "stock_add", ticker = "ACME", category = "news", source = "test", title = "A title" }); + + var journal = await Journal(); + journal.ShouldContain(e => e.Path.EndsWith("/_index.json") && e.Action == FileAction.Create); + journal.ShouldContain(e => e.Path.Contains("/stocks/ACME/") && !e.Path.EndsWith("_index.json")); + } +}