ClawdDotNet/src/ClawdDotNet.Core/Files/FileHistory.cs

using System.Globalization;
using System.Security.Cryptography;
using System.Text.Json.Serialization;
using ClawdDotNet.Core.Storage;
using Microsoft.Data.Sqlite;

namespace ClawdDotNet.Core.Files;

/// <summary>#719: How long the history of the workspaces is kept (<c>fileHistory</c> in <c>instance.json</c>).</summary>
public sealed class FileHistorySettings
{
    [JsonPropertyName("enabled")]
    public bool Enabled { get; set; } = true;

    /// <summary>Entries older than this are removed — the newest one of each file always stays.</summary>
    [JsonPropertyName("retentionDays")]
    public int RetentionDays { get; set; } = 90;

    /// <summary>At most this many versions per file.</summary>
    [JsonPropertyName("maxVersionsPerFile")]
    public int MaxVersionsPerFile { get; set; } = 50;

    /// <summary>Larger files are journaled with their hash, but their content is not kept.</summary>
    [JsonPropertyName("maxObjectMegabytes")]
    public int MaxObjectMegabytes { get; set; } = 20;

    /// <summary>How often changes made outside the tools are looked for.</summary>
    [JsonPropertyName("reconcileMinutes")]
    public int ReconcileMinutes { get; set; } = 60;
}

/// <summary>#719: What happened to a file.</summary>
public static class FileAction
{
    public const string Create = "create";
    public const string Replace = "replace";
    public const string Append = "append";
    public const string Copy = "copy";
    public const string Delete = "delete";
    public const string Restore = "restore";

    /// <summary>Changed outside every tool — a human at the file system, found by the reconciliation.</summary>
    public const string External = "external";
}

/// <summary>#719: One line of the history. <see cref="Stored"/>: the version after it can be shown and restored.</summary>
public sealed record FileHistoryEntry(
    long Id,
    string Workspace,
    string Path,
    string Action,
    string? AgentId,
    string? RunId,
    string? HashBefore,
    string? HashAfter,
    long? SizeAfter,
    bool Stored,
    DateTime OccurredAtUtc,
    string? Note);

/// <summary>#719: A change in progress: the version before, captured before the tool touches the file.</summary>
public sealed record FileChange(string FullPath, string Workspace, string? HashBefore);

/// <summary>
/// #719: The history of the workspaces — who created, replaced, appended to or deleted which
/// file, in which run, and what was in it before. For post-mortems of failed work and for
/// data that does not add up.
///
/// <para><b>Our own store, not Git.</b> Git would bring diff and history ready-made, but a
/// commit per write with many agents writing in parallel means locks and waiting; the link
/// to the run and the audit would be extra work; binary files would bloat it for good; and
/// removing what is past its retention means rewriting history. This store fits the pattern
/// the instance already has: SQLite as the truth, append-only, everything in the backup.</para>
///
/// <para><b>Each content once.</b> Versions are kept under their SHA-256 in
/// <c>Agents/.history/objects/</c>, so an unchanged file costs nothing and an append costs
/// one version. The journal in <c>state.db</c> links every change to its run — and through
/// it to the receipt and the transcript. No <c>.bak</c> or <c>.tmp</c> names: the backup
/// leaves those out.</para>
///
/// <para><b>One way in.</b> Every tool that changes a file calls <see cref="BeforeAsync"/>
/// before and <see cref="RecordAsync"/> after, under the file's lock. What changes outside
/// the tools is found by <see cref="ReconcileAsync"/> and recorded as external.</para>
/// </summary>
public sealed class FileHistory
{
    public const string FolderName = ".history";

    private readonly string _agentsDirectory;
    private readonly SqliteStorage _storage;
    private readonly Func<FileHistorySettings> _settings;
    private readonly TimeProvider _clock;

    public FileHistory(string agentsDirectory, SqliteStorage storage, Func<FileHistorySettings>? settings = null, TimeProvider? clock = null)
    {
        _agentsDirectory = Path.GetFullPath(agentsDirectory);
        _storage = storage;
        _settings = settings ?? (() => new FileHistorySettings());
        _clock = clock ?? TimeProvider.System;
    }

    private string ObjectsDirectory => Path.Combine(_agentsDirectory, FolderName, "objects");

    private bool Enabled => _settings().Enabled;

    /// <summary>The workspace's own files — never the history itself.</summary>
    public bool Covers(string fullPath)
        => !IsInside(Path.GetFullPath(fullPath), Path.Combine(_agentsDirectory, FolderName));

    /// <summary>
    /// #195: Whether a delete of this file can be undone — the history is on, covers the file
    /// and keeps content of its size. Only then is deleting it no irreversible act.
    /// </summary>
    public bool CanKeep(string fullPath)
    {
        fullPath = Path.GetFullPath(fullPath);
        return Enabled && Covers(fullPath) && File.Exists(fullPath)
               && new FileInfo(fullPath).Length <= (long)_settings().MaxObjectMegabytes * 1024 * 1024;
    }

    /// <summary>#195: Whether the content of a version is still kept.</summary>
    public bool CanKeepContent(string? hash)
        => !string.IsNullOrWhiteSpace(hash) && hash.Length == 64 && hash.All(Uri.IsHexDigit) && File.Exists(ObjectPath(hash.ToLowerInvariant()));

    /// <summary>#195: How long a deleted file's content stays restorable.</summary>
    public int RetentionDays => Math.Max(1, _settings().RetentionDays);

    // ─── Recording ───

    /// <summary>Keeps the version the file has now, before a tool changes it.</summary>
    public Task<FileChange> BeforeAsync(string fullPath, string workspace, CancellationToken ct)
    {
        fullPath = Path.GetFullPath(fullPath);
        string? hash = null;
        if (Enabled && Covers(fullPath) && File.Exists(fullPath))
            hash = Store(fullPath).Hash;
        return Task.FromResult(new FileChange(fullPath, workspace, hash));
    }

    /// <summary>
    /// Journals what the tool did. A write that left the content as it was is not a change and
    /// is not recorded. The action is read from the file: gone is a delete, new is a create.
    /// </summary>
    public async Task<FileHistoryEntry?> RecordAsync(
        FileChange change, string action, string? agentId, string? runId, CancellationToken ct, string? note = null)
    {
        if (!Enabled || !Covers(change.FullPath))
            return null;

        string? after = null;
        long? size = null;
        var stored = false;
        if (File.Exists(change.FullPath))
        {
            (after, stored) = Store(change.FullPath);
            size = new FileInfo(change.FullPath).Length;
        }

        if (after == change.HashBefore && action != FileAction.Restore)
            return null;
        if (after is null)
            action = FileAction.Delete;
        else if (change.HashBefore is null && action is FileAction.Replace or FileAction.Append)
            action = FileAction.Create;

        var entry = new FileHistoryEntry(0, change.Workspace, Relative(change.FullPath), action, agentId, runId,
            change.HashBefore, after, size, stored, Now, note);
        return await AppendAsync(entry, change.FullPath, ct);
    }

    private async Task<FileHistoryEntry> AppendAsync(FileHistoryEntry entry, string fullPath, CancellationToken ct)
    {
        var id = await _storage.WriteAsync(async conn =>
        {
            using var cmd = conn.CreateCommand();
            cmd.CommandText = """
                INSERT INTO FileHistory (Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note)
                VALUES (@ws, @path, @action, @agent, @run, @before, @after, @size, @stored, @at, @note);
                SELECT last_insert_rowid();
                """;
            cmd.Parameters.AddWithValue("@ws", entry.Workspace);
            cmd.Parameters.AddWithValue("@path", entry.Path);
            cmd.Parameters.AddWithValue("@action", entry.Action);
            cmd.Parameters.AddWithValue("@agent", (object?)entry.AgentId ?? DBNull.Value);
            cmd.Parameters.AddWithValue("@run", (object?)entry.RunId ?? DBNull.Value);
            cmd.Parameters.AddWithValue("@before", (object?)entry.HashBefore ?? DBNull.Value);
            cmd.Parameters.AddWithValue("@after", (object?)entry.HashAfter ?? DBNull.Value);
            cmd.Parameters.AddWithValue("@size", (object?)entry.SizeAfter ?? DBNull.Value);
            cmd.Parameters.AddWithValue("@stored", entry.Stored ? 1 : 0);
            cmd.Parameters.AddWithValue("@at", Format(entry.OccurredAtUtc));
            cmd.Parameters.AddWithValue("@note", (object?)entry.Note ?? DBNull.Value);
            var rowId = (long)(await cmd.ExecuteScalarAsync(ct))!;
            await SetStateAsync(conn, entry.Path, entry.Workspace, fullPath, entry.HashAfter, ct);
            return rowId;
        }, ct);
        return entry with { Id = id };
    }

    /// <summary>What the history knows the file to be now — the reconciliation compares against it.</summary>
    private static async Task SetStateAsync(SqliteConnection conn, string path, string workspace, string fullPath, string? hash, CancellationToken ct)
    {
        using var cmd = conn.CreateCommand();
        if (hash is null)
        {
            cmd.CommandText = "DELETE FROM FileHistoryState WHERE Path = @path";
            cmd.Parameters.AddWithValue("@path", path);
        }
        else
        {
            var info = new FileInfo(fullPath);
            cmd.CommandText = """
                INSERT INTO FileHistoryState (Path, Workspace, Hash, Size, Modified) VALUES (@path, @ws, @hash, @size, @mod)
                ON CONFLICT(Path) DO UPDATE SET Workspace = excluded.Workspace, Hash = excluded.Hash, Size = excluded.Size, Modified = excluded.Modified
                """;
            cmd.Parameters.AddWithValue("@path", path);
            cmd.Parameters.AddWithValue("@ws", workspace);
            cmd.Parameters.AddWithValue("@hash", hash);
            cmd.Parameters.AddWithValue("@size", info.Exists ? info.Length : 0);
            cmd.Parameters.AddWithValue("@mod", info.Exists ? info.LastWriteTimeUtc.Ticks : 0);
        }
        await cmd.ExecuteNonQueryAsync(ct);
    }

    /// <summary>Stores the file's content under its hash, once. Too large: the hash only.</summary>
    private (string Hash, bool Stored) Store(string fullPath)
    {
        string hash;
        using (var stream = new FileStream(fullPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete))
            hash = Convert.ToHexStringLower(SHA256.HashData(stream));

        var target = ObjectPath(hash);
        if (File.Exists(target))
            return (hash, true);
        if (new FileInfo(fullPath).Length > (long)_settings().MaxObjectMegabytes * 1024 * 1024)
            return (hash, false);

        Directory.CreateDirectory(Path.GetDirectoryName(target)!);
        var temp = target + "." + Guid.NewGuid().ToString("N")[..8] + ".part";
        File.Copy(fullPath, temp, overwrite: true);
        try
        {
            File.Move(temp, target, overwrite: false);
        }
        catch (IOException) when (File.Exists(target))
        {
            File.Delete(temp); // someone else stored the same content meanwhile
        }
        return (hash, true);
    }

    private string ObjectPath(string hash) => Path.Combine(ObjectsDirectory, hash[..2], hash);

    // ─── Reading ───

    /// <summary>The newest entries, filtered — for the history view.</summary>
    public async Task<IReadOnlyList<FileHistoryEntry>> ListAsync(
        string? workspace, string? path, string? agentId, string? runId, int limit, CancellationToken ct, long? beforeId = null)
    {
        await using var conn = await _storage.OpenConnectionAsync(ct);
        using var cmd = conn.CreateCommand();
        var where = new List<string>();
        if (!string.IsNullOrWhiteSpace(workspace)) { where.Add("Workspace = @ws COLLATE NOCASE"); cmd.Parameters.AddWithValue("@ws", workspace); }
        if (!string.IsNullOrWhiteSpace(path)) { where.Add("Path = @path"); cmd.Parameters.AddWithValue("@path", path); }
        if (!string.IsNullOrWhiteSpace(agentId)) { where.Add("AgentId = @agent COLLATE NOCASE"); cmd.Parameters.AddWithValue("@agent", agentId); }
        if (!string.IsNullOrWhiteSpace(runId)) { where.Add("RunId = @run"); cmd.Parameters.AddWithValue("@run", runId); }
        if (beforeId is { } before) { where.Add("Id < @beforeId"); cmd.Parameters.AddWithValue("@beforeId", before); }
        cmd.CommandText = $"""
            SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note
              FROM FileHistory
             {(where.Count > 0 ? "WHERE " + string.Join(" AND ", where) : "")}
             ORDER BY Id DESC
             LIMIT @limit
            """;
        cmd.Parameters.AddWithValue("@limit", Math.Clamp(limit, 1, 1000));

        var result = new List<FileHistoryEntry>();
        await using var reader = await cmd.ExecuteReaderAsync(ct);
        while (await reader.ReadAsync(ct))
            result.Add(Read(reader));
        return result;
    }

    public async Task<FileHistoryEntry?> FindAsync(long id, CancellationToken ct)
    {
        await using var conn = await _storage.OpenConnectionAsync(ct);
        using var cmd = conn.CreateCommand();
        cmd.CommandText = """
            SELECT Id, Workspace, Path, Action, AgentId, RunId, HashBefore, HashAfter, SizeAfter, Stored, OccurredAt, Note
              FROM FileHistory WHERE Id = @id
            """;
        cmd.Parameters.AddWithValue("@id", id);
        await using var reader = await cmd.ExecuteReaderAsync(ct);
        return await reader.ReadAsync(ct) ? Read(reader) : null;
    }

    /// <summary>A stored version's bytes, or <c>null</c> if it was not kept (too large, or pruned).</summary>
    public byte[]? Content(string? hash)
    {
        if (string.IsNullOrWhiteSpace(hash) || hash.Length != 64 || !hash.All(Uri.IsHexDigit))
            return null;
        var path = ObjectPath(hash.ToLowerInvariant());
        return File.Exists(path) ? File.ReadAllBytes(path) : null;
    }

    /// <summary>The file on disk that a journal path stands for.</summary>
    public string FullPathOf(string relativePath)
        => Path.GetFullPath(Path.IsPathRooted(relativePath) ? relativePath : Path.Combine(_agentsDirectory, relativePath));

    // ─── Restoring ───

    /// <summary>
    /// Puts a version back — by a human only, with a reason, as a journal line of its own. The
    /// target must lie in a workspace the caller names; the history's own folder never is one.
    /// </summary>
    public async Task<FileHistoryEntry> RestoreAsync(
        FileHistoryEntry source, bool versionBefore, string by, string reason,
        IReadOnlyList<string> workspaceRoots, CancellationToken ct)
    {
        if (string.IsNullOrWhiteSpace(reason))
            throw new ArgumentException("A restore needs a reason.");
        var hash = versionBefore ? source.HashBefore : source.HashAfter;
        var content = Content(hash)
            ?? throw new InvalidOperationException(hash is null
                ? "That version is the file's absence — there is nothing to restore."
                : "That version was not kept (too large, or past its retention).");

        var target = FullPathOf(source.Path);
        if (!Covers(target) || !workspaceRoots.Any(root => IsInside(target, Path.GetFullPath(root))))
            throw new InvalidOperationException("The file does not lie in a workspace.");

        var change = await BeforeAsync(target, source.Workspace, ct);
        Directory.CreateDirectory(Path.GetDirectoryName(target)!);
        await AtomicFile.WriteAllBytesAsync(target, content, ct);
        return (await RecordAsync(change, FileAction.Restore, by, null, ct,
            $"Restored the version of entry {source.Id} ({(versionBefore ? "before" : "after")}): {reason.Trim()}"))!;
    }

    // ─── Outside the tools ───

    /// <summary>
    /// Looks for changes made outside the tools and journals them as external. The first time
    /// a file is seen, it is only noted — the whole existing workspace is no change. Size and
    /// modification time decide whether a file is hashed at all, so an unchanged tree costs a
    /// directory walk.
    /// </summary>
    public async Task<int> ReconcileAsync(IReadOnlyList<(string Workspace, string Root)> workspaces, CancellationToken ct)
    {
        if (!Enabled)
            return 0;

        var known = new Dictionary<string, (string Workspace, string Hash, long Size, long Modified)>(StringComparer.Ordinal);
        await using (var conn = await _storage.OpenConnectionAsync(ct))
        {
            using var cmd = conn.CreateCommand();
            cmd.CommandText = "SELECT Path, Workspace, Hash, Size, Modified FROM FileHistoryState";
            await using var reader = await cmd.ExecuteReaderAsync(ct);
            while (await reader.ReadAsync(ct))
                known[reader.GetString(0)] = (reader.GetString(1), reader.GetString(2), reader.GetInt64(3), reader.GetInt64(4));
        }

        var changes = 0;
        var seen = new HashSet<string>(StringComparer.Ordinal);
        var coveredRoots = new List<string>();
        foreach (var (workspace, root) in workspaces)
        {
            if (!Directory.Exists(root))
                continue;
            coveredRoots.Add(Path.GetFullPath(root));
            foreach (var file in RegularFiles(root))
            {
                ct.ThrowIfCancellationRequested();
                if (!Covers(file))
                    continue;
                var relative = Relative(file);
                if (!seen.Add(relative))
                    continue; // a workspace inside another — counted once

                var info = new FileInfo(file);
                if (known.TryGetValue(relative, out var state)
                    && state.Size == info.Length && state.Modified == info.LastWriteTimeUtc.Ticks)
                    continue;

                if (state.Hash is null)
                {
                    // Seen for the first time: noted, not a change.
                    await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, HashOnly(file), ct), ct);
                    continue;
                }

                var (hash, stored) = Store(file);
                if (hash == state.Hash)
                {
                    await _storage.WriteAsync(conn => SetStateAsync(conn, relative, workspace, file, hash, ct), ct);
                    continue;
                }
                await AppendAsync(new FileHistoryEntry(0, workspace, relative, FileAction.External, null, null,
                    state.Hash, hash, info.Length, stored, Now, null), file, ct);
                changes++;
            }
        }

        // Known files that are gone — deleted outside the tools.
        foreach (var (relative, state) in known)
        {
            if (seen.Contains(relative))
                continue;
            var full = FullPathOf(relative);
            if (File.Exists(full) || !coveredRoots.Any(root => IsInside(full, root)))
                continue;
            await AppendAsync(new FileHistoryEntry(0, state.Workspace, relative, FileAction.External, null, null,
                state.Hash, null, null, false, Now, "deleted outside the tools"), full, ct);
            changes++;
        }
        return changes;
    }

    private static string HashOnly(string file)
    {
        using var stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete);
        return Convert.ToHexStringLower(SHA256.HashData(stream));
    }

    /// <summary>Regular files only — no links out of the workspace, no pipes that would block.</summary>
    private static IEnumerable<string> RegularFiles(string root)
    {
        var pending = new Stack<string>([Path.GetFullPath(root)]);
        while (pending.Count > 0)
        {
            var dir = pending.Pop();
            IEnumerable<FileSystemInfo> entries;
            try { entries = new DirectoryInfo(dir).EnumerateFileSystemInfos().ToList(); }
            catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) { continue; }
            foreach (var entry in entries)
            {
                switch (Runtimes.FileKind.Of(entry))
                {
                    case Runtimes.FileKind.Kind.Directory: pending.Push(entry.FullName); break;
                    case Runtimes.FileKind.Kind.Regular: yield return entry.FullName; break;
                }
            }
        }
    }

    // ─── Retention ───

    /// <summary>
    /// Removes entries past their retention and beyond the version limit — never a file's
    /// newest — and then every stored version no entry points to any more.
    /// </summary>
    public async Task<int> PurgeAsync(CancellationToken ct)
    {
        var settings = _settings();
        var cutoff = Format(Now.AddDays(-Math.Max(1, settings.RetentionDays)));
        var keep = Math.Max(1, settings.MaxVersionsPerFile);

        var removed = await _storage.WriteAsync(async conn =>
        {
            using var cmd = conn.CreateCommand();
            cmd.CommandText = """
                DELETE FROM FileHistory WHERE Id IN (
                    SELECT Id FROM (
                        SELECT Id, OccurredAt,
                               ROW_NUMBER() OVER (PARTITION BY Path ORDER BY Id DESC) AS Newest
                          FROM FileHistory)
                     WHERE Newest > 1 AND (OccurredAt < @cutoff OR Newest > @keep))
                """;
            cmd.Parameters.AddWithValue("@cutoff", cutoff);
            cmd.Parameters.AddWithValue("@keep", keep);
            return await cmd.ExecuteNonQueryAsync(ct);
        }, ct);

        // Objects nobody points to. Collected after the deletion: a version stored meanwhile is
        // either referenced or younger than the walk, and the walk skips unfinished ".part" files.
        var referenced = new HashSet<string>(StringComparer.Ordinal);
        await using (var conn = await _storage.OpenConnectionAsync(ct))
        {
            using var cmd = conn.CreateCommand();
            cmd.CommandText = "SELECT HashBefore FROM FileHistory UNION SELECT HashAfter FROM FileHistory UNION SELECT Hash FROM FileHistoryState";
            await using var reader = await cmd.ExecuteReaderAsync(ct);
            while (await reader.ReadAsync(ct))
                if (!reader.IsDBNull(0))
                    referenced.Add(reader.GetString(0));
        }
        if (Directory.Exists(ObjectsDirectory))
        {
            var threshold = Now.AddMinutes(-10);
            foreach (var file in Directory.EnumerateFiles(ObjectsDirectory, "*", SearchOption.AllDirectories))
            {
                var name = Path.GetFileName(file);
                if (name.Length != 64 || referenced.Contains(name) || File.GetLastWriteTimeUtc(file) > threshold)
                    continue;
                try { File.Delete(file); }
                catch (IOException) { }
            }
        }
        return removed;
    }

    // ─── Helpers ───

    private DateTime Now => _clock.GetUtcNow().UtcDateTime;

    private static string Format(DateTime utc) => utc.ToString("O", CultureInfo.InvariantCulture);

    /// <summary>Relative to the Agents folder with '/', or the full path for a workspace elsewhere.</summary>
    private string Relative(string fullPath)
    {
        var relative = Path.GetRelativePath(_agentsDirectory, fullPath);
        return relative.StartsWith("..", StringComparison.Ordinal) || Path.IsPathRooted(relative)
            ? fullPath
            : relative.Replace(Path.DirectorySeparatorChar, '/');
    }

    private static bool IsInside(string path, string root)
    {
        var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
        root = Path.TrimEndingDirectorySeparator(root);
        return path.Equals(root, comparison) || path.StartsWith(root + Path.DirectorySeparatorChar, comparison);
    }

    private static FileHistoryEntry Read(SqliteDataReader r) => new(
        r.GetInt64(0), r.GetString(1), r.GetString(2), r.GetString(3),
        r.IsDBNull(4) ? null : r.GetString(4),
        r.IsDBNull(5) ? null : r.GetString(5),
        r.IsDBNull(6) ? null : r.GetString(6),
        r.IsDBNull(7) ? null : r.GetString(7),
        r.IsDBNull(8) ? null : r.GetInt64(8),
        r.GetInt32(9) != 0,
        DateTime.Parse(r.GetString(10), CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind),
        r.IsDBNull(11) ? null : r.GetString(11));
}