ClawdDotNet/tests/ClawdDotNet.Core.Tests/Backup/BackupServiceTests.cs

using System.IO.Compression;
using System.Text.Json;
using ClawdDotNet.Core.Backup;
using ClawdDotNet.Core.Memory;
using ClawdDotNet.Core.Security;
using ClawdDotNet.Core.Storage;
using ClawdDotNet.Core.Tests.Infrastructure;
using Shouldly;

namespace ClawdDotNet.Core.Tests.Backup;

/// <summary>
/// An unchecked restore is not a backup, but a guess.
/// That's why the focus here is on the complete round trip.
///
/// Runs in <see cref="SecretKeyCollection"/>, because rewriting the secrets
/// when backing up uses the same static key as the secret tests.
/// </summary>
[Collection(SecretKeyCollection.Name)]
public sealed class BackupServiceTests : IDisposable
{
    private readonly string _root;
    private readonly string _instanceDir;
    private readonly BackupService _service = new();

    private const string ApiKeyPlain = "sk-or-v1-streng-geheim-12345";
    private const string MailPasswordPlain = "mail-passwort-geheim";

    public BackupServiceTests()
    {
        _root = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"));
        _instanceDir = Path.Combine(_root, "Instance-Test");
        Directory.CreateDirectory(_instanceDir);
    }

    public void Dispose()
    {
        ClawdDotNet.Core.Storage.SqliteStorage.ReleasePooledConnections(_root);
        try { Directory.Delete(_root, recursive: true); }
        catch { /* Cleaning up is secondary */ }
    }

    // ─── Building a realistic instance ───

    private async Task BuildInstanceAsync()
    {
        // Instance configuration — credentials are DPAPI-protected as in operation.
        var instanceSettings = $$"""
            {
              "instanceId": "test-01",
              "instanceName": "Testinstanz",
              "openRouterApiKey": "{{SecretProtector.Protect(ApiKeyPlain)}}",
              "webServerPort": 8080,
              "einUnbekanntesFeld": "muss erhalten bleiben"
            }
            """;
        AtomicFile.WriteAllText(Path.Combine(_instanceDir, "InstanceSettings.json"), instanceSettings);

        // Agent
        var agentDir = Path.Combine(_instanceDir, "Agents", "Agent-Analyst");
        Directory.CreateDirectory(Path.Combine(agentDir, "Workspace"));
        Directory.CreateDirectory(Path.Combine(agentDir, "Logs"));

        var agentSettings = $$"""
            {
              "agentId": "analyst",
              "displayName": "Analyst",
              "model": "anthropic/claude-sonnet-4-5",
              "tools": {
                "Mail": {
                  "smtpHost": "smtp.example.com",
                  "password": "{{SecretProtector.Protect(MailPasswordPlain)}}"
                }
              }
            }
            """;
        AtomicFile.WriteAllText(Path.Combine(agentDir, "AgentSettings.json"), agentSettings);
        AtomicFile.WriteAllText(Path.Combine(agentDir, "Identity.md"), "# Identity\nDer Analyst.");
        AtomicFile.WriteAllText(Path.Combine(agentDir, "Soul.md"), "# Soul\nGründlich und knapp.");
        AtomicFile.WriteAllText(Path.Combine(agentDir, "ChatHistory.json"), """[{"role":"user"}]""");
        AtomicFile.WriteAllText(Path.Combine(agentDir, "Workspace", "bericht.md"), "# Bericht\nInhalt.");
        AtomicFile.WriteAllText(Path.Combine(agentDir, "Logs", "lauf.log"), "viele Zeilen Protokoll");

        // Database with a memory — the most valuable part.
        var storage = new SqliteStorage(Path.Combine(_instanceDir, "state.db"));
        var memory = new SqliteMemoryRepository(storage);
        await memory.RememberAsync(new MemoryEntry
        {
            Scope = MemoryScope.Agent,
            OwnerId = "analyst",
            Subject = "NVDA",
            Content = "Muss die Sicherung überstehen",
            Key = "kernaussage",
            CreatedBy = "analyst"
        }, default);

        // Close connections so VACUUM INTO meets a quiet file.
        ClawdDotNet.Core.Storage.SqliteStorage.ReleasePooledConnections(_instanceDir);
    }

    private string ZipPath => Path.Combine(_root, "sicherung.zip");
    private string RestoreDir => Path.Combine(_root, "wiederhergestellt");

    // ═══════════════════════════════════════════════════════════
    //  The round trip
    // ═══════════════════════════════════════════════════════════

    [Fact]
    public async Task The_full_round_trip_restores_everything()
    {
        await BuildInstanceAsync();

        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });

        var result = await _service.RestoreAsync(ZipPath, RestoreDir,
            new RestoreOptions { Passphrase = "geheim" });

        result.Written.ShouldNotBeEmpty();

        // Personality
        File.ReadAllText(Path.Combine(RestoreDir, "Agents", "Agent-Analyst", "Identity.md"))
            .ShouldContain("Der Analyst");
        File.ReadAllText(Path.Combine(RestoreDir, "Agents", "Agent-Analyst", "Soul.md"))
            .ShouldContain("Gründlich");

        // Work state
        File.ReadAllText(Path.Combine(RestoreDir, "Agents", "Agent-Analyst", "Workspace", "bericht.md"))
            .ShouldContain("Inhalt");

        // Database including memory
        File.Exists(Path.Combine(RestoreDir, "state.db")).ShouldBeTrue();

        var memory = new SqliteMemoryRepository(new SqliteStorage(Path.Combine(RestoreDir, "state.db")));
        var recalled = await memory.RecallAsync(
            new MemoryQuery { Scope = MemoryScope.Agent, OwnerId = "analyst" }, default);

        recalled.ShouldHaveSingleItem();
        recalled[0].Content.ShouldBe("Muss die Sicherung überstehen");
    }

    [Fact]
    public async Task Credentials_are_usable_again_after_restoring()
    {
        // The actual purpose of the passphrase: a backup is needed when the
        // machine is broken — DPAPI-protected values would then be unreadable.
        await BuildInstanceAsync();

        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });
        await _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Passphrase = "geheim" });

        var settings = JsonDocument.Parse(
            File.ReadAllText(Path.Combine(RestoreDir, "InstanceSettings.json")));

        var stored = settings.RootElement.GetProperty("openRouterApiKey").GetString();

        SecretProtector.IsProtected(stored).ShouldBeTrue("wieder mit DPAPI geschützt");
        SecretProtector.Unprotect(stored).ShouldBe(ApiKeyPlain);
    }

    /// <summary>
    /// The move to another machine — the migration path from #190.
    ///
    /// <para>The tests above back up and restore in the same run and
    /// use the same key in doing so. They therefore cannot show what matters
    /// when moving: that the values are readable even if the key of the
    /// source machine does <b>not</b> come along. Exactly that is the case Windows → Linux —
    /// the existing values are DPAPI-encrypted, and
    /// <c>SecretProtector.UnprotectV1</c> deliberately throws for them on Linux.</para>
    ///
    /// <para>Here the key therefore changes in the middle of the test. Without the change
    /// the test would be green without proving anything.</para>
    /// </summary>
    [Fact]
    public async Task The_move_to_another_machine_holds()
    {
        var targetMachine = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"));

        try
        {
            // Machine 1: values lie under the local key.
            await BuildInstanceAsync();
            var sourceKey = File.ReadAllBytes(SecretKeyStore.KeyFilePath);

            await _service.CreateAsync(_instanceDir, ZipPath,
                new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "umzug" });

            // Machine 2: different machine, different key. The key does not travel
            // along — it deliberately lies outside the instance directory.
            SecretKeyStore.UseDirectory(targetMachine);

            await _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Passphrase = "umzug" });

            File.ReadAllBytes(SecretKeyStore.KeyFilePath)
                .ShouldNotBe(sourceKey, "sonst wäre es derselbe Rechner");

            var settings = JsonDocument.Parse(
                File.ReadAllText(Path.Combine(RestoreDir, "InstanceSettings.json")));

            var stored = settings.RootElement.GetProperty("openRouterApiKey").GetString();
            SecretProtector.Unprotect(stored).ShouldBe(ApiKeyPlain);

            // The nested values too, not only the obvious one at the top.
            var agent = JsonDocument.Parse(File.ReadAllText(
                Path.Combine(RestoreDir, "Agents", "Agent-Analyst", "AgentSettings.json")));

            var password = agent.RootElement
                .GetProperty("tools").GetProperty("Mail").GetProperty("password").GetString();

            SecretProtector.Unprotect(password).ShouldBe(MailPasswordPlain);
        }
        finally
        {
            // A directory of its own for the next test of the collection: that of the
            // source machine is no longer available here, and the real
            // user key must certainly not be it.
            SecretKeyStore.UseDirectory(
                Path.Combine(Path.GetTempPath(), "clawd-tests", "keys", Guid.NewGuid().ToString("N")));
            try { Directory.Delete(targetMachine, recursive: true); } catch { /* secondary */ }
        }
    }

    [Fact]
    public async Task Without_the_passphrase_the_move_is_lost()
    {
        // The flip side, and the reason why the passphrase has to be offered when backing
        // up: without it a backup on the target machine is worthless as far as the
        // credentials are concerned. That is not an error, but the price for the
        // key not travelling along — but it has to be noticed instead of happening silently.
        var targetMachine = Path.Combine(Path.GetTempPath(), "clawd-tests", Guid.NewGuid().ToString("N"));

        try
        {
            await BuildInstanceAsync();

            await _service.CreateAsync(_instanceDir, ZipPath,
                new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "umzug" });

            SecretKeyStore.UseDirectory(targetMachine);

            await Should.ThrowAsync<Exception>(
                () => _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Passphrase = "falsch" }));
        }
        finally
        {
            SecretKeyStore.UseDirectory(
                Path.Combine(Path.GetTempPath(), "clawd-tests", "keys", Guid.NewGuid().ToString("N")));
            try { Directory.Delete(targetMachine, recursive: true); } catch { /* secondary */ }
        }
    }

    [Fact]
    public async Task Nested_credentials_in_tool_configurations_come_back_too()
    {
        await BuildInstanceAsync();

        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });
        await _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Passphrase = "geheim" });

        var agent = JsonDocument.Parse(File.ReadAllText(
            Path.Combine(RestoreDir, "Agents", "Agent-Analyst", "AgentSettings.json")));

        var password = agent.RootElement
            .GetProperty("tools").GetProperty("Mail").GetProperty("password").GetString();

        SecretProtector.Unprotect(password).ShouldBe(MailPasswordPlain);
    }

    [Fact]
    public async Task Unknown_fields_survive_the_round_trip()
    {
        // A backup must not throw anything away just because a program version does not know
        // a field.
        await BuildInstanceAsync();

        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });
        await _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Passphrase = "geheim" });

        File.ReadAllText(Path.Combine(RestoreDir, "InstanceSettings.json"))
            .ShouldContain("einUnbekanntesFeld");
    }

    // ═══════════════════════════════════════════════════════════
    //  Leaving out credentials
    // ═══════════════════════════════════════════════════════════

    [Fact]
    public async Task Without_credentials_the_archive_contains_no_secrets()
    {
        await BuildInstanceAsync();

        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Exclude });

        // Search the entire archive in plain text.
        var content = ReadWholeArchive(ZipPath);

        content.ShouldNotContain(ApiKeyPlain);
        content.ShouldNotContain(MailPasswordPlain);
    }

    [Fact]
    public async Task A_backup_with_passphrase_does_not_show_the_secrets_in_plaintext()
    {
        await BuildInstanceAsync();

        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });

        var content = ReadWholeArchive(ZipPath);

        content.ShouldNotContain(ApiKeyPlain);
        content.ShouldNotContain(MailPasswordPlain);
        content.ShouldContain("pbe:v1:");
    }

    // ═══════════════════════════════════════════════════════════
    //  Manifest and error cases
    // ═══════════════════════════════════════════════════════════

    [Fact]
    public async Task The_manifest_describes_the_backup()
    {
        await BuildInstanceAsync();

        var result = await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });

        var manifest = await _service.InspectAsync(ZipPath);

        manifest.InstanceId.ShouldBe("test-01");
        manifest.InstanceName.ShouldBe("Testinstanz");
        manifest.HasSecrets.ShouldBeTrue();
        manifest.SecretCount.ShouldBe(2, "API-Schlüssel und Mail-Passwort");
        manifest.Files.ShouldNotBeEmpty();
        result.SizeBytes.ShouldBeGreaterThan(0);
    }

    [Fact]
    public async Task Without_passphrase_nothing_is_restored()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "geheim" });

        var ex = await Should.ThrowAsync<BackupException>(
            () => _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions()));

        ex.Message.ShouldContain("Passphrase");
    }

    [Fact]
    public async Task A_wrong_passphrase_is_recognised()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath,
            new BackupOptions { Secrets = SecretMode.Passphrase, Passphrase = "richtig" });

        await Should.ThrowAsync<SecretProtectionException>(
            () => _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Passphrase = "falsch" }));
    }

    [Fact]
    public async Task A_modified_archive_is_noticed()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions());

        // Change a file in the archive afterwards.
        using (var archive = ZipFile.Open(ZipPath, ZipArchiveMode.Update))
        {
            var entry = archive.Entries.First(e => e.FullName.EndsWith("Identity.md"));
            using var stream = entry.Open();
            stream.SetLength(0);
            using var writer = new StreamWriter(stream);
            writer.Write("manipuliert");
        }

        var ex = await Should.ThrowAsync<BackupException>(
            () => _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions()));

        ex.Message.ShouldContain("Prüfsumme");
    }

    [Fact]
    public async Task An_archive_without_manifest_is_rejected()
    {
        var foreign = Path.Combine(_root, "fremd.zip");
        var source = Path.Combine(_root, "quelle");
        Directory.CreateDirectory(source);
        File.WriteAllText(Path.Combine(source, "irgendwas.txt"), "Inhalt");
        ZipFile.CreateFromDirectory(source, foreign);

        var ex = await Should.ThrowAsync<BackupException>(() => _service.InspectAsync(foreign));

        ex.Message.ShouldContain("Manifest");
    }

    // ═══════════════════════════════════════════════════════════
    //  Preview and overwriting
    // ═══════════════════════════════════════════════════════════

    [Fact]
    public async Task The_preview_writes_nothing()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions());

        var result = await _service.RestoreAsync(ZipPath, RestoreDir,
            new RestoreOptions { DryRun = true });

        result.Written.ShouldBeEmpty();
        Directory.Exists(RestoreDir).ShouldBeFalse();
    }

    [Fact]
    public async Task Existing_files_are_not_overwritten_without_consent()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions());

        Directory.CreateDirectory(RestoreDir);
        var present = Path.Combine(RestoreDir, "InstanceSettings.json");
        AtomicFile.WriteAllText(present, """{"wichtig":"nicht verlieren"}""");

        var result = await _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions());

        result.HasConflicts.ShouldBeTrue();
        result.Skipped.ShouldContain("InstanceSettings.json");
        File.ReadAllText(present).ShouldContain("nicht verlieren");
    }

    [Fact]
    public async Task With_consent_it_is_overwritten()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions());

        Directory.CreateDirectory(RestoreDir);
        var present = Path.Combine(RestoreDir, "InstanceSettings.json");
        AtomicFile.WriteAllText(present, """{"alt":true}""");

        await _service.RestoreAsync(ZipPath, RestoreDir, new RestoreOptions { Overwrite = true });

        File.ReadAllText(present).ShouldContain("Testinstanz");
    }

    // ═══════════════════════════════════════════════════════════
    //  Scope
    // ═══════════════════════════════════════════════════════════

    [Fact]
    public async Task Logs_are_left_out_by_default()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions());

        var manifest = await _service.InspectAsync(ZipPath);

        manifest.Files.ShouldNotContain(f => f.Path.Contains("Logs/"));
    }

    [Fact]
    public async Task Logs_can_be_included()
    {
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions { IncludeLogs = true });

        var manifest = await _service.InspectAsync(ZipPath);

        manifest.Files.ShouldContain(f => f.Path.Contains("Logs/"));
    }

    [Fact]
    public async Task The_WAL_side_files_do_not_end_up_in_the_archive()
    {
        // VACUUM INTO already incorporates them — backing them up too would be misleading.
        await BuildInstanceAsync();
        await _service.CreateAsync(_instanceDir, ZipPath, new BackupOptions());

        var manifest = await _service.InspectAsync(ZipPath);

        manifest.Files.ShouldNotContain(f => f.Path.EndsWith("-wal") || f.Path.EndsWith("-shm"));
        manifest.Files.ShouldContain(f => f.Path == "state.db");
    }

    [Fact]
    public async Task A_missing_instance_directory_is_reported()
    {
        var ex = await Should.ThrowAsync<BackupException>(
            () => _service.CreateAsync(Path.Combine(_root, "gibtsnicht"), ZipPath, new BackupOptions()));

        ex.Message.ShouldContain("nicht gefunden");
    }

    [Fact]
    public async Task Passphrase_mode_without_passphrase_is_rejected()
    {
        await BuildInstanceAsync();

        await Should.ThrowAsync<BackupException>(() => _service.CreateAsync(
            _instanceDir, ZipPath, new BackupOptions { Secrets = SecretMode.Passphrase }));
    }

    // ─── Helpers ───

    private static string ReadWholeArchive(string zipPath)
    {
        using var archive = ZipFile.OpenRead(zipPath);
        var sb = new System.Text.StringBuilder();

        foreach (var entry in archive.Entries)
        {
            using var stream = entry.Open();
            using var buffer = new MemoryStream();
            stream.CopyTo(buffer);
            sb.Append(System.Text.Encoding.UTF8.GetString(buffer.ToArray()));
        }

        return sb.ToString();
    }
}