ClawdDotNet/tests/ClawdDotNet.Manager.Tests/AutostartTests.cs

using System.Text.Json;
using ClawdDotNet.Core.Config;
using ClawdDotNet.Core.Instances;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;

namespace ClawdDotNet.Manager.Tests;

/// <summary>
/// "Autostart by manager" — the switch that keeps an instance running (#528).
///
/// <para><b>Two things in one switch, because it is the same promise:</b> the manager
/// starts the instance when coming up, and it starts it again after a crash. Without
/// the switch it touches nothing — a manager that boots everything it finds after a restart of the machine
/// also starts what someone stopped on purpose.</para>
///
/// <para><b>And the circuit breaker is the actual content.</b> Booting again
/// is the easy half; the hard one is recognizing when to stop.</para>
/// </summary>
public sealed class AutostartTests : IAsyncDisposable
{
    private readonly string _root = Path.Combine(
        Path.GetTempPath(), "clawd-tests", "auto-" + Guid.NewGuid().ToString("N")[..8]);

    private readonly FakeLauncher _launcher = new();
    private readonly FakeChannel _channel = new();
    private readonly FakeTokenSource _tokens = new();
    private readonly InstanceDirectoryManager _directories;

    private static readonly SupervisionOptions Options = new(
        ProbeInterval: TimeSpan.FromMilliseconds(50),
        StartTimeout: TimeSpan.FromMilliseconds(600),
        SilentChecksBeforeAlarm: 3,
        StopGrace: TimeSpan.FromMilliseconds(300),
        SilentKillGrace: TimeSpan.FromMilliseconds(300),
        MaxDrainWait: TimeSpan.FromSeconds(2));

    public AutostartTests()
    {
        Directory.CreateDirectory(_root);
        _directories = new InstanceDirectoryManager(Path.Combine(_root, "Instances"));

        // The socket only answers if a process is really running - otherwise an
        // answer stored in advance would mean "already running", and the manager would take over
        // instead of starting.
        _channel.Launcher = _launcher;
    }

    public async ValueTask DisposeAsync()
    {
        await Task.CompletedTask;
        try { Directory.Delete(_root, recursive: true); }
        catch { /* A leftover test folder does not turn a run red. */ }
    }

    private ManagerCore Manager(RestartPolicy? policy = null)
    {
        var register = new InstanceRegistry(
            _directories, _launcher, _channel, NullLoggerFactory.Instance, Options);

        return new ManagerCore(
            new ManagerSettingsStore(_root), register, _tokens, NullLoggerFactory.Instance,
            restartPolicy: policy);
    }

    /// <summary>Creates an instance and sets the switch in its instance file.</summary>
    private string Instance(string name, bool autostart)
    {
        var path = _directories.CreateInstance(name);
        var datei = Path.Combine(path, "InstanceSettings.json");

        var config = JsonSerializer.Deserialize<InstanceConfig>(File.ReadAllText(datei))!;
        config.AutostartByManager = autostart;
        File.WriteAllText(datei, JsonSerializer.Serialize(config));

        _channel.AnswerFor[Path.GetFullPath(path)] = new InstanceIdentity("i-" + name, name, "0.1.2");

        return path;
    }

    private static SupervisionEvent Crash(string path, string name) => new(
        InstanceEventKind.Crashed, Path.GetFullPath(path), name, 4711, 137,
        "unerwartet beendet", DateTimeOffset.UtcNow);

    // ─── When coming up ───

    [Fact]
    public async Task An_instance_with_autostart_is_started_on_startup()
    {
        Instance("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        _launcher.Starts.ShouldBe(1);
        manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Running);
    }

    [Fact]
    public async Task Without_autostart_it_stays_off()
    {
        Instance("Alpha", autostart: false);

        await using var manager = Manager();
        await manager.StartAsync();

        _launcher.Starts.ShouldBe(0);
        manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// What is already running is not started again: a second process on the same
    /// <c>state.db</c> would be the opposite of helpful.
    /// </summary>
    [Fact]
    public async Task What_already_runs_is_not_started_again()
    {
        var path = Instance("Alpha", autostart: true);

        await using (var firstOne = Manager())
            await firstOne.StartAsync();

        _launcher.Starts.ShouldBe(1);

        await using var secondOne = Manager();
        await secondOne.StartAsync();

        _launcher.Starts.ShouldBe(1, "Uebernommen, nicht neu gestartet.");
        secondOne.Instances.Instances.Single().ProcessId.ShouldBe(_launcher.Last!.Id);
    }

    [Fact]
    public async Task The_switch_is_in_the_list()
    {
        Instance("Mit", autostart: true);
        Instance("Ohne", autostart: false);

        await using var manager = Manager();
        await manager.StartAsync();

        manager.Instances.Instances.Single(i => i.Name == "Mit").AutostartByManager.ShouldBeTrue();
        manager.Instances.Instances.Single(i => i.Name == "Ohne").AutostartByManager.ShouldBeFalse();
    }

    // ─── After a crash ───

    [Fact]
    public async Task After_a_crash_it_is_restarted()
    {
        var path = Instance("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        var before = _launcher.Starts;

        // The process is gone; supervision has reported it.
        _launcher.Last!.Exit(137);
        await manager.RestartAfterCrashAsync(Crash(path, "Alpha"));

        _launcher.Starts.ShouldBe(before + 1);
    }

    [Fact]
    public async Task Without_autostart_nothing_is_done_after_a_crash()
    {
        var path = Instance("Alpha", autostart: false);

        await using var manager = Manager();
        await manager.StartAsync();
        await manager.StartInstanceAsync(path);

        _launcher.Last!.Exit(137);
        await manager.RestartAfterCrashAsync(Crash(path, "Alpha"));

        _launcher.Starts.ShouldBe(1, "Nur der Start von Hand, kein Neustart.");
    }

    /// <summary>
    /// A silent instance is ended and restarted — but only after the
    /// old process is provably gone.
    ///
    /// <para><b>The case is really produced here and not claimed.</b> The
    /// process lives on, only the socket is silent — exactly the situation from #529. A test
    /// that instead builds the event by hand and calls the restart method checks
    /// the half that works anyway, and leaves out the dangerous one: that before
    /// the start nobody sits on the database any more.</para>
    /// </summary>
    [Fact]
    public async Task A_silent_instance_is_first_stopped_and_then_restarted()
    {
        var path = Instance("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        var old = _launcher.Last.ShouldNotBeNull();
        _launcher.Starts.ShouldBe(1);

        // From now on it hangs: the process lives, the socket no longer answers.
        _channel.GoSilent(path);

        var supervisor = manager.Instances.SupervisorFor(path).ShouldNotBeNull();

        // Three passes until the alarm - nothing is done before that.
        for (var i = 0; i < 3; i++) await supervisor.CheckAsync();

        supervisor.Condition.ShouldBe(InstanceCondition.Silent);
        old.WasKilled.ShouldBeFalse("Vor Ablauf der Frist wird nicht abgeschossen.");

        // The deadline expires (SilentKillGrace: 300ms).
        await Task.Delay(400);
        await supervisor.CheckAsync();

        old.WasKilled.ShouldBeTrue("Nach der Frist wird der haengende Prozess beendet.");

        // The crash path restarts - and the new process is a different one.
        await WaitFor(() => _launcher.Starts >= 2);

        _launcher.Last.ShouldNotBeNull().Id.ShouldNotBe(old.Id);
        old.HasExited.ShouldBeTrue("Der alte Prozess darf beim Start nicht mehr leben.");
    }

    /// <summary>
    /// An instance that is cleaning up is not killed — even if it takes a while.
    ///
    /// <para>It does answer; it is doing exactly what it was asked to do. The difference
    /// between "hangs" and "cleans up" is the whole reason why the pulse says more than
    /// "someone answered".</para>
    /// </summary>
    [Fact]
    public async Task Whoever_cleans_up_does_not_count_as_silent()
    {
        var path = Instance("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        var process = _launcher.Last.ShouldNotBeNull();

        _channel.AnswerFor[Path.GetFullPath(path)] = new InstanceIdentity(
            "i-Alpha", "Alpha", "0.1.2",
            ShutdownPhase.Draining, OpenWork: 2, OpenSummary: "Agentenläufe: 2");

        var supervisor = manager.Instances.SupervisorFor(path).ShouldNotBeNull();

        for (var i = 0; i < 5; i++) await supervisor.CheckAsync();

        supervisor.Condition.ShouldBe(InstanceCondition.Running);
        process.WasKilled.ShouldBeFalse();
        _launcher.Starts.ShouldBe(1);
    }

    private static async Task WaitFor(Func<bool> condition)
    {
        for (var i = 0; i < 100 && !condition(); i++)
            await Task.Delay(20);
    }

    /// <summary>
    /// After the limit the manager pauses.
    ///
    /// <para>What fails because of a broken configuration fails the same way on the tenth attempt
    /// — then instead of a dead instance you have a dead instance, a full
    /// log and a bug tracker full of repetitions.</para>
    /// </summary>
    [Fact]
    public async Task After_three_restarts_in_the_time_window_the_manager_pauses()
    {
        var path = Instance("Wackelig", autostart: true);

        await using var manager = Manager(new RestartPolicy(MaxRestarts: 3, Window: TimeSpan.FromMinutes(10)));
        await manager.StartAsync();

        var afterStart = _launcher.Starts;

        for (var i = 0; i < 6; i++)
        {
            _launcher.Last!.Exit(137);
            await manager.RestartAfterCrashAsync(Crash(path, "Wackelig"));
        }

        (_launcher.Starts - afterStart).ShouldBe(3, "Drei Versuche, dann Schluss.");
    }

    /// <summary>
    /// Whoever starts by hand has either changed something or knows what they are doing. Charging them with
    /// the earlier attempts would mean punishing them for the crash.
    /// </summary>
    [Fact]
    public async Task A_manual_start_clears_the_counter()
    {
        var path = Instance("Wackelig", autostart: true);

        await using var manager = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10)));
        await manager.StartAsync();

        for (var i = 0; i < 5; i++)
        {
            _launcher.Last!.Exit(137);
            await manager.RestartAfterCrashAsync(Crash(path, "Wackelig"));
        }

        var exhausted = _launcher.Starts;

        // A new manager starts at zero - just like a start by hand.
        await using var fresh = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10)));
        await fresh.StartAsync();

        _launcher.Last!.Exit(137);
        await fresh.RestartAfterCrashAsync(Crash(path, "Wackelig"));

        _launcher.Starts.ShouldBeGreaterThan(exhausted);
    }
}

/// <summary>The circuit breaker on its own.</summary>
public sealed class RestartLedgerTests
{
    private sealed class Clock(DateTimeOffset jetzt) : TimeProvider
    {
        public DateTimeOffset Jetzt { get; set; } = jetzt;
        public override DateTimeOffset GetUtcNow() => Jetzt;
    }

    [Fact]
    public void Yes_up_to_the_limit_no_after_it()
    {
        var book = new RestartLedger(new RestartPolicy(3, TimeSpan.FromMinutes(10)));

        book.TryRecord("/a", out _).ShouldBeTrue();
        book.TryRecord("/a", out _).ShouldBeTrue();
        book.TryRecord("/a", out var third).ShouldBeTrue();
        third.ShouldBe(3);

        book.TryRecord("/a", out _).ShouldBeFalse();
    }

    [Fact]
    public void Every_instance_has_its_own_account()
    {
        var book = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10)));

        book.TryRecord("/a", out _).ShouldBeTrue();
        book.TryRecord("/a", out _).ShouldBeFalse();

        book.TryRecord("/b", out _).ShouldBeTrue("Die eine Instanz sperrt die andere nicht.");
    }

    /// <summary>
    /// The time window moves: an instance that crashes once a month and comes back
    /// up never runs into the limit — one that falls every minute does immediately.
    /// </summary>
    [Fact]
    public void Old_attempts_fall_out_of_the_window()
    {
        var clock = new Clock(DateTimeOffset.UnixEpoch);
        var book = new RestartLedger(new RestartPolicy(2, TimeSpan.FromMinutes(10)), clock);

        book.TryRecord("/a", out _).ShouldBeTrue();
        book.TryRecord("/a", out _).ShouldBeTrue();
        book.TryRecord("/a", out _).ShouldBeFalse();

        clock.Jetzt = clock.Jetzt.AddMinutes(11);

        book.TryRecord("/a", out var repeated).ShouldBeTrue();
        repeated.ShouldBe(1, "Die alten Versuche zaehlen nicht mehr mit.");
    }

    [Fact]
    public void Forgetting_resets()
    {
        var book = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10)));

        book.TryRecord("/a", out _).ShouldBeTrue();
        book.TryRecord("/a", out _).ShouldBeFalse();

        book.Forget("/a");

        book.TryRecord("/a", out _).ShouldBeTrue();
    }
}

/// <summary>
/// A start that meets an outdated note.
///
/// <para><b>The defect that only the autostart tests revealed.</b> Between two
/// supervision passes there are seconds, and in them an instance can break away. Whoever
/// asked to "start" in this gap got "already running" — a success with nothing
/// behind it. The next pass then reported a crash nobody expected any more.</para>
/// </summary>
public sealed class StaleStateTests : IAsyncDisposable
{
    private readonly string _root = Path.Combine(
        Path.GetTempPath(), "clawd-tests", "stale-" + Guid.NewGuid().ToString("N")[..8]);

    private readonly FakeLauncher _launcher = new();
    private readonly FakeChannel _channel = new();

    public StaleStateTests()
    {
        Directory.CreateDirectory(_root);
        _channel.Launcher = _launcher;
        _channel.Answer = new InstanceIdentity("i-1", "Alpha", "0.1.2");
    }

    public async ValueTask DisposeAsync()
    {
        await Task.CompletedTask;
        try { Directory.Delete(_root, recursive: true); }
        catch { /* A leftover test folder does not turn a run red. */ }
    }

    [Fact]
    public async Task A_start_verifies_instead_of_trusting_its_own_state()
    {
        var dir = Path.Combine(_root, "Instance-Alpha");
        Directory.CreateDirectory(dir);

        await using var supervisor = new InstanceSupervisor(
            dir, _launcher, _channel, NullLogger.Instance,
            new SupervisionOptions(
                TimeSpan.FromSeconds(30),          // deliberately sluggish: no pass in between
                TimeSpan.FromMilliseconds(600),
                3,
                TimeSpan.FromMilliseconds(300),
                TimeSpan.FromMilliseconds(300),
                TimeSpan.FromSeconds(2)));

        (await supervisor.StartAsync()).ShouldBeTrue();
        supervisor.Condition.ShouldBe(InstanceCondition.Running);

        // The instance breaks away. Supervision has not noticed yet - the next
        // pass would only come in thirty seconds.
        _launcher.Last!.Exit(137);
        supervisor.Condition.ShouldBe(InstanceCondition.Running, "Noch nichts bemerkt.");

        (await supervisor.StartAsync()).ShouldBeTrue();

        _launcher.Starts.ShouldBe(2, "Es wurde wirklich gestartet, nicht nur zugestimmt.");
        supervisor.Condition.ShouldBe(InstanceCondition.Running);
    }
}