ClawdDotNet/tests/ClawdDotNet.Manager.Tests/InstanceSupervisorTests.cs

using ClawdDotNet.Core.Instances;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;

namespace ClawdDotNet.Manager.Tests;

/// <summary>
/// The supervision of an instance (#528, stage 3).
///
/// <para><b>The occasion.</b> On 2026-09-08 an instance hung and was ended by
/// Windows. Not a single line about this event is in the instance logs,
/// no bug tracker entry, no watchdog alarm — because reporting path and watchdog lived in exactly the
/// process they should have reported on (#529). The acceptance condition for
/// this stage is therefore: <b>kill an instance, and the manager notices it and
/// reports it.</b></para>
///
/// <para>The harder case stands below: a process that is <em>alive</em> and no longer
/// works. No process list sees it — only the socket.</para>
/// </summary>
public sealed class InstanceSupervisorTests : IDisposable
{
    private readonly string _dir = Path.Combine(
        Path.GetTempPath(), "clawd-tests", "sup-" + Guid.NewGuid().ToString("N")[..8]);

    private readonly FakeLauncher _launcher = new();
    private readonly FakeChannel _channel = new();
    private readonly List<SupervisionEvent> _events = [];

    private static readonly InstanceIdentity Test = new("i-1", "Test", "0.1.2");

    /// <summary>
    /// Short deadlines: the test is supposed to check supervision, not patience.
    ///
    /// <para>Just above the probe interval of 250 ms, so the waiting loops manage one
    /// pass each — this is about the decision at the end of the deadline, not about
    /// how often there was asking in between.</para>
    /// </summary>
    private static readonly SupervisionOptions Options = new(
        ProbeInterval: TimeSpan.FromMilliseconds(50),
        StartTimeout: TimeSpan.FromMilliseconds(600),
        SilentChecksBeforeAlarm: 3,
        StopGrace: TimeSpan.FromMilliseconds(300),
        SilentKillGrace: TimeSpan.FromMilliseconds(300),
        MaxDrainWait: TimeSpan.FromSeconds(2));

    public InstanceSupervisorTests()
    {
        Directory.CreateDirectory(_dir);

        // The socket only answers if a process is really running - exactly as in
        // operation. An answer stored in advance would otherwise mean "already running", and
        // supervision would take over instead of starting.
        _channel.Launcher = _launcher;
    }

    public void Dispose()
    {
        try { Directory.Delete(_dir, recursive: true); }
        catch { /* A leftover test folder does not turn a run red. */ }
    }

    private InstanceSupervisor Supervisor(SupervisionOptions? options = null)
    {
        var supervisor = new InstanceSupervisor(
            _dir, _launcher, _channel, NullLogger.Instance, options ?? Options);

        supervisor.Observed += _events.Add;
        return supervisor;
    }

    private IEnumerable<InstanceEventKind> Arten => _events.Select(e => e.Kind);

    // ─── Starting ───

    [Fact]
    public async Task A_start_only_counts_when_the_instance_answers()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        (await supervisor.StartAsync()).ShouldBeTrue();

        supervisor.Condition.ShouldBe(InstanceCondition.Running);
        supervisor.InstanceName.ShouldBe("Test");
        Arten.ShouldBe([InstanceEventKind.Started]);

        var state = InstanceStateStore.Read(_dir).ShouldNotBeNull();
        state.InstanceId.ShouldBe("i-1");
        state.ProcessId.ShouldBe(_launcher.Last!.Id);
        state.ManagerProcessId.ShouldBe(Environment.ProcessId);
    }

    /// <summary>
    /// "The process is running" is not the same as "the instance is running".
    ///
    /// <para>Between the two lie database, agents and Kestrel. Whoever only waits for
    /// <c>Process.Start</c> considers an instance started that is about to fail on a
    /// broken configuration.</para>
    /// </summary>
    [Fact]
    public async Task A_process_that_never_reports_does_not_count_as_started()
    {
        _channel.Answer = null; // runs, but says nothing

        await using var supervisor = Supervisor();
        (await supervisor.StartAsync()).ShouldBeFalse();

        supervisor.Condition.ShouldBe(InstanceCondition.Stopped);
        Arten.ShouldBe([InstanceEventKind.StartFailed]);

        // Do not leave it standing: a process that runs and stays silent keeps the socket
        // occupied and thus blocks the next attempt.
        _launcher.Last!.WasKilled.ShouldBeTrue();
        InstanceStateStore.Read(_dir).ShouldBeNull();
    }

    [Fact]
    public async Task A_process_that_does_not_start_at_all_reports_it()
    {
        _launcher.FailStartWith = new InvalidOperationException("Programmdatei fehlt");

        await using var supervisor = Supervisor();
        (await supervisor.StartAsync()).ShouldBeFalse();

        Arten.ShouldBe([InstanceEventKind.StartFailed]);
        _events[0].Message.ShouldContain("Programmdatei fehlt");
    }

    // ─── The case this is about ───

    [Fact]
    public async Task A_killed_process_is_noticed_and_reported()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        // Someone kills the instance. Nobody asked for it.
        _launcher.Last!.Exit(code: 137);
        _channel.Answer = null;

        await supervisor.CheckAsync();

        supervisor.Condition.ShouldBe(InstanceCondition.Crashed);

        var crash = _events.Single(e => e.Kind == InstanceEventKind.Crashed);
        crash.ExitCode.ShouldBe(137);
        crash.InstanceName.ShouldBe("Test");
        crash.WorthReporting.ShouldBeTrue("Das ist die Meldung, die es 2026-09-08 nicht gab.");

        // And the state file no longer points to a process that does not exist.
        InstanceStateStore.Read(_dir).ShouldBeNull();
    }

    /// <summary>
    /// The harder case: the process lives and no longer works.
    ///
    /// <para>Exactly that was the state on 2026-09-08 — the application stood flawlessly in
    /// the process list until Windows ended it as "not responding". A
    /// supervision that only counts processes sees nothing of it.</para>
    /// </summary>
    [Fact]
    public async Task A_living_but_silent_process_is_noticed()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        _channel.Answer = null; // hangs - the process lives on

        // Not at the first hiccup: that would be everyday life, and an alarm at every
        // hiccup is after a week the alarm nobody reads any more.
        await supervisor.CheckAsync();
        await supervisor.CheckAsync();
        Arten.ShouldNotContain(InstanceEventKind.Unresponsive);
        supervisor.Condition.ShouldBe(InstanceCondition.Running);

        await supervisor.CheckAsync();

        supervisor.Condition.ShouldBe(InstanceCondition.Silent);
        var alarm = _events.Single(e => e.Kind == InstanceEventKind.Unresponsive);
        alarm.WorthReporting.ShouldBeTrue();
        _launcher.Last!.HasExited.ShouldBeFalse("Der Prozess laeuft ja - das ist der Punkt.");
    }

    [Fact]
    public async Task A_silent_process_is_reported_once_and_not_on_every_pass()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        _channel.Answer = null;

        for (var i = 0; i < 20; i++)
            await supervisor.CheckAsync();

        _events.Count(e => e.Kind == InstanceEventKind.Unresponsive).ShouldBe(1);
    }

    [Fact]
    public async Task Whoever_answers_again_is_listed_as_running_again()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        _channel.Answer = null;
        for (var i = 0; i < 3; i++) await supervisor.CheckAsync();
        supervisor.Condition.ShouldBe(InstanceCondition.Silent);

        _channel.Answer = Test;
        await supervisor.CheckAsync();

        supervisor.Condition.ShouldBe(InstanceCondition.Running);
        Arten.ShouldContain(InstanceEventKind.Recovered);
    }

    // ─── Stopping ───

    [Fact]
    public async Task A_requested_exit_is_not_a_crash()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        // The instance ends upon the request - as in operation.
        _launcher.Last!.Exit(code: 0);
        _channel.Answer = null;

        (await supervisor.StopAsync()).ShouldBeTrue();

        _channel.StopRequests.ShouldBe(1, "Erst bitten, nicht schiessen.");
        _launcher.Last.WasKilled.ShouldBeFalse();

        supervisor.Condition.ShouldBe(InstanceCondition.Stopped);
        Arten.ShouldNotContain(InstanceEventKind.Crashed);
        _events.Last().Kind.ShouldBe(InstanceEventKind.StoppedOrderly);
        _events.Last().WorthReporting.ShouldBeFalse();
    }

    [Fact]
    public async Task Whoever_does_not_listen_is_stopped_after_the_deadline()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        // Keeps answering and does not end - the request goes unheard.
        _channel.StopRequestArrives = false;

        (await supervisor.StopAsync()).ShouldBeTrue();

        _channel.StopRequests.ShouldBe(1);
        _launcher.Last!.WasKilled.ShouldBeTrue();
        supervisor.Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// Silence is no proof of death as long as a process handle says the opposite.
    ///
    /// <para><b>Here lay the most dangerous fallacy.</b> The waiting loop broke off
    /// as soon as the socket no longer answered, reported "ended" and returned <c>true</c>
    /// — without touching the process. With a silent instance, however, the socket
    /// is quiet from the start: the call reported success immediately, the process kept
    /// running, and the restart put a second one on the same <c>state.db</c>. Exactly
    /// the damage this project is built against.</para>
    /// </summary>
    [Fact]
    public async Task A_silent_but_living_process_does_not_count_as_ended()
    {
        _channel.Answer = Test;

        await using var supervisor = Supervisor();
        await supervisor.StartAsync();

        var process = _launcher.Last.ShouldNotBeNull();

        // It hangs: the socket is silent, the process lives.
        _channel.GoSilent(_dir);

        (await supervisor.StopAsync()).ShouldBeTrue();

        process.WasKilled.ShouldBeTrue(
            "Wer schweigt und lebt, muss beendet werden - nicht fuer beendet erklaert.");

        process.HasExited.ShouldBeTrue();
        supervisor.Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// Whoever is cleaning up is not killed — the deadline is extended.
    ///
    /// <para>The instance gives itself a deadline for running out. If the manager's were
    /// shorter, it would shoot into the middle of the cleanup it has just asked
    /// for.</para>
    /// </summary>
    [Fact]
    public async Task Whoever_cleans_up_gets_the_deadline_extended()
    {
        _channel.Answer = Test;

        // #722: Supervision asks every 250 ms and only extends the deadline on "cleaning up".
        // With a 300 ms deadline the next question came too late under parallel load, and the test saw
        // a kill where none belongs. A one-second deadline leaves the tick room; the
        // instance still cleans up considerably longer than the deadline alone allows.
        await using var supervisor = Supervisor(Options with { StopGrace = TimeSpan.FromSeconds(1), MaxDrainWait = TimeSpan.FromSeconds(20) });
        await supervisor.StartAsync();

        var process = _launcher.Last.ShouldNotBeNull();

        // It answers and reports: I am cleaning up. Longer than StopGrace (1 s), but
        // shorter than MaxDrainWait.
        _channel.Answer = Test with { Phase = ShutdownPhase.Draining, OpenWork = 1 };

        _channel.OnStopRequest = _ => Task.Run(async () =>
        {
            await Task.Delay(2500);
            process.Exit(0);
            _channel.Answer = null;
        });

        (await supervisor.StopAsync()).ShouldBeTrue();

        process.WasKilled.ShouldBeFalse("Wer aufraeumt, wird nicht abgeschossen.");
        supervisor.Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// An instance that the manager did not start itself and that does not listen, it
    /// cannot end either — and it has to say so instead of reporting success.
    ///
    /// <para>The case arises after a manager restart when the process ID can no longer
    /// be grabbed (see <see cref="Eine_wiederverwendete_Prozesskennung_wird_nicht_fuer_die_Instanz_gehalten"/>).</para>
    /// </summary>
    [Fact]
    public async Task What_cannot_be_grasped_is_not_reported_as_ended()
    {
        _channel.Answer = Test;
        _channel.StopRequestArrives = false;

        // It was already running before this manager came - and without a state file its
        // process cannot be grabbed by it.
        _launcher.StartedElsewhere(_dir, 4711);

        await using var supervisor = Supervisor();
        await supervisor.ReattachAsync();          // taken over, but without a process
        supervisor.ProcessId.ShouldBeNull();

        (await supervisor.StopAsync()).ShouldBeFalse();

        supervisor.Condition.ShouldBe(InstanceCondition.Running);
        Arten.ShouldNotContain(InstanceEventKind.StoppedOrderly);
    }

    // ─── Rediscovering (E-M6) ───

    [Fact]
    public async Task A_running_instance_is_adopted_after_a_manager_restart()
    {
        _channel.Answer = Test;

        // The first manager starts it.
        await using (var firstOne = Supervisor())
            await firstOne.StartAsync();

        _events.Clear();

        // Second manager, the same machine, the same running instance.
        await using var secondOne = Supervisor();
        (await secondOne.ReattachAsync()).ShouldBeTrue();

        secondOne.Condition.ShouldBe(InstanceCondition.Running);
        secondOne.InstanceName.ShouldBe("Test");
        secondOne.ProcessId.ShouldBe(_launcher.Last!.Id);

        // Taking over is not starting - there is nothing to report.
        _events.ShouldBeEmpty();
        _launcher.Starts.ShouldBe(1);
    }

    /// <summary>
    /// A reused process ID must not lead to taking over a foreign
    /// program.
    ///
    /// <para>After a restart of the machine, 4711 belongs with some probability
    /// to something entirely different. Supervising it then — and ending it when stopping —
    /// would be considerably worse than not finding the instance at all.</para>
    /// </summary>
    [Fact]
    public async Task Eine_wiederverwendete_Prozesskennung_wird_nicht_fuer_die_Instanz_gehalten()
    {
        _channel.Answer = Test;

        await using (var firstOne = Supervisor())
            await firstOne.StartAsync();

        var oldOnes = _launcher.Last!;

        // The same ID, started much later: a different process.
        _launcher.Running[oldOnes.Id] = new FakeProcess(oldOnes.Id, oldOnes.StartedAt.AddHours(3));

        await using var secondOne = Supervisor();
        (await secondOne.ReattachAsync()).ShouldBeTrue("Der Socket antwortet ja.");

        secondOne.ProcessId.ShouldBeNull(
            "Der fremde Prozess darf nicht als der eigene uebernommen werden.");
    }

    [Fact]
    public async Task A_state_file_without_a_running_instance_is_cleared_away()
    {
        _channel.Answer = Test;

        await using (var firstOne = Supervisor())
            await firstOne.StartAsync();

        InstanceStateStore.Read(_dir).ShouldNotBeNull();

        // The manager was gone, the instance is gone by now too.
        _channel.Answer = null;

        await using var secondOne = Supervisor();
        (await secondOne.ReattachAsync()).ShouldBeFalse();

        secondOne.Condition.ShouldBe(InstanceCondition.Stopped);
        InstanceStateStore.Read(_dir).ShouldBeNull(
            "Eine Datei, die auf niemanden zeigt, laesst es aussehen, als liefe etwas.");
    }

    [Fact]
    public async Task Without_state_file_an_answering_instance_is_still_adopted()
    {
        // The case after a crash of the manager before it could write - or
        // after someone deleted the file. The socket answers, so it is running.
        _channel.Answer = Test;
        _launcher.StartedElsewhere(_dir, 4711);

        await using var supervisor = Supervisor();
        (await supervisor.ReattachAsync()).ShouldBeTrue();

        supervisor.Condition.ShouldBe(InstanceCondition.Running);
        supervisor.ProcessId.ShouldBeNull("Es gibt keinen Prozess, den dieser Manager kennt.");
        InstanceStateStore.Read(_dir).ShouldNotBeNull();
    }

    // ─── What is reported and what is not ───

    /// <summary>
    /// The manager reports what is relevant from its point of view — and only that (concept,
    /// section 4). What concerns the instance stays in the instance and goes out with its
    /// own sub-token.
    /// </summary>
    [Theory]
    [InlineData(InstanceEventKind.Crashed, true)]
    [InlineData(InstanceEventKind.Unresponsive, true)]
    [InlineData(InstanceEventKind.StartFailed, true)]
    [InlineData(InstanceEventKind.Started, false)]
    [InlineData(InstanceEventKind.StoppedOrderly, false)]
    [InlineData(InstanceEventKind.Recovered, false)]
    public void Only_what_nobody_ordered_is_reported(InstanceEventKind kind, bool expected)
    {
        new SupervisionEvent(kind, _dir, "Test", 1, null, "", DateTimeOffset.UtcNow)
            .WorthReporting.ShouldBe(expected);
    }
}