ClawdDotNet/tests/ClawdDotNet.Manager.Tests/InstanceSupervisorTests.cs

using ClawdDotNet.Core.Instances;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;

namespace ClawdDotNet.Manager.Tests;

/// <summary>
/// The supervision of an instance (#528, stage 3).
///
/// <para><b>The occasion.</b> On 2026-09-08 an instance hung and was ended by
/// Windows. Not a single line about this event is in the instance logs,
/// no bug tracker entry, no watchdog alarm — because reporting path and watchdog lived in exactly the
/// process they should have reported on (#529). The acceptance condition for
/// this stage is therefore: <b>kill an instance, and the manager notices it and
/// reports it.</b></para>
///
/// <para>The harder case stands below: a process that is <em>alive</em> and no longer
/// works. No process list sees it — only the socket.</para>
/// </summary>
public sealed class InstanceSupervisorTests : IDisposable
{
    private readonly string _dir = Path.Combine(
        Path.GetTempPath(), "clawd-tests", "sup-" + Guid.NewGuid().ToString("N")[..8]);

    private readonly FakeLauncher _launcher = new();
    private readonly FakeChannel _kanal = new();
    private readonly List<SupervisionEvent> _ereignisse = [];

    private static readonly InstanceIdentity Test = new("i-1", "Test", "0.1.2");

    /// <summary>
    /// Short deadlines: the test is supposed to check supervision, not patience.
    ///
    /// <para>Just above the probe interval of 250 ms, so the waiting loops manage one
    /// pass each — this is about the decision at the end of the deadline, not about
    /// how often there was asking in between.</para>
    /// </summary>
    private static readonly SupervisionOptions Options = new(
        ProbeInterval: TimeSpan.FromMilliseconds(50),
        StartTimeout: TimeSpan.FromMilliseconds(600),
        SilentChecksBeforeAlarm: 3,
        StopGrace: TimeSpan.FromMilliseconds(300),
        SilentKillGrace: TimeSpan.FromMilliseconds(300),
        MaxDrainWait: TimeSpan.FromSeconds(2));

    public InstanceSupervisorTests()
    {
        Directory.CreateDirectory(_dir);

        // The socket only answers if a process is really running - exactly as in
        // operation. An answer stored in advance would otherwise mean "already running", and
        // supervision would take over instead of starting.
        _kanal.Launcher = _launcher;
    }

    public void Dispose()
    {
        try { Directory.Delete(_dir, recursive: true); }
        catch { /* A leftover test folder does not turn a run red. */ }
    }

    private InstanceSupervisor Aufsicht(SupervisionOptions? options = null)
    {
        var supervisor = new InstanceSupervisor(
            _dir, _launcher, _kanal, NullLogger.Instance, options ?? Options);

        supervisor.Observed += _ereignisse.Add;
        return supervisor;
    }

    private IEnumerable<InstanceEventKind> Arten => _ereignisse.Select(e => e.Kind);

    // ─── Starting ───

    [Fact]
    public async Task Ein_Start_gilt_erst_wenn_die_Instanz_antwortet()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        (await aufsicht.StartAsync()).ShouldBeTrue();

        aufsicht.Condition.ShouldBe(InstanceCondition.Running);
        aufsicht.InstanceName.ShouldBe("Test");
        Arten.ShouldBe([InstanceEventKind.Started]);

        var state = InstanceStateStore.Read(_dir).ShouldNotBeNull();
        state.InstanceId.ShouldBe("i-1");
        state.ProcessId.ShouldBe(_launcher.Last!.Id);
        state.ManagerProcessId.ShouldBe(Environment.ProcessId);
    }

    /// <summary>
    /// "The process is running" is not the same as "the instance is running".
    ///
    /// <para>Between the two lie database, agents and Kestrel. Whoever only waits for
    /// <c>Process.Start</c> considers an instance started that is about to fail on a
    /// broken configuration.</para>
    /// </summary>
    [Fact]
    public async Task Ein_Prozess_der_sich_nie_meldet_gilt_nicht_als_gestartet()
    {
        _kanal.Answer = null; // runs, but says nothing

        await using var aufsicht = Aufsicht();
        (await aufsicht.StartAsync()).ShouldBeFalse();

        aufsicht.Condition.ShouldBe(InstanceCondition.Stopped);
        Arten.ShouldBe([InstanceEventKind.StartFailed]);

        // Do not leave it standing: a process that runs and stays silent keeps the socket
        // occupied and thus blocks the next attempt.
        _launcher.Last!.WasKilled.ShouldBeTrue();
        InstanceStateStore.Read(_dir).ShouldBeNull();
    }

    [Fact]
    public async Task Ein_Prozess_der_gar_nicht_startet_meldet_das()
    {
        _launcher.FailStartWith = new InvalidOperationException("Programmdatei fehlt");

        await using var aufsicht = Aufsicht();
        (await aufsicht.StartAsync()).ShouldBeFalse();

        Arten.ShouldBe([InstanceEventKind.StartFailed]);
        _ereignisse[0].Message.ShouldContain("Programmdatei fehlt");
    }

    // ─── The case this is about ───

    [Fact]
    public async Task Ein_abgeschossener_Prozess_wird_bemerkt_und_gemeldet()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        // Someone kills the instance. Nobody asked for it.
        _launcher.Last!.Exit(code: 137);
        _kanal.Answer = null;

        await aufsicht.CheckAsync();

        aufsicht.Condition.ShouldBe(InstanceCondition.Crashed);

        var absturz = _ereignisse.Single(e => e.Kind == InstanceEventKind.Crashed);
        absturz.ExitCode.ShouldBe(137);
        absturz.InstanceName.ShouldBe("Test");
        absturz.WorthReporting.ShouldBeTrue("Das ist die Meldung, die es 2026-09-08 nicht gab.");

        // And the state file no longer points to a process that does not exist.
        InstanceStateStore.Read(_dir).ShouldBeNull();
    }

    /// <summary>
    /// The harder case: the process lives and no longer works.
    ///
    /// <para>Exactly that was the state on 2026-09-08 — the application stood flawlessly in
    /// the process list until Windows ended it as "not responding". A
    /// supervision that only counts processes sees nothing of it.</para>
    /// </summary>
    [Fact]
    public async Task Ein_lebender_aber_stummer_Prozess_faellt_auf()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        _kanal.Answer = null; // hangs - the process lives on

        // Not at the first hiccup: that would be everyday life, and an alarm at every
        // hiccup is after a week the alarm nobody reads any more.
        await aufsicht.CheckAsync();
        await aufsicht.CheckAsync();
        Arten.ShouldNotContain(InstanceEventKind.Unresponsive);
        aufsicht.Condition.ShouldBe(InstanceCondition.Running);

        await aufsicht.CheckAsync();

        aufsicht.Condition.ShouldBe(InstanceCondition.Silent);
        var alarm = _ereignisse.Single(e => e.Kind == InstanceEventKind.Unresponsive);
        alarm.WorthReporting.ShouldBeTrue();
        _launcher.Last!.HasExited.ShouldBeFalse("Der Prozess laeuft ja - das ist der Punkt.");
    }

    [Fact]
    public async Task Ein_stummer_Prozess_wird_einmal_gemeldet_und_nicht_bei_jedem_Durchgang()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        _kanal.Answer = null;

        for (var i = 0; i < 20; i++)
            await aufsicht.CheckAsync();

        _ereignisse.Count(e => e.Kind == InstanceEventKind.Unresponsive).ShouldBe(1);
    }

    [Fact]
    public async Task Wer_wieder_antwortet_wird_wieder_als_laufend_gefuehrt()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        _kanal.Answer = null;
        for (var i = 0; i < 3; i++) await aufsicht.CheckAsync();
        aufsicht.Condition.ShouldBe(InstanceCondition.Silent);

        _kanal.Answer = Test;
        await aufsicht.CheckAsync();

        aufsicht.Condition.ShouldBe(InstanceCondition.Running);
        Arten.ShouldContain(InstanceEventKind.Recovered);
    }

    // ─── Stopping ───

    [Fact]
    public async Task Ein_erbetenes_Ende_ist_kein_Absturz()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        // The instance ends upon the request - as in operation.
        _launcher.Last!.Exit(code: 0);
        _kanal.Answer = null;

        (await aufsicht.StopAsync()).ShouldBeTrue();

        _kanal.StopRequests.ShouldBe(1, "Erst bitten, nicht schiessen.");
        _launcher.Last.WasKilled.ShouldBeFalse();

        aufsicht.Condition.ShouldBe(InstanceCondition.Stopped);
        Arten.ShouldNotContain(InstanceEventKind.Crashed);
        _ereignisse.Last().Kind.ShouldBe(InstanceEventKind.StoppedOrderly);
        _ereignisse.Last().WorthReporting.ShouldBeFalse();
    }

    [Fact]
    public async Task Wer_nicht_hoert_wird_nach_der_Frist_beendet()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        // Keeps answering and does not end - the request goes unheard.
        _kanal.StopRequestArrives = false;

        (await aufsicht.StopAsync()).ShouldBeTrue();

        _kanal.StopRequests.ShouldBe(1);
        _launcher.Last!.WasKilled.ShouldBeTrue();
        aufsicht.Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// Silence is no proof of death as long as a process handle says the opposite.
    ///
    /// <para><b>Here lay the most dangerous fallacy.</b> The waiting loop broke off
    /// as soon as the socket no longer answered, reported "ended" and returned <c>true</c>
    /// — without touching the process. With a silent instance, however, the socket
    /// is quiet from the start: the call reported success immediately, the process kept
    /// running, and the restart put a second one on the same <c>state.db</c>. Exactly
    /// the damage this project is built against.</para>
    /// </summary>
    [Fact]
    public async Task Ein_stummer_aber_lebender_Prozess_gilt_nicht_als_beendet()
    {
        _kanal.Answer = Test;

        await using var aufsicht = Aufsicht();
        await aufsicht.StartAsync();

        var prozess = _launcher.Last.ShouldNotBeNull();

        // It hangs: the socket is silent, the process lives.
        _kanal.Verstummen(_dir);

        (await aufsicht.StopAsync()).ShouldBeTrue();

        prozess.WasKilled.ShouldBeTrue(
            "Wer schweigt und lebt, muss beendet werden - nicht fuer beendet erklaert.");

        prozess.HasExited.ShouldBeTrue();
        aufsicht.Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// Whoever is cleaning up is not killed — the deadline is extended.
    ///
    /// <para>The instance gives itself a deadline for running out. If the manager's were
    /// shorter, it would shoot into the middle of the cleanup it has just asked
    /// for.</para>
    /// </summary>
    [Fact]
    public async Task Wer_aufraeumt_bekommt_die_Frist_nachgezogen()
    {
        _kanal.Answer = Test;

        // #722: Supervision asks every 250 ms and only extends the deadline on "cleaning up".
        // With a 300 ms deadline the next question came too late under parallel load, and the test saw
        // a kill where none belongs. A one-second deadline leaves the tick room; the
        // instance still cleans up considerably longer than the deadline alone allows.
        await using var aufsicht = Aufsicht(Options with { StopGrace = TimeSpan.FromSeconds(1), MaxDrainWait = TimeSpan.FromSeconds(20) });
        await aufsicht.StartAsync();

        var prozess = _launcher.Last.ShouldNotBeNull();

        // It answers and reports: I am cleaning up. Longer than StopGrace (1 s), but
        // shorter than MaxDrainWait.
        _kanal.Answer = Test with { Phase = ShutdownPhase.Draining, OpenWork = 1 };

        _kanal.OnStopRequest = _ => Task.Run(async () =>
        {
            await Task.Delay(2500);
            prozess.Exit(0);
            _kanal.Answer = null;
        });

        (await aufsicht.StopAsync()).ShouldBeTrue();

        prozess.WasKilled.ShouldBeFalse("Wer aufraeumt, wird nicht abgeschossen.");
        aufsicht.Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// An instance that the manager did not start itself and that does not listen, it
    /// cannot end either — and it has to say so instead of reporting success.
    ///
    /// <para>The case arises after a manager restart when the process ID can no longer
    /// be grabbed (see <see cref="Eine_wiederverwendete_Prozesskennung_wird_nicht_fuer_die_Instanz_gehalten"/>).</para>
    /// </summary>
    [Fact]
    public async Task Was_nicht_zu_fassen_ist_wird_nicht_als_beendet_gemeldet()
    {
        _kanal.Answer = Test;
        _kanal.StopRequestArrives = false;

        // It was already running before this manager came - and without a state file its
        // process cannot be grabbed by it.
        _launcher.Fremdstart(_dir, 4711);

        await using var aufsicht = Aufsicht();
        await aufsicht.ReattachAsync();          // taken over, but without a process
        aufsicht.ProcessId.ShouldBeNull();

        (await aufsicht.StopAsync()).ShouldBeFalse();

        aufsicht.Condition.ShouldBe(InstanceCondition.Running);
        Arten.ShouldNotContain(InstanceEventKind.StoppedOrderly);
    }

    // ─── Rediscovering (E-M6) ───

    [Fact]
    public async Task Eine_laufende_Instanz_wird_nach_einem_Managerneustart_uebernommen()
    {
        _kanal.Answer = Test;

        // The first manager starts it.
        await using (var erster = Aufsicht())
            await erster.StartAsync();

        _ereignisse.Clear();

        // Second manager, the same machine, the same running instance.
        await using var zweiter = Aufsicht();
        (await zweiter.ReattachAsync()).ShouldBeTrue();

        zweiter.Condition.ShouldBe(InstanceCondition.Running);
        zweiter.InstanceName.ShouldBe("Test");
        zweiter.ProcessId.ShouldBe(_launcher.Last!.Id);

        // Taking over is not starting - there is nothing to report.
        _ereignisse.ShouldBeEmpty();
        _launcher.Starts.ShouldBe(1);
    }

    /// <summary>
    /// A reused process ID must not lead to taking over a foreign
    /// program.
    ///
    /// <para>After a restart of the machine, 4711 belongs with some probability
    /// to something entirely different. Supervising it then — and ending it when stopping —
    /// would be considerably worse than not finding the instance at all.</para>
    /// </summary>
    [Fact]
    public async Task Eine_wiederverwendete_Prozesskennung_wird_nicht_fuer_die_Instanz_gehalten()
    {
        _kanal.Answer = Test;

        await using (var erster = Aufsicht())
            await erster.StartAsync();

        var alte = _launcher.Last!;

        // The same ID, started much later: a different process.
        _launcher.Running[alte.Id] = new FakeProcess(alte.Id, alte.StartedAt.AddHours(3));

        await using var zweiter = Aufsicht();
        (await zweiter.ReattachAsync()).ShouldBeTrue("Der Socket antwortet ja.");

        zweiter.ProcessId.ShouldBeNull(
            "Der fremde Prozess darf nicht als der eigene uebernommen werden.");
    }

    [Fact]
    public async Task Eine_Zustandsdatei_ohne_laufende_Instanz_wird_weggeraeumt()
    {
        _kanal.Answer = Test;

        await using (var erster = Aufsicht())
            await erster.StartAsync();

        InstanceStateStore.Read(_dir).ShouldNotBeNull();

        // The manager was gone, the instance is gone by now too.
        _kanal.Answer = null;

        await using var zweiter = Aufsicht();
        (await zweiter.ReattachAsync()).ShouldBeFalse();

        zweiter.Condition.ShouldBe(InstanceCondition.Stopped);
        InstanceStateStore.Read(_dir).ShouldBeNull(
            "Eine Datei, die auf niemanden zeigt, laesst es aussehen, als liefe etwas.");
    }

    [Fact]
    public async Task Ohne_Zustandsdatei_wird_eine_antwortende_Instanz_trotzdem_uebernommen()
    {
        // The case after a crash of the manager before it could write - or
        // after someone deleted the file. The socket answers, so it is running.
        _kanal.Answer = Test;
        _launcher.Fremdstart(_dir, 4711);

        await using var aufsicht = Aufsicht();
        (await aufsicht.ReattachAsync()).ShouldBeTrue();

        aufsicht.Condition.ShouldBe(InstanceCondition.Running);
        aufsicht.ProcessId.ShouldBeNull("Es gibt keinen Prozess, den dieser Manager kennt.");
        InstanceStateStore.Read(_dir).ShouldNotBeNull();
    }

    // ─── What is reported and what is not ───

    /// <summary>
    /// The manager reports what is relevant from its point of view — and only that (concept,
    /// section 4). What concerns the instance stays in the instance and goes out with its
    /// own sub-token.
    /// </summary>
    [Theory]
    [InlineData(InstanceEventKind.Crashed, true)]
    [InlineData(InstanceEventKind.Unresponsive, true)]
    [InlineData(InstanceEventKind.StartFailed, true)]
    [InlineData(InstanceEventKind.Started, false)]
    [InlineData(InstanceEventKind.StoppedOrderly, false)]
    [InlineData(InstanceEventKind.Recovered, false)]
    public void Gemeldet_wird_nur_was_niemand_bestellt_hat(InstanceEventKind kind, bool erwartet)
    {
        new SupervisionEvent(kind, _dir, "Test", 1, null, "", DateTimeOffset.UtcNow)
            .WorthReporting.ShouldBe(erwartet);
    }
}