ClawdDotNet/tests/ClawdDotNet.Manager.Tests/AutostartTests.cs

using System.Text.Json;
using ClawdDotNet.Core.Config;
using ClawdDotNet.Core.Instances;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;

namespace ClawdDotNet.Manager.Tests;

/// <summary>
/// "Autostart by manager" — the switch that keeps an instance running (#528).
///
/// <para><b>Two things in one switch, because it is the same promise:</b> the manager
/// starts the instance when coming up, and it starts it again after a crash. Without
/// the switch it touches nothing — a manager that boots everything it finds after a restart of the machine
/// also starts what someone stopped on purpose.</para>
///
/// <para><b>And the circuit breaker is the actual content.</b> Booting again
/// is the easy half; the hard one is recognizing when to stop.</para>
/// </summary>
public sealed class AutostartTests : IAsyncDisposable
{
    private readonly string _root = Path.Combine(
        Path.GetTempPath(), "clawd-tests", "auto-" + Guid.NewGuid().ToString("N")[..8]);

    private readonly FakeLauncher _launcher = new();
    private readonly FakeChannel _kanal = new();
    private readonly FakeTokenSource _tokens = new();
    private readonly InstanceDirectoryManager _verzeichnisse;

    private static readonly SupervisionOptions Options = new(
        ProbeInterval: TimeSpan.FromMilliseconds(50),
        StartTimeout: TimeSpan.FromMilliseconds(600),
        SilentChecksBeforeAlarm: 3,
        StopGrace: TimeSpan.FromMilliseconds(300),
        SilentKillGrace: TimeSpan.FromMilliseconds(300),
        MaxDrainWait: TimeSpan.FromSeconds(2));

    public AutostartTests()
    {
        Directory.CreateDirectory(_root);
        _verzeichnisse = new InstanceDirectoryManager(Path.Combine(_root, "Instances"));

        // The socket only answers if a process is really running - otherwise an
        // answer stored in advance would mean "already running", and the manager would take over
        // instead of starting.
        _kanal.Launcher = _launcher;
    }

    public async ValueTask DisposeAsync()
    {
        await Task.CompletedTask;
        try { Directory.Delete(_root, recursive: true); }
        catch { /* A leftover test folder does not turn a run red. */ }
    }

    private ManagerCore Manager(RestartPolicy? policy = null)
    {
        var register = new InstanceRegistry(
            _verzeichnisse, _launcher, _kanal, NullLoggerFactory.Instance, Options);

        return new ManagerCore(
            new ManagerSettingsStore(_root), register, _tokens, NullLoggerFactory.Instance,
            restartPolicy: policy);
    }

    /// <summary>Creates an instance and sets the switch in its instance file.</summary>
    private string Instanz(string name, bool autostart)
    {
        var pfad = _verzeichnisse.CreateInstance(name);
        var datei = Path.Combine(pfad, "InstanceSettings.json");

        var config = JsonSerializer.Deserialize<InstanceConfig>(File.ReadAllText(datei))!;
        config.AutostartByManager = autostart;
        File.WriteAllText(datei, JsonSerializer.Serialize(config));

        _kanal.AnswerFor[Path.GetFullPath(pfad)] = new InstanceIdentity("i-" + name, name, "0.1.2");

        return pfad;
    }

    private static SupervisionEvent Absturz(string pfad, string name) => new(
        InstanceEventKind.Crashed, Path.GetFullPath(pfad), name, 4711, 137,
        "unerwartet beendet", DateTimeOffset.UtcNow);

    // ─── When coming up ───

    [Fact]
    public async Task Eine_Instanz_mit_Autostart_wird_beim_Hochfahren_gestartet()
    {
        Instanz("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        _launcher.Starts.ShouldBe(1);
        manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Running);
    }

    [Fact]
    public async Task Ohne_Autostart_bleibt_sie_aus()
    {
        Instanz("Alpha", autostart: false);

        await using var manager = Manager();
        await manager.StartAsync();

        _launcher.Starts.ShouldBe(0);
        manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Stopped);
    }

    /// <summary>
    /// What is already running is not started again: a second process on the same
    /// <c>state.db</c> would be the opposite of helpful.
    /// </summary>
    [Fact]
    public async Task Was_schon_laeuft_wird_nicht_noch_einmal_gestartet()
    {
        var pfad = Instanz("Alpha", autostart: true);

        await using (var erster = Manager())
            await erster.StartAsync();

        _launcher.Starts.ShouldBe(1);

        await using var zweiter = Manager();
        await zweiter.StartAsync();

        _launcher.Starts.ShouldBe(1, "Uebernommen, nicht neu gestartet.");
        zweiter.Instances.Instances.Single().ProcessId.ShouldBe(_launcher.Last!.Id);
    }

    [Fact]
    public async Task Der_Schalter_steht_in_der_Liste()
    {
        Instanz("Mit", autostart: true);
        Instanz("Ohne", autostart: false);

        await using var manager = Manager();
        await manager.StartAsync();

        manager.Instances.Instances.Single(i => i.Name == "Mit").AutostartByManager.ShouldBeTrue();
        manager.Instances.Instances.Single(i => i.Name == "Ohne").AutostartByManager.ShouldBeFalse();
    }

    // ─── After a crash ───

    [Fact]
    public async Task Nach_einem_Absturz_wird_wieder_gestartet()
    {
        var pfad = Instanz("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        var vorher = _launcher.Starts;

        // The process is gone; supervision has reported it.
        _launcher.Last!.Exit(137);
        await manager.RestartAfterCrashAsync(Absturz(pfad, "Alpha"));

        _launcher.Starts.ShouldBe(vorher + 1);
    }

    [Fact]
    public async Task Ohne_Autostart_wird_nach_einem_Absturz_nichts_getan()
    {
        var pfad = Instanz("Alpha", autostart: false);

        await using var manager = Manager();
        await manager.StartAsync();
        await manager.StartInstanceAsync(pfad);

        _launcher.Last!.Exit(137);
        await manager.RestartAfterCrashAsync(Absturz(pfad, "Alpha"));

        _launcher.Starts.ShouldBe(1, "Nur der Start von Hand, kein Neustart.");
    }

    /// <summary>
    /// A silent instance is ended and restarted — but only after the
    /// old process is provably gone.
    ///
    /// <para><b>The case is really produced here and not claimed.</b> The
    /// process lives on, only the socket is silent — exactly the situation from #529. A test
    /// that instead builds the event by hand and calls the restart method checks
    /// the half that works anyway, and leaves out the dangerous one: that before
    /// the start nobody sits on the database any more.</para>
    /// </summary>
    [Fact]
    public async Task Eine_verstummte_Instanz_wird_erst_beendet_und_dann_neu_gestartet()
    {
        var pfad = Instanz("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        var alt = _launcher.Last.ShouldNotBeNull();
        _launcher.Starts.ShouldBe(1);

        // From now on it hangs: the process lives, the socket no longer answers.
        _kanal.Verstummen(pfad);

        var aufsicht = manager.Instances.SupervisorFor(pfad).ShouldNotBeNull();

        // Three passes until the alarm - nothing is done before that.
        for (var i = 0; i < 3; i++) await aufsicht.CheckAsync();

        aufsicht.Condition.ShouldBe(InstanceCondition.Silent);
        alt.WasKilled.ShouldBeFalse("Vor Ablauf der Frist wird nicht abgeschossen.");

        // The deadline expires (SilentKillGrace: 300ms).
        await Task.Delay(400);
        await aufsicht.CheckAsync();

        alt.WasKilled.ShouldBeTrue("Nach der Frist wird der haengende Prozess beendet.");

        // The crash path restarts - and the new process is a different one.
        await Warte(() => _launcher.Starts >= 2);

        _launcher.Last.ShouldNotBeNull().Id.ShouldNotBe(alt.Id);
        alt.HasExited.ShouldBeTrue("Der alte Prozess darf beim Start nicht mehr leben.");
    }

    /// <summary>
    /// An instance that is cleaning up is not killed — even if it takes a while.
    ///
    /// <para>It does answer; it is doing exactly what it was asked to do. The difference
    /// between "hangs" and "cleans up" is the whole reason why the pulse says more than
    /// "someone answered".</para>
    /// </summary>
    [Fact]
    public async Task Wer_aufraeumt_gilt_nicht_als_verstummt()
    {
        var pfad = Instanz("Alpha", autostart: true);

        await using var manager = Manager();
        await manager.StartAsync();

        var prozess = _launcher.Last.ShouldNotBeNull();

        _kanal.AnswerFor[Path.GetFullPath(pfad)] = new InstanceIdentity(
            "i-Alpha", "Alpha", "0.1.2",
            ShutdownPhase.Draining, OpenWork: 2, OpenSummary: "Agentenläufe: 2");

        var aufsicht = manager.Instances.SupervisorFor(pfad).ShouldNotBeNull();

        for (var i = 0; i < 5; i++) await aufsicht.CheckAsync();

        aufsicht.Condition.ShouldBe(InstanceCondition.Running);
        prozess.WasKilled.ShouldBeFalse();
        _launcher.Starts.ShouldBe(1);
    }

    private static async Task Warte(Func<bool> bedingung)
    {
        for (var i = 0; i < 100 && !bedingung(); i++)
            await Task.Delay(20);
    }

    /// <summary>
    /// After the limit the manager pauses.
    ///
    /// <para>What fails because of a broken configuration fails the same way on the tenth attempt
    /// — then instead of a dead instance you have a dead instance, a full
    /// log and a bug tracker full of repetitions.</para>
    /// </summary>
    [Fact]
    public async Task Nach_drei_Neustarts_im_Zeitfenster_setzt_der_Manager_aus()
    {
        var pfad = Instanz("Wackelig", autostart: true);

        await using var manager = Manager(new RestartPolicy(MaxRestarts: 3, Window: TimeSpan.FromMinutes(10)));
        await manager.StartAsync();

        var nachStart = _launcher.Starts;

        for (var i = 0; i < 6; i++)
        {
            _launcher.Last!.Exit(137);
            await manager.RestartAfterCrashAsync(Absturz(pfad, "Wackelig"));
        }

        (_launcher.Starts - nachStart).ShouldBe(3, "Drei Versuche, dann Schluss.");
    }

    /// <summary>
    /// Whoever starts by hand has either changed something or knows what they are doing. Charging them with
    /// the earlier attempts would mean punishing them for the crash.
    /// </summary>
    [Fact]
    public async Task Ein_Start_von_Hand_raeumt_den_Zaehler_ab()
    {
        var pfad = Instanz("Wackelig", autostart: true);

        await using var manager = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10)));
        await manager.StartAsync();

        for (var i = 0; i < 5; i++)
        {
            _launcher.Last!.Exit(137);
            await manager.RestartAfterCrashAsync(Absturz(pfad, "Wackelig"));
        }

        var erschoepft = _launcher.Starts;

        // A new manager starts at zero - just like a start by hand.
        await using var frisch = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10)));
        await frisch.StartAsync();

        _launcher.Last!.Exit(137);
        await frisch.RestartAfterCrashAsync(Absturz(pfad, "Wackelig"));

        _launcher.Starts.ShouldBeGreaterThan(erschoepft);
    }
}

/// <summary>The circuit breaker on its own.</summary>
public sealed class RestartLedgerTests
{
    private sealed class Uhr(DateTimeOffset jetzt) : TimeProvider
    {
        public DateTimeOffset Jetzt { get; set; } = jetzt;
        public override DateTimeOffset GetUtcNow() => Jetzt;
    }

    [Fact]
    public void Bis_zur_Grenze_ja_danach_nein()
    {
        var buch = new RestartLedger(new RestartPolicy(3, TimeSpan.FromMinutes(10)));

        buch.TryRecord("/a", out _).ShouldBeTrue();
        buch.TryRecord("/a", out _).ShouldBeTrue();
        buch.TryRecord("/a", out var dritter).ShouldBeTrue();
        dritter.ShouldBe(3);

        buch.TryRecord("/a", out _).ShouldBeFalse();
    }

    [Fact]
    public void Jede_Instanz_hat_ihr_eigenes_Konto()
    {
        var buch = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10)));

        buch.TryRecord("/a", out _).ShouldBeTrue();
        buch.TryRecord("/a", out _).ShouldBeFalse();

        buch.TryRecord("/b", out _).ShouldBeTrue("Die eine Instanz sperrt die andere nicht.");
    }

    /// <summary>
    /// The time window moves: an instance that crashes once a month and comes back
    /// up never runs into the limit — one that falls every minute does immediately.
    /// </summary>
    [Fact]
    public void Alte_Versuche_fallen_aus_dem_Fenster()
    {
        var uhr = new Uhr(DateTimeOffset.UnixEpoch);
        var buch = new RestartLedger(new RestartPolicy(2, TimeSpan.FromMinutes(10)), uhr);

        buch.TryRecord("/a", out _).ShouldBeTrue();
        buch.TryRecord("/a", out _).ShouldBeTrue();
        buch.TryRecord("/a", out _).ShouldBeFalse();

        uhr.Jetzt = uhr.Jetzt.AddMinutes(11);

        buch.TryRecord("/a", out var wieder).ShouldBeTrue();
        wieder.ShouldBe(1, "Die alten Versuche zaehlen nicht mehr mit.");
    }

    [Fact]
    public void Vergessen_setzt_zurueck()
    {
        var buch = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10)));

        buch.TryRecord("/a", out _).ShouldBeTrue();
        buch.TryRecord("/a", out _).ShouldBeFalse();

        buch.Forget("/a");

        buch.TryRecord("/a", out _).ShouldBeTrue();
    }
}

/// <summary>
/// A start that meets an outdated note.
///
/// <para><b>The defect that only the autostart tests revealed.</b> Between two
/// supervision passes there are seconds, and in them an instance can break away. Whoever
/// asked to "start" in this gap got "already running" — a success with nothing
/// behind it. The next pass then reported a crash nobody expected any more.</para>
/// </summary>
public sealed class StaleStateTests : IAsyncDisposable
{
    private readonly string _root = Path.Combine(
        Path.GetTempPath(), "clawd-tests", "stale-" + Guid.NewGuid().ToString("N")[..8]);

    private readonly FakeLauncher _launcher = new();
    private readonly FakeChannel _kanal = new();

    public StaleStateTests()
    {
        Directory.CreateDirectory(_root);
        _kanal.Launcher = _launcher;
        _kanal.Answer = new InstanceIdentity("i-1", "Alpha", "0.1.2");
    }

    public async ValueTask DisposeAsync()
    {
        await Task.CompletedTask;
        try { Directory.Delete(_root, recursive: true); }
        catch { /* A leftover test folder does not turn a run red. */ }
    }

    [Fact]
    public async Task Ein_Start_prueft_nach_statt_dem_eigenen_Zustand_zu_glauben()
    {
        var dir = Path.Combine(_root, "Instance-Alpha");
        Directory.CreateDirectory(dir);

        await using var aufsicht = new InstanceSupervisor(
            dir, _launcher, _kanal, NullLogger.Instance,
            new SupervisionOptions(
                TimeSpan.FromSeconds(30),          // deliberately sluggish: no pass in between
                TimeSpan.FromMilliseconds(600),
                3,
                TimeSpan.FromMilliseconds(300),
                TimeSpan.FromMilliseconds(300),
                TimeSpan.FromSeconds(2)));

        (await aufsicht.StartAsync()).ShouldBeTrue();
        aufsicht.Condition.ShouldBe(InstanceCondition.Running);

        // The instance breaks away. Supervision has not noticed yet - the next
        // pass would only come in thirty seconds.
        _launcher.Last!.Exit(137);
        aufsicht.Condition.ShouldBe(InstanceCondition.Running, "Noch nichts bemerkt.");

        (await aufsicht.StartAsync()).ShouldBeTrue();

        _launcher.Starts.ShouldBe(2, "Es wurde wirklich gestartet, nicht nur zugestimmt.");
        aufsicht.Condition.ShouldBe(InstanceCondition.Running);
    }
}