using System.Text.Json; using ClawdDotNet.Core.Config; using ClawdDotNet.Core.Instances; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; namespace ClawdDotNet.Manager.Tests; /// /// "Autostart by manager" — the switch that keeps an instance running (#528). /// /// Two things in one switch, because it is the same promise: the manager /// starts the instance when coming up, and it starts it again after a crash. Without /// the switch it touches nothing — a manager that boots everything it finds after a restart of the machine /// also starts what someone stopped on purpose. /// /// And the circuit breaker is the actual content. Booting again /// is the easy half; the hard one is recognizing when to stop. /// public sealed class AutostartTests : IAsyncDisposable { private readonly string _root = Path.Combine( Path.GetTempPath(), "clawd-tests", "auto-" + Guid.NewGuid().ToString("N")[..8]); private readonly FakeLauncher _launcher = new(); private readonly FakeChannel _kanal = new(); private readonly FakeTokenSource _tokens = new(); private readonly InstanceDirectoryManager _verzeichnisse; private static readonly SupervisionOptions Options = new( ProbeInterval: TimeSpan.FromMilliseconds(50), StartTimeout: TimeSpan.FromMilliseconds(600), SilentChecksBeforeAlarm: 3, StopGrace: TimeSpan.FromMilliseconds(300), SilentKillGrace: TimeSpan.FromMilliseconds(300), MaxDrainWait: TimeSpan.FromSeconds(2)); public AutostartTests() { Directory.CreateDirectory(_root); _verzeichnisse = new InstanceDirectoryManager(Path.Combine(_root, "Instances")); // The socket only answers if a process is really running - otherwise an // answer stored in advance would mean "already running", and the manager would take over // instead of starting. _kanal.Launcher = _launcher; } public async ValueTask DisposeAsync() { await Task.CompletedTask; try { Directory.Delete(_root, recursive: true); } catch { /* A leftover test folder does not turn a run red. */ } } private ManagerCore Manager(RestartPolicy? policy = null) { var register = new InstanceRegistry( _verzeichnisse, _launcher, _kanal, NullLoggerFactory.Instance, Options); return new ManagerCore( new ManagerSettingsStore(_root), register, _tokens, NullLoggerFactory.Instance, restartPolicy: policy); } /// Creates an instance and sets the switch in its instance file. private string Instanz(string name, bool autostart) { var pfad = _verzeichnisse.CreateInstance(name); var datei = Path.Combine(pfad, "InstanceSettings.json"); var config = JsonSerializer.Deserialize(File.ReadAllText(datei))!; config.AutostartByManager = autostart; File.WriteAllText(datei, JsonSerializer.Serialize(config)); _kanal.AnswerFor[Path.GetFullPath(pfad)] = new InstanceIdentity("i-" + name, name, "0.1.2"); return pfad; } private static SupervisionEvent Absturz(string pfad, string name) => new( InstanceEventKind.Crashed, Path.GetFullPath(pfad), name, 4711, 137, "unerwartet beendet", DateTimeOffset.UtcNow); // ─── When coming up ─── [Fact] public async Task Eine_Instanz_mit_Autostart_wird_beim_Hochfahren_gestartet() { Instanz("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); _launcher.Starts.ShouldBe(1); manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Running); } [Fact] public async Task Ohne_Autostart_bleibt_sie_aus() { Instanz("Alpha", autostart: false); await using var manager = Manager(); await manager.StartAsync(); _launcher.Starts.ShouldBe(0); manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Stopped); } /// /// What is already running is not started again: a second process on the same /// state.db would be the opposite of helpful. /// [Fact] public async Task Was_schon_laeuft_wird_nicht_noch_einmal_gestartet() { var pfad = Instanz("Alpha", autostart: true); await using (var erster = Manager()) await erster.StartAsync(); _launcher.Starts.ShouldBe(1); await using var zweiter = Manager(); await zweiter.StartAsync(); _launcher.Starts.ShouldBe(1, "Uebernommen, nicht neu gestartet."); zweiter.Instances.Instances.Single().ProcessId.ShouldBe(_launcher.Last!.Id); } [Fact] public async Task Der_Schalter_steht_in_der_Liste() { Instanz("Mit", autostart: true); Instanz("Ohne", autostart: false); await using var manager = Manager(); await manager.StartAsync(); manager.Instances.Instances.Single(i => i.Name == "Mit").AutostartByManager.ShouldBeTrue(); manager.Instances.Instances.Single(i => i.Name == "Ohne").AutostartByManager.ShouldBeFalse(); } // ─── After a crash ─── [Fact] public async Task Nach_einem_Absturz_wird_wieder_gestartet() { var pfad = Instanz("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); var vorher = _launcher.Starts; // The process is gone; supervision has reported it. _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Absturz(pfad, "Alpha")); _launcher.Starts.ShouldBe(vorher + 1); } [Fact] public async Task Ohne_Autostart_wird_nach_einem_Absturz_nichts_getan() { var pfad = Instanz("Alpha", autostart: false); await using var manager = Manager(); await manager.StartAsync(); await manager.StartInstanceAsync(pfad); _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Absturz(pfad, "Alpha")); _launcher.Starts.ShouldBe(1, "Nur der Start von Hand, kein Neustart."); } /// /// A silent instance is ended and restarted — but only after the /// old process is provably gone. /// /// The case is really produced here and not claimed. The /// process lives on, only the socket is silent — exactly the situation from #529. A test /// that instead builds the event by hand and calls the restart method checks /// the half that works anyway, and leaves out the dangerous one: that before /// the start nobody sits on the database any more. /// [Fact] public async Task Eine_verstummte_Instanz_wird_erst_beendet_und_dann_neu_gestartet() { var pfad = Instanz("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); var alt = _launcher.Last.ShouldNotBeNull(); _launcher.Starts.ShouldBe(1); // From now on it hangs: the process lives, the socket no longer answers. _kanal.Verstummen(pfad); var aufsicht = manager.Instances.SupervisorFor(pfad).ShouldNotBeNull(); // Three passes until the alarm - nothing is done before that. for (var i = 0; i < 3; i++) await aufsicht.CheckAsync(); aufsicht.Condition.ShouldBe(InstanceCondition.Silent); alt.WasKilled.ShouldBeFalse("Vor Ablauf der Frist wird nicht abgeschossen."); // The deadline expires (SilentKillGrace: 300ms). await Task.Delay(400); await aufsicht.CheckAsync(); alt.WasKilled.ShouldBeTrue("Nach der Frist wird der haengende Prozess beendet."); // The crash path restarts - and the new process is a different one. await Warte(() => _launcher.Starts >= 2); _launcher.Last.ShouldNotBeNull().Id.ShouldNotBe(alt.Id); alt.HasExited.ShouldBeTrue("Der alte Prozess darf beim Start nicht mehr leben."); } /// /// An instance that is cleaning up is not killed — even if it takes a while. /// /// It does answer; it is doing exactly what it was asked to do. The difference /// between "hangs" and "cleans up" is the whole reason why the pulse says more than /// "someone answered". /// [Fact] public async Task Wer_aufraeumt_gilt_nicht_als_verstummt() { var pfad = Instanz("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); var prozess = _launcher.Last.ShouldNotBeNull(); _kanal.AnswerFor[Path.GetFullPath(pfad)] = new InstanceIdentity( "i-Alpha", "Alpha", "0.1.2", ShutdownPhase.Draining, OpenWork: 2, OpenSummary: "Agentenläufe: 2"); var aufsicht = manager.Instances.SupervisorFor(pfad).ShouldNotBeNull(); for (var i = 0; i < 5; i++) await aufsicht.CheckAsync(); aufsicht.Condition.ShouldBe(InstanceCondition.Running); prozess.WasKilled.ShouldBeFalse(); _launcher.Starts.ShouldBe(1); } private static async Task Warte(Func bedingung) { for (var i = 0; i < 100 && !bedingung(); i++) await Task.Delay(20); } /// /// After the limit the manager pauses. /// /// What fails because of a broken configuration fails the same way on the tenth attempt /// — then instead of a dead instance you have a dead instance, a full /// log and a bug tracker full of repetitions. /// [Fact] public async Task Nach_drei_Neustarts_im_Zeitfenster_setzt_der_Manager_aus() { var pfad = Instanz("Wackelig", autostart: true); await using var manager = Manager(new RestartPolicy(MaxRestarts: 3, Window: TimeSpan.FromMinutes(10))); await manager.StartAsync(); var nachStart = _launcher.Starts; for (var i = 0; i < 6; i++) { _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Absturz(pfad, "Wackelig")); } (_launcher.Starts - nachStart).ShouldBe(3, "Drei Versuche, dann Schluss."); } /// /// Whoever starts by hand has either changed something or knows what they are doing. Charging them with /// the earlier attempts would mean punishing them for the crash. /// [Fact] public async Task Ein_Start_von_Hand_raeumt_den_Zaehler_ab() { var pfad = Instanz("Wackelig", autostart: true); await using var manager = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10))); await manager.StartAsync(); for (var i = 0; i < 5; i++) { _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Absturz(pfad, "Wackelig")); } var erschoepft = _launcher.Starts; // A new manager starts at zero - just like a start by hand. await using var frisch = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10))); await frisch.StartAsync(); _launcher.Last!.Exit(137); await frisch.RestartAfterCrashAsync(Absturz(pfad, "Wackelig")); _launcher.Starts.ShouldBeGreaterThan(erschoepft); } } /// The circuit breaker on its own. public sealed class RestartLedgerTests { private sealed class Uhr(DateTimeOffset jetzt) : TimeProvider { public DateTimeOffset Jetzt { get; set; } = jetzt; public override DateTimeOffset GetUtcNow() => Jetzt; } [Fact] public void Bis_zur_Grenze_ja_danach_nein() { var buch = new RestartLedger(new RestartPolicy(3, TimeSpan.FromMinutes(10))); buch.TryRecord("/a", out _).ShouldBeTrue(); buch.TryRecord("/a", out _).ShouldBeTrue(); buch.TryRecord("/a", out var dritter).ShouldBeTrue(); dritter.ShouldBe(3); buch.TryRecord("/a", out _).ShouldBeFalse(); } [Fact] public void Jede_Instanz_hat_ihr_eigenes_Konto() { var buch = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10))); buch.TryRecord("/a", out _).ShouldBeTrue(); buch.TryRecord("/a", out _).ShouldBeFalse(); buch.TryRecord("/b", out _).ShouldBeTrue("Die eine Instanz sperrt die andere nicht."); } /// /// The time window moves: an instance that crashes once a month and comes back /// up never runs into the limit — one that falls every minute does immediately. /// [Fact] public void Alte_Versuche_fallen_aus_dem_Fenster() { var uhr = new Uhr(DateTimeOffset.UnixEpoch); var buch = new RestartLedger(new RestartPolicy(2, TimeSpan.FromMinutes(10)), uhr); buch.TryRecord("/a", out _).ShouldBeTrue(); buch.TryRecord("/a", out _).ShouldBeTrue(); buch.TryRecord("/a", out _).ShouldBeFalse(); uhr.Jetzt = uhr.Jetzt.AddMinutes(11); buch.TryRecord("/a", out var wieder).ShouldBeTrue(); wieder.ShouldBe(1, "Die alten Versuche zaehlen nicht mehr mit."); } [Fact] public void Vergessen_setzt_zurueck() { var buch = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10))); buch.TryRecord("/a", out _).ShouldBeTrue(); buch.TryRecord("/a", out _).ShouldBeFalse(); buch.Forget("/a"); buch.TryRecord("/a", out _).ShouldBeTrue(); } } /// /// A start that meets an outdated note. /// /// The defect that only the autostart tests revealed. Between two /// supervision passes there are seconds, and in them an instance can break away. Whoever /// asked to "start" in this gap got "already running" — a success with nothing /// behind it. The next pass then reported a crash nobody expected any more. /// public sealed class StaleStateTests : IAsyncDisposable { private readonly string _root = Path.Combine( Path.GetTempPath(), "clawd-tests", "stale-" + Guid.NewGuid().ToString("N")[..8]); private readonly FakeLauncher _launcher = new(); private readonly FakeChannel _kanal = new(); public StaleStateTests() { Directory.CreateDirectory(_root); _kanal.Launcher = _launcher; _kanal.Answer = new InstanceIdentity("i-1", "Alpha", "0.1.2"); } public async ValueTask DisposeAsync() { await Task.CompletedTask; try { Directory.Delete(_root, recursive: true); } catch { /* A leftover test folder does not turn a run red. */ } } [Fact] public async Task Ein_Start_prueft_nach_statt_dem_eigenen_Zustand_zu_glauben() { var dir = Path.Combine(_root, "Instance-Alpha"); Directory.CreateDirectory(dir); await using var aufsicht = new InstanceSupervisor( dir, _launcher, _kanal, NullLogger.Instance, new SupervisionOptions( TimeSpan.FromSeconds(30), // deliberately sluggish: no pass in between TimeSpan.FromMilliseconds(600), 3, TimeSpan.FromMilliseconds(300), TimeSpan.FromMilliseconds(300), TimeSpan.FromSeconds(2))); (await aufsicht.StartAsync()).ShouldBeTrue(); aufsicht.Condition.ShouldBe(InstanceCondition.Running); // The instance breaks away. Supervision has not noticed yet - the next // pass would only come in thirty seconds. _launcher.Last!.Exit(137); aufsicht.Condition.ShouldBe(InstanceCondition.Running, "Noch nichts bemerkt."); (await aufsicht.StartAsync()).ShouldBeTrue(); _launcher.Starts.ShouldBe(2, "Es wurde wirklich gestartet, nicht nur zugestimmt."); aufsicht.Condition.ShouldBe(InstanceCondition.Running); } }