using System.Text.Json; using ClawdDotNet.Core.Config; using ClawdDotNet.Core.Instances; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; namespace ClawdDotNet.Manager.Tests; /// /// "Autostart by manager" — the switch that keeps an instance running (#528). /// /// Two things in one switch, because it is the same promise: the manager /// starts the instance when coming up, and it starts it again after a crash. Without /// the switch it touches nothing — a manager that boots everything it finds after a restart of the machine /// also starts what someone stopped on purpose. /// /// And the circuit breaker is the actual content. Booting again /// is the easy half; the hard one is recognizing when to stop. /// public sealed class AutostartTests : IAsyncDisposable { private readonly string _root = Path.Combine( Path.GetTempPath(), "clawd-tests", "auto-" + Guid.NewGuid().ToString("N")[..8]); private readonly FakeLauncher _launcher = new(); private readonly FakeChannel _channel = new(); private readonly FakeTokenSource _tokens = new(); private readonly InstanceDirectoryManager _directories; private static readonly SupervisionOptions Options = new( ProbeInterval: TimeSpan.FromMilliseconds(50), StartTimeout: TimeSpan.FromMilliseconds(600), SilentChecksBeforeAlarm: 3, StopGrace: TimeSpan.FromMilliseconds(300), SilentKillGrace: TimeSpan.FromMilliseconds(300), MaxDrainWait: TimeSpan.FromSeconds(2)); public AutostartTests() { Directory.CreateDirectory(_root); _directories = new InstanceDirectoryManager(Path.Combine(_root, "Instances")); // The socket only answers if a process is really running - otherwise an // answer stored in advance would mean "already running", and the manager would take over // instead of starting. _channel.Launcher = _launcher; } public async ValueTask DisposeAsync() { await Task.CompletedTask; try { Directory.Delete(_root, recursive: true); } catch { /* A leftover test folder does not turn a run red. */ } } private ManagerCore Manager(RestartPolicy? policy = null) { var register = new InstanceRegistry( _directories, _launcher, _channel, NullLoggerFactory.Instance, Options); return new ManagerCore( new ManagerSettingsStore(_root), register, _tokens, NullLoggerFactory.Instance, restartPolicy: policy); } /// Creates an instance and sets the switch in its instance file. private string Instance(string name, bool autostart) { var path = _directories.CreateInstance(name); var datei = Path.Combine(path, "InstanceSettings.json"); var config = JsonSerializer.Deserialize(File.ReadAllText(datei))!; config.AutostartByManager = autostart; File.WriteAllText(datei, JsonSerializer.Serialize(config)); _channel.AnswerFor[Path.GetFullPath(path)] = new InstanceIdentity("i-" + name, name, "0.1.2"); return path; } private static SupervisionEvent Crash(string path, string name) => new( InstanceEventKind.Crashed, Path.GetFullPath(path), name, 4711, 137, "unerwartet beendet", DateTimeOffset.UtcNow); // ─── When coming up ─── [Fact] public async Task An_instance_with_autostart_is_started_on_startup() { Instance("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); _launcher.Starts.ShouldBe(1); manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Running); } [Fact] public async Task Without_autostart_it_stays_off() { Instance("Alpha", autostart: false); await using var manager = Manager(); await manager.StartAsync(); _launcher.Starts.ShouldBe(0); manager.Instances.Instances.Single().Condition.ShouldBe(InstanceCondition.Stopped); } /// /// What is already running is not started again: a second process on the same /// state.db would be the opposite of helpful. /// [Fact] public async Task What_already_runs_is_not_started_again() { var path = Instance("Alpha", autostart: true); await using (var firstOne = Manager()) await firstOne.StartAsync(); _launcher.Starts.ShouldBe(1); await using var secondOne = Manager(); await secondOne.StartAsync(); _launcher.Starts.ShouldBe(1, "Uebernommen, nicht neu gestartet."); secondOne.Instances.Instances.Single().ProcessId.ShouldBe(_launcher.Last!.Id); } [Fact] public async Task The_switch_is_in_the_list() { Instance("Mit", autostart: true); Instance("Ohne", autostart: false); await using var manager = Manager(); await manager.StartAsync(); manager.Instances.Instances.Single(i => i.Name == "Mit").AutostartByManager.ShouldBeTrue(); manager.Instances.Instances.Single(i => i.Name == "Ohne").AutostartByManager.ShouldBeFalse(); } // ─── After a crash ─── [Fact] public async Task After_a_crash_it_is_restarted() { var path = Instance("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); var before = _launcher.Starts; // The process is gone; supervision has reported it. _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Crash(path, "Alpha")); _launcher.Starts.ShouldBe(before + 1); } [Fact] public async Task Without_autostart_nothing_is_done_after_a_crash() { var path = Instance("Alpha", autostart: false); await using var manager = Manager(); await manager.StartAsync(); await manager.StartInstanceAsync(path); _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Crash(path, "Alpha")); _launcher.Starts.ShouldBe(1, "Nur der Start von Hand, kein Neustart."); } /// /// A silent instance is ended and restarted — but only after the /// old process is provably gone. /// /// The case is really produced here and not claimed. The /// process lives on, only the socket is silent — exactly the situation from #529. A test /// that instead builds the event by hand and calls the restart method checks /// the half that works anyway, and leaves out the dangerous one: that before /// the start nobody sits on the database any more. /// [Fact] public async Task A_silent_instance_is_first_stopped_and_then_restarted() { var path = Instance("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); var old = _launcher.Last.ShouldNotBeNull(); _launcher.Starts.ShouldBe(1); // From now on it hangs: the process lives, the socket no longer answers. _channel.GoSilent(path); var supervisor = manager.Instances.SupervisorFor(path).ShouldNotBeNull(); // Three passes until the alarm - nothing is done before that. for (var i = 0; i < 3; i++) await supervisor.CheckAsync(); supervisor.Condition.ShouldBe(InstanceCondition.Silent); old.WasKilled.ShouldBeFalse("Vor Ablauf der Frist wird nicht abgeschossen."); // The deadline expires (SilentKillGrace: 300ms). await Task.Delay(400); await supervisor.CheckAsync(); old.WasKilled.ShouldBeTrue("Nach der Frist wird der haengende Prozess beendet."); // The crash path restarts - and the new process is a different one. await WaitFor(() => _launcher.Starts >= 2); _launcher.Last.ShouldNotBeNull().Id.ShouldNotBe(old.Id); old.HasExited.ShouldBeTrue("Der alte Prozess darf beim Start nicht mehr leben."); } /// /// An instance that is cleaning up is not killed — even if it takes a while. /// /// It does answer; it is doing exactly what it was asked to do. The difference /// between "hangs" and "cleans up" is the whole reason why the pulse says more than /// "someone answered". /// [Fact] public async Task Whoever_cleans_up_does_not_count_as_silent() { var path = Instance("Alpha", autostart: true); await using var manager = Manager(); await manager.StartAsync(); var process = _launcher.Last.ShouldNotBeNull(); _channel.AnswerFor[Path.GetFullPath(path)] = new InstanceIdentity( "i-Alpha", "Alpha", "0.1.2", ShutdownPhase.Draining, OpenWork: 2, OpenSummary: "Agentenläufe: 2"); var supervisor = manager.Instances.SupervisorFor(path).ShouldNotBeNull(); for (var i = 0; i < 5; i++) await supervisor.CheckAsync(); supervisor.Condition.ShouldBe(InstanceCondition.Running); process.WasKilled.ShouldBeFalse(); _launcher.Starts.ShouldBe(1); } private static async Task WaitFor(Func condition) { for (var i = 0; i < 100 && !condition(); i++) await Task.Delay(20); } /// /// After the limit the manager pauses. /// /// What fails because of a broken configuration fails the same way on the tenth attempt /// — then instead of a dead instance you have a dead instance, a full /// log and a bug tracker full of repetitions. /// [Fact] public async Task After_three_restarts_in_the_time_window_the_manager_pauses() { var path = Instance("Wackelig", autostart: true); await using var manager = Manager(new RestartPolicy(MaxRestarts: 3, Window: TimeSpan.FromMinutes(10))); await manager.StartAsync(); var afterStart = _launcher.Starts; for (var i = 0; i < 6; i++) { _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Crash(path, "Wackelig")); } (_launcher.Starts - afterStart).ShouldBe(3, "Drei Versuche, dann Schluss."); } /// /// Whoever starts by hand has either changed something or knows what they are doing. Charging them with /// the earlier attempts would mean punishing them for the crash. /// [Fact] public async Task A_manual_start_clears_the_counter() { var path = Instance("Wackelig", autostart: true); await using var manager = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10))); await manager.StartAsync(); for (var i = 0; i < 5; i++) { _launcher.Last!.Exit(137); await manager.RestartAfterCrashAsync(Crash(path, "Wackelig")); } var exhausted = _launcher.Starts; // A new manager starts at zero - just like a start by hand. await using var fresh = Manager(new RestartPolicy(3, TimeSpan.FromMinutes(10))); await fresh.StartAsync(); _launcher.Last!.Exit(137); await fresh.RestartAfterCrashAsync(Crash(path, "Wackelig")); _launcher.Starts.ShouldBeGreaterThan(exhausted); } } /// The circuit breaker on its own. public sealed class RestartLedgerTests { private sealed class Clock(DateTimeOffset jetzt) : TimeProvider { public DateTimeOffset Jetzt { get; set; } = jetzt; public override DateTimeOffset GetUtcNow() => Jetzt; } [Fact] public void Yes_up_to_the_limit_no_after_it() { var book = new RestartLedger(new RestartPolicy(3, TimeSpan.FromMinutes(10))); book.TryRecord("/a", out _).ShouldBeTrue(); book.TryRecord("/a", out _).ShouldBeTrue(); book.TryRecord("/a", out var third).ShouldBeTrue(); third.ShouldBe(3); book.TryRecord("/a", out _).ShouldBeFalse(); } [Fact] public void Every_instance_has_its_own_account() { var book = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10))); book.TryRecord("/a", out _).ShouldBeTrue(); book.TryRecord("/a", out _).ShouldBeFalse(); book.TryRecord("/b", out _).ShouldBeTrue("Die eine Instanz sperrt die andere nicht."); } /// /// The time window moves: an instance that crashes once a month and comes back /// up never runs into the limit — one that falls every minute does immediately. /// [Fact] public void Old_attempts_fall_out_of_the_window() { var clock = new Clock(DateTimeOffset.UnixEpoch); var book = new RestartLedger(new RestartPolicy(2, TimeSpan.FromMinutes(10)), clock); book.TryRecord("/a", out _).ShouldBeTrue(); book.TryRecord("/a", out _).ShouldBeTrue(); book.TryRecord("/a", out _).ShouldBeFalse(); clock.Jetzt = clock.Jetzt.AddMinutes(11); book.TryRecord("/a", out var repeated).ShouldBeTrue(); repeated.ShouldBe(1, "Die alten Versuche zaehlen nicht mehr mit."); } [Fact] public void Forgetting_resets() { var book = new RestartLedger(new RestartPolicy(1, TimeSpan.FromMinutes(10))); book.TryRecord("/a", out _).ShouldBeTrue(); book.TryRecord("/a", out _).ShouldBeFalse(); book.Forget("/a"); book.TryRecord("/a", out _).ShouldBeTrue(); } } /// /// A start that meets an outdated note. /// /// The defect that only the autostart tests revealed. Between two /// supervision passes there are seconds, and in them an instance can break away. Whoever /// asked to "start" in this gap got "already running" — a success with nothing /// behind it. The next pass then reported a crash nobody expected any more. /// public sealed class StaleStateTests : IAsyncDisposable { private readonly string _root = Path.Combine( Path.GetTempPath(), "clawd-tests", "stale-" + Guid.NewGuid().ToString("N")[..8]); private readonly FakeLauncher _launcher = new(); private readonly FakeChannel _channel = new(); public StaleStateTests() { Directory.CreateDirectory(_root); _channel.Launcher = _launcher; _channel.Answer = new InstanceIdentity("i-1", "Alpha", "0.1.2"); } public async ValueTask DisposeAsync() { await Task.CompletedTask; try { Directory.Delete(_root, recursive: true); } catch { /* A leftover test folder does not turn a run red. */ } } [Fact] public async Task A_start_verifies_instead_of_trusting_its_own_state() { var dir = Path.Combine(_root, "Instance-Alpha"); Directory.CreateDirectory(dir); await using var supervisor = new InstanceSupervisor( dir, _launcher, _channel, NullLogger.Instance, new SupervisionOptions( TimeSpan.FromSeconds(30), // deliberately sluggish: no pass in between TimeSpan.FromMilliseconds(600), 3, TimeSpan.FromMilliseconds(300), TimeSpan.FromMilliseconds(300), TimeSpan.FromSeconds(2))); (await supervisor.StartAsync()).ShouldBeTrue(); supervisor.Condition.ShouldBe(InstanceCondition.Running); // The instance breaks away. Supervision has not noticed yet - the next // pass would only come in thirty seconds. _launcher.Last!.Exit(137); supervisor.Condition.ShouldBe(InstanceCondition.Running, "Noch nichts bemerkt."); (await supervisor.StartAsync()).ShouldBeTrue(); _launcher.Starts.ShouldBe(2, "Es wurde wirklich gestartet, nicht nur zugestimmt."); supervisor.Condition.ShouldBe(InstanceCondition.Running); } }