using System.Text.Json; namespace ClawdDotNet.Core.Deploymentcenter.Watchdog; /// What the server reports back for a heartbeat. /// The monitor state derived from it (up, warning, …). /// Which of the checks sent along failed. public sealed record WatchdogPingResult(string State, IReadOnlyList FailingChecks); /// /// Sends heartbeats and events to the Deploymentcenter's watchdog module. /// public interface IWatchdogClient { Task SendHeartbeatAsync( InstanceHealth health, int intervalSeconds, CancellationToken ct); Task SendEventAsync( string kind, string severity, string? message, object? meta, CancellationToken ct); } /// /// Watchdog connection: POST /api/watchdog/v1/ping and /// POST /api/watchdog/v1/event. /// /// One monitor per instance. The key of the monitor is the pair /// source + instance (per the database schema: /// UNIQUE KEY uq_monitor (source, instance)). All ClawdDotNet instances report /// under the same source and carry their own instance — so every /// running instance is its own monitor with its own state, its own interval and /// its own metrics history. If one of three crashes, exactly its monitor fails. /// /// The monitor comes into being on the first heartbeat by itself (INSERT … ON DUPLICATE /// KEY UPDATE) — there is no prior registration any more, and none is /// needed. /// public sealed class WatchdogClient : IWatchdogClient, IDisposable { private readonly DeploymentcenterApi _api; private readonly bool _ownsApi; private readonly string _source; private readonly string _instance; private readonly string _group; private readonly string _os; private readonly string _version; public WatchdogClient( DeploymentcenterApi api, string source, string instance, string group, string os, string version, bool ownsApi = false) { _api = api; _ownsApi = ownsApi; _source = source; _instance = instance; _group = group; _os = os; _version = version; } public static WatchdogClient Create( string baseUrl, string token, string source, string instance, string group, string os, string version, HttpClient? httpClient = null) => new(new DeploymentcenterApi(baseUrl, token, httpClient), source, instance, group, os, version, ownsApi: true); public async Task SendHeartbeatAsync( InstanceHealth health, int intervalSeconds, CancellationToken ct) { var payload = new Dictionary { ["source"] = _source, ["instance"] = _instance, ["type"] = "heartbeat", ["status"] = health.Status, ["interval"] = intervalSeconds, ["message"] = health.Message, ["group"] = _group, ["os"] = _os, // Ends up in watchdog_monitors.app_version. That way the dashboard shows which // version an instance is currently running — with several instances the // difference between "runs" and "still runs on the old version". ["version"] = _version }; // Leave out empty objects: the server only takes over health_json if something // comes along — an empty one would not replace the last known state, // but would needlessly cost space in the log. if (health.Checks.Count > 0) { payload["checks"] = health.Checks.ToDictionary( c => c.Key, c => (object)new { ok = c.Value.Ok, message = c.Value.Message }); } if (health.Metrics.Count > 0) payload["metrics"] = health.Metrics; var response = await _api.PostAsync("/api/watchdog/v1/ping", payload, ct) .ConfigureAwait(false); return ReadPingResult(response); } private static WatchdogPingResult ReadPingResult(JsonElement response) { if (!response.TryGetProperty("monitor", out var monitor) || monitor.ValueKind != JsonValueKind.Object) { return new WatchdogPingResult("unknown", []); } var state = monitor.TryGetProperty("state", out var s) && s.ValueKind == JsonValueKind.String ? s.GetString() ?? "unknown" : "unknown"; var failing = new List(); if (monitor.TryGetProperty("failing_checks", out var checks) && checks.ValueKind == JsonValueKind.Array) { failing.AddRange(checks.EnumerateArray() .Where(e => e.ValueKind == JsonValueKind.String) .Select(e => e.GetString()!)); } return new WatchdogPingResult(state, failing); } /// /// A one-time occurrence instead of a cyclic report. Permitted /// values see — the server /// rejects others. /// public async Task SendEventAsync( string kind, string severity, string? message, object? meta, CancellationToken ct) { var payload = new { source = _source, instance = _instance, kind, severity, message, meta }; await _api.PostAsync("/api/watchdog/v1/event", payload, ct).ConfigureAwait(false); } public void Dispose() { if (_ownsApi) _api.Dispose(); } } /// /// The event kinds accepted by the server. The former connection sent /// start and stop — neither is on this list, and they were /// silently stored as started. /// public static class WatchdogEventKind { public const string Started = "started"; public const string StoppedGraceful = "stopped_graceful"; public const string CrashSuspected = "crash_suspected"; public const string HardError = "hard_error"; public const string Recovered = "recovered"; public const string WarningRaised = "warning_raised"; public const string WarningCleared = "warning_cleared"; public const string MaintenanceStart = "maintenance_start"; public const string MaintenanceEnd = "maintenance_end"; public const string WatchdogStarted = "watchdog_started"; }