using System.Text.Json;
namespace ClawdDotNet.Core.Deploymentcenter.Watchdog;
/// What the server reports back for a heartbeat.
/// The monitor state derived from it (up, warning, …).
/// Which of the checks sent along failed.
public sealed record WatchdogPingResult(string State, IReadOnlyList FailingChecks);
///
/// Sends heartbeats and events to the Deploymentcenter's watchdog module.
///
public interface IWatchdogClient
{
Task SendHeartbeatAsync(
InstanceHealth health, int intervalSeconds, CancellationToken ct);
Task SendEventAsync(
string kind, string severity, string? message, object? meta, CancellationToken ct);
}
///
/// Watchdog connection: POST /api/watchdog/v1/ping and
/// POST /api/watchdog/v1/event.
///
/// One monitor per instance. The key of the monitor is the pair
/// source + instance (per the database schema:
/// UNIQUE KEY uq_monitor (source, instance)). All ClawdDotNet instances report
/// under the same source and carry their own instance — so every
/// running instance is its own monitor with its own state, its own interval and
/// its own metrics history. If one of three crashes, exactly its monitor fails.
///
/// The monitor comes into being on the first heartbeat by itself (INSERT … ON DUPLICATE
/// KEY UPDATE) — there is no prior registration any more, and none is
/// needed.
///
public sealed class WatchdogClient : IWatchdogClient, IDisposable
{
private readonly DeploymentcenterApi _api;
private readonly bool _ownsApi;
private readonly string _source;
private readonly string _instance;
private readonly string _group;
private readonly string _os;
private readonly string _version;
public WatchdogClient(
DeploymentcenterApi api, string source, string instance, string group, string os,
string version, bool ownsApi = false)
{
_api = api;
_ownsApi = ownsApi;
_source = source;
_instance = instance;
_group = group;
_os = os;
_version = version;
}
public static WatchdogClient Create(
string baseUrl, string token, string source, string instance, string group, string os,
string version, HttpClient? httpClient = null)
=> new(new DeploymentcenterApi(baseUrl, token, httpClient),
source, instance, group, os, version, ownsApi: true);
public async Task SendHeartbeatAsync(
InstanceHealth health, int intervalSeconds, CancellationToken ct)
{
var payload = new Dictionary
{
["source"] = _source,
["instance"] = _instance,
["type"] = "heartbeat",
["status"] = health.Status,
["interval"] = intervalSeconds,
["message"] = health.Message,
["group"] = _group,
["os"] = _os,
// Ends up in watchdog_monitors.app_version. That way the dashboard shows which
// version an instance is currently running — with several instances the
// difference between "runs" and "still runs on the old version".
["version"] = _version
};
// Leave out empty objects: the server only takes over health_json if something
// comes along — an empty one would not replace the last known state,
// but would needlessly cost space in the log.
if (health.Checks.Count > 0)
{
payload["checks"] = health.Checks.ToDictionary(
c => c.Key,
c => (object)new { ok = c.Value.Ok, message = c.Value.Message });
}
if (health.Metrics.Count > 0)
payload["metrics"] = health.Metrics;
var response = await _api.PostAsync("/api/watchdog/v1/ping", payload, ct)
.ConfigureAwait(false);
return ReadPingResult(response);
}
private static WatchdogPingResult ReadPingResult(JsonElement response)
{
if (!response.TryGetProperty("monitor", out var monitor)
|| monitor.ValueKind != JsonValueKind.Object)
{
return new WatchdogPingResult("unknown", []);
}
var state = monitor.TryGetProperty("state", out var s) && s.ValueKind == JsonValueKind.String
? s.GetString() ?? "unknown"
: "unknown";
var failing = new List();
if (monitor.TryGetProperty("failing_checks", out var checks)
&& checks.ValueKind == JsonValueKind.Array)
{
failing.AddRange(checks.EnumerateArray()
.Where(e => e.ValueKind == JsonValueKind.String)
.Select(e => e.GetString()!));
}
return new WatchdogPingResult(state, failing);
}
///
/// A one-time occurrence instead of a cyclic report. Permitted
/// values see — the server
/// rejects others.
///
public async Task SendEventAsync(
string kind, string severity, string? message, object? meta, CancellationToken ct)
{
var payload = new
{
source = _source,
instance = _instance,
kind,
severity,
message,
meta
};
await _api.PostAsync("/api/watchdog/v1/event", payload, ct).ConfigureAwait(false);
}
public void Dispose()
{
if (_ownsApi)
_api.Dispose();
}
}
///
/// The event kinds accepted by the server. The former connection sent
/// start and stop — neither is on this list, and they were
/// silently stored as started.
///
public static class WatchdogEventKind
{
public const string Started = "started";
public const string StoppedGraceful = "stopped_graceful";
public const string CrashSuspected = "crash_suspected";
public const string HardError = "hard_error";
public const string Recovered = "recovered";
public const string WarningRaised = "warning_raised";
public const string WarningCleared = "warning_cleared";
public const string MaintenanceStart = "maintenance_start";
public const string MaintenanceEnd = "maintenance_end";
public const string WatchdogStarted = "watchdog_started";
}