ClawdDotNet/src/ClawdDotNet.Core/Deploymentcenter/Watchdog/WatchdogClient.cs

using System.Text.Json;

namespace ClawdDotNet.Core.Deploymentcenter.Watchdog;

/// <summary>What the server reports back for a heartbeat.</summary>
/// <param name="State">The monitor state derived from it (<c>up</c>, <c>warning</c>, …).</param>
/// <param name="FailingChecks">Which of the checks sent along failed.</param>
public sealed record WatchdogPingResult(string State, IReadOnlyList<string> FailingChecks);

/// <summary>
/// Sends heartbeats and events to the Deploymentcenter's watchdog module.
/// </summary>
public interface IWatchdogClient
{
    Task<WatchdogPingResult> SendHeartbeatAsync(
        InstanceHealth health, int intervalSeconds, CancellationToken ct);

    Task SendEventAsync(
        string kind, string severity, string? message, object? meta, CancellationToken ct);
}

/// <summary>
/// Watchdog connection: <c>POST /api/watchdog/v1/ping</c> and
/// <c>POST /api/watchdog/v1/event</c>.
///
/// <para><b>One monitor per instance.</b> The key of the monitor is the pair
/// <c>source</c> + <c>instance</c> (per the database schema:
/// <c>UNIQUE KEY uq_monitor (source, instance)</c>). All ClawdDotNet instances report
/// under the same <c>source</c> and carry their own <c>instance</c> — so every
/// running instance is its own monitor with its own state, its own interval and
/// its own metrics history. If one of three crashes, exactly its monitor fails.</para>
///
/// <para>The monitor comes into being on the first heartbeat by itself (<c>INSERT … ON DUPLICATE
/// KEY UPDATE</c>) — there is no prior registration any more, and none is
/// needed.</para>
/// </summary>
public sealed class WatchdogClient : IWatchdogClient, IDisposable
{
    private readonly DeploymentcenterApi _api;
    private readonly bool _ownsApi;
    private readonly string _source;
    private readonly string _instance;
    private readonly string _group;
    private readonly string _os;
    private readonly string _version;

    public WatchdogClient(
        DeploymentcenterApi api, string source, string instance, string group, string os,
        string version, bool ownsApi = false)
    {
        _api = api;
        _ownsApi = ownsApi;
        _source = source;
        _instance = instance;
        _group = group;
        _os = os;
        _version = version;
    }

    public static WatchdogClient Create(
        string baseUrl, string token, string source, string instance, string group, string os,
        string version, HttpClient? httpClient = null)
        => new(new DeploymentcenterApi(baseUrl, token, httpClient),
               source, instance, group, os, version, ownsApi: true);

    public async Task<WatchdogPingResult> SendHeartbeatAsync(
        InstanceHealth health, int intervalSeconds, CancellationToken ct)
    {
        var payload = new Dictionary<string, object?>
        {
            ["source"] = _source,
            ["instance"] = _instance,
            ["type"] = "heartbeat",
            ["status"] = health.Status,
            ["interval"] = intervalSeconds,
            ["message"] = health.Message,
            ["group"] = _group,
            ["os"] = _os,

            // Ends up in watchdog_monitors.app_version. That way the dashboard shows which
            // version an instance is currently running — with several instances the
            // difference between "runs" and "still runs on the old version".
            ["version"] = _version
        };

        // Leave out empty objects: the server only takes over health_json if something
        // comes along — an empty one would not replace the last known state,
        // but would needlessly cost space in the log.
        if (health.Checks.Count > 0)
        {
            payload["checks"] = health.Checks.ToDictionary(
                c => c.Key,
                c => (object)new { ok = c.Value.Ok, message = c.Value.Message });
        }

        if (health.Metrics.Count > 0)
            payload["metrics"] = health.Metrics;

        var response = await _api.PostAsync("/api/watchdog/v1/ping", payload, ct)
            .ConfigureAwait(false);

        return ReadPingResult(response);
    }

    private static WatchdogPingResult ReadPingResult(JsonElement response)
    {
        if (!response.TryGetProperty("monitor", out var monitor)
            || monitor.ValueKind != JsonValueKind.Object)
        {
            return new WatchdogPingResult("unknown", []);
        }

        var state = monitor.TryGetProperty("state", out var s) && s.ValueKind == JsonValueKind.String
            ? s.GetString() ?? "unknown"
            : "unknown";

        var failing = new List<string>();
        if (monitor.TryGetProperty("failing_checks", out var checks)
            && checks.ValueKind == JsonValueKind.Array)
        {
            failing.AddRange(checks.EnumerateArray()
                .Where(e => e.ValueKind == JsonValueKind.String)
                .Select(e => e.GetString()!));
        }

        return new WatchdogPingResult(state, failing);
    }

    /// <summary>
    /// A one-time occurrence instead of a cyclic report. Permitted
    /// <paramref name="kind"/> values see <see cref="WatchdogEventKind"/> — the server
    /// rejects others.
    /// </summary>
    public async Task SendEventAsync(
        string kind, string severity, string? message, object? meta, CancellationToken ct)
    {
        var payload = new
        {
            source = _source,
            instance = _instance,
            kind,
            severity,
            message,
            meta
        };

        await _api.PostAsync("/api/watchdog/v1/event", payload, ct).ConfigureAwait(false);
    }

    public void Dispose()
    {
        if (_ownsApi)
            _api.Dispose();
    }
}

/// <summary>
/// The event kinds accepted by the server. The former connection sent
/// <c>start</c> and <c>stop</c> — neither is on this list, and they were
/// silently stored as <c>started</c>.
/// </summary>
public static class WatchdogEventKind
{
    public const string Started = "started";
    public const string StoppedGraceful = "stopped_graceful";
    public const string CrashSuspected = "crash_suspected";
    public const string HardError = "hard_error";
    public const string Recovered = "recovered";
    public const string WarningRaised = "warning_raised";
    public const string WarningCleared = "warning_cleared";
    public const string MaintenanceStart = "maintenance_start";
    public const string MaintenanceEnd = "maintenance_end";
    public const string WatchdogStarted = "watchdog_started";
}