|
|
|
@@ -0,0 +1,204 @@
|
|
|
|
|
using System.Text.Json;
|
|
|
|
|
using System.Text.RegularExpressions;
|
|
|
|
|
using ClawdDotNet.Core.Tools;
|
|
|
|
|
using Microsoft.Extensions.Logging;
|
|
|
|
|
|
|
|
|
|
namespace ClawdDotNet.Tools.WebMonitor;
|
|
|
|
|
|
|
|
|
|
public sealed class WebMonitorTool : IAgentTool
|
|
|
|
|
{
|
|
|
|
|
public string Name => "WebMonitor";
|
|
|
|
|
|
|
|
|
|
public string Description => """
|
|
|
|
|
Überwacht Webseiten auf neue Einträge und liefert nur die Deltas seit dem letzten Check.
|
|
|
|
|
Speichert den Stand in der Datenbank. Ideal für Capitol Trades, SEC-Filings, etc.
|
|
|
|
|
Aktionen: check, history, status
|
|
|
|
|
""";
|
|
|
|
|
|
|
|
|
|
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
|
|
|
|
|
{
|
|
|
|
|
"type": "object",
|
|
|
|
|
"required": ["action", "monitorId"],
|
|
|
|
|
"properties": {
|
|
|
|
|
"action": {
|
|
|
|
|
"type": "string",
|
|
|
|
|
"enum": ["check", "history", "status"],
|
|
|
|
|
"description": "check=jetzt prüfen und Deltas liefern, history=bisherige Einträge, status=letzter Check-Zeitpunkt"
|
|
|
|
|
},
|
|
|
|
|
"monitorId": {
|
|
|
|
|
"type": "string",
|
|
|
|
|
"description": "z.B. capitol_trades, sec_filings — muss in Config definiert sein"
|
|
|
|
|
},
|
|
|
|
|
"limit": {
|
|
|
|
|
"type": "integer",
|
|
|
|
|
"description": "max. Anzahl Einträge für history, default 50"
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
""").RootElement.Clone();
|
|
|
|
|
|
|
|
|
|
public async Task<ToolResult> ExecuteAsync(
|
|
|
|
|
JsonElement input,
|
|
|
|
|
AgentToolContext context,
|
|
|
|
|
CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var action = input.GetProperty("action").GetString()!;
|
|
|
|
|
var monitorId = input.GetProperty("monitorId").GetString()!;
|
|
|
|
|
|
|
|
|
|
IReadOnlyDictionary<string, object?>? config = context.ToolConfig.TryGetValue("WebMonitor", out var c) && c is JsonElement je
|
|
|
|
|
? JsonSerializer.Deserialize<Dictionary<string, object?>>(je.GetRawText())
|
|
|
|
|
: context.ToolConfig;
|
|
|
|
|
|
|
|
|
|
if (config == null || !config.TryGetValue("monitors", out var m) || m is not JsonElement monitorsJe)
|
|
|
|
|
return ToolResult.Fail("WebMonitor Konfiguration oder Monitore fehlen.");
|
|
|
|
|
|
|
|
|
|
var monitors = JsonSerializer.Deserialize<Dictionary<string, JsonElement>>(monitorsJe.GetRawText());
|
|
|
|
|
if (monitors == null || !monitors.TryGetValue(monitorId, out var monitorConfigJe))
|
|
|
|
|
return ToolResult.Fail($"Monitor '{monitorId}' nicht in der Config gefunden.");
|
|
|
|
|
|
|
|
|
|
var monitorConfig = JsonSerializer.Deserialize<Dictionary<string, object?>>(monitorConfigJe.GetRawText())!;
|
|
|
|
|
|
|
|
|
|
return action switch
|
|
|
|
|
{
|
|
|
|
|
"check" => await CheckForNewEntriesAsync(monitorId, monitorConfig, context, ct),
|
|
|
|
|
"history" => await GetHistoryAsync(monitorId, input, context, ct),
|
|
|
|
|
"status" => await GetStatusAsync(monitorId, context, ct),
|
|
|
|
|
_ => ToolResult.Fail($"Unbekannte Aktion: {action}")
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
|
|
|
|
context.Logger.LogError(ex, "Fehler in WebMonitor");
|
|
|
|
|
return ToolResult.Fail($"Fehler: {ex.Message}");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async Task<ToolResult> CheckForNewEntriesAsync(string monitorId, Dictionary<string, object?> monitorConfig, AgentToolContext context, CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
var url = monitorConfig["url"]?.ToString()!;
|
|
|
|
|
var parser = monitorConfig["parser"]?.ToString() ?? "generic";
|
|
|
|
|
var idPattern = monitorConfig["idPattern"]?.ToString();
|
|
|
|
|
|
|
|
|
|
using var http = new HttpClient();
|
|
|
|
|
http.DefaultRequestHeaders.UserAgent.ParseAdd("ClawdDotNet-Agent/1.0");
|
|
|
|
|
http.DefaultRequestHeaders.CacheControl = new System.Net.Http.Headers.CacheControlHeaderValue { NoCache = true };
|
|
|
|
|
|
|
|
|
|
var html = await http.GetStringAsync(url, ct);
|
|
|
|
|
var fetchedAt = DateTime.UtcNow;
|
|
|
|
|
|
|
|
|
|
var entries = parser switch
|
|
|
|
|
{
|
|
|
|
|
"capitol_trades" => ParseCapitolTrades(html),
|
|
|
|
|
_ => ParseGeneric(html, idPattern)
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
var stateKey = $"webmonitor:{context.AgentId}:{monitorId}:maxId";
|
|
|
|
|
var lastMaxIdStr = await context.StateStore.GetAsync(stateKey, ct);
|
|
|
|
|
var lastKnown = long.TryParse(lastMaxIdStr, out var l) ? l : 0L;
|
|
|
|
|
|
|
|
|
|
var newEntries = entries
|
|
|
|
|
.Where(e => e.NumericId > lastKnown)
|
|
|
|
|
.OrderBy(e => e.NumericId)
|
|
|
|
|
.ToList();
|
|
|
|
|
|
|
|
|
|
if (newEntries.Count > 0)
|
|
|
|
|
{
|
|
|
|
|
var newMax = newEntries.Max(e => e.NumericId).ToString();
|
|
|
|
|
await context.StateStore.SetAsync(stateKey, newMax, ct);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
await context.StateStore.SetAsync($"webmonitor:{context.AgentId}:{monitorId}:lastCheck", fetchedAt.ToString("O"), ct);
|
|
|
|
|
|
|
|
|
|
var result = new
|
|
|
|
|
{
|
|
|
|
|
fetchedAt = fetchedAt,
|
|
|
|
|
dataAsOf = fetchedAt,
|
|
|
|
|
source = url,
|
|
|
|
|
monitorId = monitorId,
|
|
|
|
|
newCount = newEntries.Count,
|
|
|
|
|
data = new
|
|
|
|
|
{
|
|
|
|
|
newEntries = newEntries,
|
|
|
|
|
message = newEntries.Count == 0 ? "Keine neuen Einträge." : $"{newEntries.Count} neue Einträge gefunden."
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static List<MonitorEntry> ParseCapitolTrades(string html)
|
|
|
|
|
{
|
|
|
|
|
var entries = new List<MonitorEntry>();
|
|
|
|
|
|
|
|
|
|
// Suche nach Trade-IDs und URLs
|
|
|
|
|
// Format: <a href="/trades/20003797558">
|
|
|
|
|
var idMatches = Regex.Matches(html, "/trades/(\\d+)");
|
|
|
|
|
|
|
|
|
|
foreach (Match match in idMatches)
|
|
|
|
|
{
|
|
|
|
|
if (long.TryParse(match.Groups[1].Value, out var id))
|
|
|
|
|
{
|
|
|
|
|
if (entries.Any(e => e.NumericId == id)) continue;
|
|
|
|
|
|
|
|
|
|
entries.Add(new MonitorEntry(
|
|
|
|
|
id,
|
|
|
|
|
id.ToString(),
|
|
|
|
|
$"https://www.capitoltrades.com/trades/{id}",
|
|
|
|
|
null,
|
|
|
|
|
null,
|
|
|
|
|
new Dictionary<string, string> { { "type", "CapitolTrades" } }
|
|
|
|
|
));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return entries;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private static List<MonitorEntry> ParseGeneric(string html, string? idPattern)
|
|
|
|
|
{
|
|
|
|
|
if (string.IsNullOrEmpty(idPattern)) return new List<MonitorEntry>();
|
|
|
|
|
|
|
|
|
|
var entries = new List<MonitorEntry>();
|
|
|
|
|
var matches = Regex.Matches(html, idPattern);
|
|
|
|
|
|
|
|
|
|
foreach (Match match in matches)
|
|
|
|
|
{
|
|
|
|
|
var idStr = match.Groups.Count > 1 ? match.Groups[1].Value : match.Value;
|
|
|
|
|
if (long.TryParse(Regex.Replace(idStr, "[^0-9]", ""), out var id))
|
|
|
|
|
{
|
|
|
|
|
entries.Add(new MonitorEntry(id, idStr, "", null, null, new Dictionary<string, string>()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return entries;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async Task<ToolResult> GetHistoryAsync(string monitorId, JsonElement input, AgentToolContext context, CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
// Simple history from StateStore or just message for now as we don't have a separate DB table for this yet
|
|
|
|
|
return ToolResult.Ok(JsonSerializer.Serialize(new { message = "History-Feature ist über den StateStore aktuell auf die letzte ID begrenzt." }));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private async Task<ToolResult> GetStatusAsync(string monitorId, AgentToolContext context, CancellationToken ct)
|
|
|
|
|
{
|
|
|
|
|
var lastCheck = await context.StateStore.GetAsync($"webmonitor:{context.AgentId}:{monitorId}:lastCheck", ct);
|
|
|
|
|
var maxId = await context.StateStore.GetAsync($"webmonitor:{context.AgentId}:{monitorId}:maxId", ct);
|
|
|
|
|
|
|
|
|
|
return ToolResult.Ok(JsonSerializer.Serialize(new
|
|
|
|
|
{
|
|
|
|
|
monitorId = monitorId,
|
|
|
|
|
lastCheck = lastCheck,
|
|
|
|
|
maxId = maxId
|
|
|
|
|
}, new JsonSerializerOptions { WriteIndented = true }));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public sealed record MonitorEntry(
|
|
|
|
|
long NumericId,
|
|
|
|
|
string RawId,
|
|
|
|
|
string DetailUrl,
|
|
|
|
|
DateTime? PublishedAt,
|
|
|
|
|
DateTime? TradedAt,
|
|
|
|
|
Dictionary<string, string> Fields
|
|
|
|
|
);
|