using System.Net; using System.Text.Json; using System.Text.RegularExpressions; using System.Xml.Linq; using ClawdDotNet.Core.Tools; using Microsoft.Extensions.Logging; namespace ClawdDotNet.Tools.WebFetch; public sealed class WebFetchTool : IAgentTool { public string Name => "WebFetch"; public string Description => """ Ruft statische Webseiten oder RSS-Feeds ab und extrahiert Text + Timestamps. Nur Domains aus der Whitelist erlaubt. Kein JavaScript-Rendering. Aktionen: fetch, rss """; public JsonElement InputSchema { get; } = JsonDocument.Parse(""" { "type": "object", "required": ["action", "url"], "properties": { "action": { "type": "string", "enum": ["fetch", "rss"], "description": "fetch=HTML-Seite abrufen und zu Text konvertieren, rss=RSS/Atom-Feed parsen" }, "url": { "type": "string" }, "selector":{ "type": "string", "description": "optional: CSS-ähnlicher Hint welcher Teil relevant ist, z.B. 'table', 'article'" } } } """).RootElement.Clone(); public async Task ExecuteAsync( JsonElement input, AgentToolContext context, CancellationToken ct) { try { var url = input.GetProperty("url").GetString()!; var action = input.GetProperty("action").GetString()!; IReadOnlyDictionary? config = context.ToolConfig.TryGetValue("WebFetch", out var c) && c is JsonElement je ? JsonSerializer.Deserialize>(je.GetRawText()) : context.ToolConfig; if (config == null) return ToolResult.Fail("WebFetch Konfiguration fehlt."); var allowedDomains = config.GetValueOrDefault("allowedDomains") is JsonElement ad ? ad.EnumerateArray().Select(x => x.GetString()!).ToList() : (config.GetValueOrDefault("allowedDomains") as IEnumerable)?.ToList() ?? new List(); // Adresse prüfen: Schema, privates Netz, Whitelist if (!UrlGuard.TryValidate(url, allowedDomains, out _, out var urlError)) return ToolResult.Fail(urlError!); return action switch { "fetch" => await FetchPageAsync(url, allowedDomains, config, ct), "rss" => await FetchRssAsync(url, allowedDomains, ct), _ => ToolResult.Fail($"Unbekannte Aktion: {action}") }; } catch (Exception ex) { context.Logger.LogError(ex, "Fehler in WebFetch"); return ToolResult.Fail($"Fehler: {ex.Message}"); } } private async Task FetchPageAsync( string url, IReadOnlyCollection allowedDomains, IReadOnlyDictionary config, CancellationToken ct) { using var http = CreateHttpClient(config); var fetchedAt = DateTime.UtcNow; var (response, redirectError) = await GetFollowingRedirectsAsync(http, url, allowedDomains, ct); if (redirectError is not null) return redirectError; using var _ = response!; if (!response.IsSuccessStatusCode) return ToolResult.Fail($"Seite nicht erreichbar: {url} → HTTP {(int)response.StatusCode} {response.ReasonPhrase}"); DateTime? dataAsOf = response.Content.Headers.LastModified?.UtcDateTime ?? response.Headers.Date?.UtcDateTime; var html = await response.Content.ReadAsStringAsync(ct); var maxKb = GetConfigInt(config, "maxResponseKb", 512); if (html.Length > maxKb * 1024) html = html[..(maxKb * 1024)]; var text = StripHtml(html); if (dataAsOf == null) dataAsOf = ExtractDateFromHtml(html); var result = new { fetchedAt = fetchedAt, dataAsOf = dataAsOf, source = url, data = new { text } }; return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); } private async Task FetchRssAsync( string url, IReadOnlyCollection allowedDomains, CancellationToken ct) { using var http = CreateHttpClient(new Dictionary()); var fetchedAt = DateTime.UtcNow; var (response, redirectError) = await GetFollowingRedirectsAsync(http, url, allowedDomains, ct); if (redirectError is not null) return redirectError; using var _ = response!; if (!response.IsSuccessStatusCode) return ToolResult.Fail($"RSS-Feed nicht erreichbar: {url} → HTTP {(int)response.StatusCode} {response.ReasonPhrase}"); var xml = await response.Content.ReadAsStringAsync(ct); var doc = XDocument.Parse(xml); var items = new List(); // Support RSS and Atom var ns = doc.Root?.Name.Namespace; if (doc.Root?.Name.LocalName == "rss") { foreach (var item in doc.Descendants("item").Take(20)) { items.Add(new { title = item.Element("title")?.Value, link = item.Element("link")?.Value, pubDate = item.Element("pubDate")?.Value, description = StripHtml(item.Element("description")?.Value ?? "") }); } } else // Atom { foreach (var entry in doc.Descendants((ns ?? XNamespace.None) + "entry").Take(20)) { items.Add(new { title = entry.Element(ns + "title")?.Value, link = entry.Element(ns + "link")?.Attribute("href")?.Value, pubDate = entry.Element(ns + "updated")?.Value ?? entry.Element(ns + "published")?.Value, description = StripHtml(entry.Element(ns + "summary")?.Value ?? entry.Element(ns + "content")?.Value ?? "") }); } } var result = new { fetchedAt = fetchedAt, dataAsOf = fetchedAt, // RSS current state source = url, data = new { items } }; return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); } private static string StripHtml(string html) { if (string.IsNullOrWhiteSpace(html)) return ""; // Script und Style entfernen html = Regex.Replace(html, ".*?", "", RegexOptions.Singleline | RegexOptions.IgnoreCase); html = Regex.Replace(html, ".*?", "", RegexOptions.Singleline | RegexOptions.IgnoreCase); // Alle anderen Tags entfernen html = Regex.Replace(html, "<.*?>", " ", RegexOptions.Singleline); // Entities dekodieren html = WebUtility.HtmlDecode(html); // Whitespace säubern html = Regex.Replace(html, @"\s+", " "); html = html.Replace(" \n ", "\n").Replace(" \r ", "\r").Trim(); return html; } private static DateTime? ExtractDateFromHtml(string html) { // Suche nach meta-Tags var patterns = new[] { " /// Folgt Weiterleitungen selbst und prüft jedes Ziel erneut gegen Whitelist und /// Netzsperren. HttpClient würde sonst ungefragt bis ins lokale Netz folgen. /// private static async Task<(HttpResponseMessage? Response, ToolResult? Error)> GetFollowingRedirectsAsync( HttpClient http, string url, IReadOnlyCollection allowedDomains, CancellationToken ct) { var current = url; for (var hop = 0; hop <= MaxRedirects; hop++) { if (!UrlGuard.TryValidate(current, allowedDomains, out var uri, out var error)) { var context = hop == 0 ? "" : $" (Weiterleitung {hop} von {url})"; return (null, ToolResult.Fail($"{error}{context}")); } var response = await http.GetAsync(uri, ct); if (!IsRedirect(response.StatusCode)) return (response, null); var location = response.Headers.Location; response.Dispose(); if (location is null) return (null, ToolResult.Fail($"Weiterleitung ohne Zieladresse bei {current}.")); // Relative Weiterleitungen gegen die aktuelle Adresse auflösen. current = location.IsAbsoluteUri ? location.ToString() : new Uri(uri!, location).ToString(); } return (null, ToolResult.Fail( $"Mehr als {MaxRedirects} Weiterleitungen ausgehend von {url} — abgebrochen.")); } private static bool IsRedirect(HttpStatusCode status) => status is HttpStatusCode.MovedPermanently or HttpStatusCode.Found or HttpStatusCode.SeeOther or HttpStatusCode.TemporaryRedirect or HttpStatusCode.PermanentRedirect; private static HttpClient CreateHttpClient(IReadOnlyDictionary config) { // Weiterleitungen bewusst NICHT automatisch verfolgen — sie werden einzeln // aufgelöst und geprüft. var handler = new HttpClientHandler { AllowAutoRedirect = false }; var client = new HttpClient(handler, disposeHandler: true); var timeout = GetConfigInt(config, "timeoutSeconds", 15); client.Timeout = TimeSpan.FromSeconds(timeout); client.DefaultRequestHeaders.UserAgent.ParseAdd( config.GetValueOrDefault("userAgent")?.ToString() ?? "ClawdDotNet-Agent/1.0"); client.DefaultRequestHeaders.CacheControl = new System.Net.Http.Headers.CacheControlHeaderValue { NoCache = true, NoStore = true }; return client; } /// /// Sicheres Lesen eines int-Werts aus der Config (funktioniert mit JsonElement und primitiven Typen). /// private static int GetConfigInt(IReadOnlyDictionary config, string key, int defaultValue) { var val = config.GetValueOrDefault(key); return val switch { null => defaultValue, int i => i, long l => (int)l, System.Text.Json.JsonElement je when je.ValueKind == System.Text.Json.JsonValueKind.Number => je.GetInt32(), _ => int.TryParse(val.ToString(), out var parsed) ? parsed : defaultValue }; } }