Initial commit: ClawdDotNet
Import des bestehenden Projektstands in Git. - .NET 10 WinForms Anwendung (Multi-Agent / Tool-System) - .gitignore fuer Build-Artefakte, Secrets und Runtime-Daten ergaenzt Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Richard
@@ -0,0 +1,230 @@
|
||||
using System.Net;
|
||||
using System.Text.Json;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Xml.Linq;
|
||||
using ClawdDotNet.Core.Tools;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ClawdDotNet.Tools.WebFetch;
|
||||
|
||||
public sealed class WebFetchTool : IAgentTool
|
||||
{
|
||||
public string Name => "WebFetch";
|
||||
|
||||
public string Description => """
|
||||
Ruft statische Webseiten oder RSS-Feeds ab und extrahiert Text + Timestamps.
|
||||
Nur Domains aus der Whitelist erlaubt. Kein JavaScript-Rendering.
|
||||
Aktionen: fetch, rss
|
||||
""";
|
||||
|
||||
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
|
||||
{
|
||||
"type": "object",
|
||||
"required": ["action", "url"],
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["fetch", "rss"],
|
||||
"description": "fetch=HTML-Seite abrufen und zu Text konvertieren, rss=RSS/Atom-Feed parsen"
|
||||
},
|
||||
"url": { "type": "string" },
|
||||
"selector":{ "type": "string",
|
||||
"description": "optional: CSS-ähnlicher Hint welcher Teil relevant ist, z.B. 'table', 'article'" }
|
||||
}
|
||||
}
|
||||
""").RootElement.Clone();
|
||||
|
||||
public async Task<ToolResult> ExecuteAsync(
|
||||
JsonElement input,
|
||||
AgentToolContext context,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var url = input.GetProperty("url").GetString()!;
|
||||
var action = input.GetProperty("action").GetString()!;
|
||||
|
||||
IReadOnlyDictionary<string, object?>? config = context.ToolConfig.TryGetValue("WebFetch", out var c) && c is JsonElement je
|
||||
? JsonSerializer.Deserialize<Dictionary<string, object?>>(je.GetRawText())
|
||||
: context.ToolConfig;
|
||||
|
||||
if (config == null) return ToolResult.Fail("WebFetch Konfiguration fehlt.");
|
||||
|
||||
var allowedDomains = config.GetValueOrDefault("allowedDomains") is JsonElement ad
|
||||
? ad.EnumerateArray().Select(x => x.GetString()!).ToList()
|
||||
: (config.GetValueOrDefault("allowedDomains") as IEnumerable<string>)?.ToList() ?? new List<string>();
|
||||
|
||||
// Domain-Whitelist prüfen
|
||||
var uri = new Uri(url);
|
||||
var host = uri.Host.Replace("www.", "").ToLowerInvariant();
|
||||
if (!allowedDomains.Any(d => host == d.ToLowerInvariant() || host.EndsWith("." + d.ToLowerInvariant())))
|
||||
return ToolResult.Fail($"Domain '{host}' nicht in der Whitelist dieses Agenten.");
|
||||
|
||||
return action switch
|
||||
{
|
||||
"fetch" => await FetchPageAsync(url, config, ct),
|
||||
"rss" => await FetchRssAsync(url, ct),
|
||||
_ => ToolResult.Fail($"Unbekannte Aktion: {action}")
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
context.Logger.LogError(ex, "Fehler in WebFetch");
|
||||
return ToolResult.Fail($"Fehler: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<ToolResult> FetchPageAsync(string url, IReadOnlyDictionary<string, object?> config, CancellationToken ct)
|
||||
{
|
||||
using var http = CreateHttpClient(config);
|
||||
var fetchedAt = DateTime.UtcNow;
|
||||
|
||||
using var response = await http.GetAsync(url, ct);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return ToolResult.Fail($"Seite nicht erreichbar: {url} → HTTP {(int)response.StatusCode} {response.ReasonPhrase}");
|
||||
|
||||
DateTime? dataAsOf = response.Content.Headers.LastModified?.UtcDateTime
|
||||
?? response.Headers.Date?.UtcDateTime;
|
||||
|
||||
var html = await response.Content.ReadAsStringAsync(ct);
|
||||
var maxKb = GetConfigInt(config, "maxResponseKb", 512);
|
||||
if (html.Length > maxKb * 1024) html = html[..(maxKb * 1024)];
|
||||
|
||||
var text = StripHtml(html);
|
||||
if (dataAsOf == null) dataAsOf = ExtractDateFromHtml(html);
|
||||
|
||||
var result = new
|
||||
{
|
||||
fetchedAt = fetchedAt,
|
||||
dataAsOf = dataAsOf,
|
||||
source = url,
|
||||
data = new { text }
|
||||
};
|
||||
|
||||
return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
|
||||
}
|
||||
|
||||
private async Task<ToolResult> FetchRssAsync(string url, CancellationToken ct)
|
||||
{
|
||||
using var http = new HttpClient();
|
||||
http.DefaultRequestHeaders.UserAgent.ParseAdd("ClawdDotNet-Agent/1.0");
|
||||
http.Timeout = TimeSpan.FromSeconds(30);
|
||||
var fetchedAt = DateTime.UtcNow;
|
||||
|
||||
using var response = await http.GetAsync(url, ct);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
return ToolResult.Fail($"RSS-Feed nicht erreichbar: {url} → HTTP {(int)response.StatusCode} {response.ReasonPhrase}");
|
||||
|
||||
var xml = await response.Content.ReadAsStringAsync(ct);
|
||||
var doc = XDocument.Parse(xml);
|
||||
|
||||
var items = new List<object>();
|
||||
// Support RSS and Atom
|
||||
var ns = doc.Root?.Name.Namespace;
|
||||
|
||||
if (doc.Root?.Name.LocalName == "rss")
|
||||
{
|
||||
foreach (var item in doc.Descendants("item").Take(20))
|
||||
{
|
||||
items.Add(new
|
||||
{
|
||||
title = item.Element("title")?.Value,
|
||||
link = item.Element("link")?.Value,
|
||||
pubDate = item.Element("pubDate")?.Value,
|
||||
description = StripHtml(item.Element("description")?.Value ?? "")
|
||||
});
|
||||
}
|
||||
}
|
||||
else // Atom
|
||||
{
|
||||
foreach (var entry in doc.Descendants((ns ?? XNamespace.None) + "entry").Take(20))
|
||||
{
|
||||
items.Add(new
|
||||
{
|
||||
title = entry.Element(ns + "title")?.Value,
|
||||
link = entry.Element(ns + "link")?.Attribute("href")?.Value,
|
||||
pubDate = entry.Element(ns + "updated")?.Value ?? entry.Element(ns + "published")?.Value,
|
||||
description = StripHtml(entry.Element(ns + "summary")?.Value ?? entry.Element(ns + "content")?.Value ?? "")
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var result = new
|
||||
{
|
||||
fetchedAt = fetchedAt,
|
||||
dataAsOf = fetchedAt, // RSS current state
|
||||
source = url,
|
||||
data = new { items }
|
||||
};
|
||||
|
||||
return ToolResult.Ok(JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true }));
|
||||
}
|
||||
|
||||
private static string StripHtml(string html)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(html)) return "";
|
||||
|
||||
// Script und Style entfernen
|
||||
html = Regex.Replace(html, "<script.*?>.*?</script>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
|
||||
html = Regex.Replace(html, "<style.*?>.*?</style>", "", RegexOptions.Singleline | RegexOptions.IgnoreCase);
|
||||
|
||||
// Alle anderen Tags entfernen
|
||||
html = Regex.Replace(html, "<.*?>", " ", RegexOptions.Singleline);
|
||||
|
||||
// Entities dekodieren
|
||||
html = WebUtility.HtmlDecode(html);
|
||||
|
||||
// Whitespace säubern
|
||||
html = Regex.Replace(html, @"\s+", " ");
|
||||
html = html.Replace(" \n ", "\n").Replace(" \r ", "\r").Trim();
|
||||
|
||||
return html;
|
||||
}
|
||||
|
||||
private static DateTime? ExtractDateFromHtml(string html)
|
||||
{
|
||||
// Suche nach meta-Tags
|
||||
var patterns = new[]
|
||||
{
|
||||
"<meta.*?property=\"article:published_time\".*?content=\"(.*?)\"",
|
||||
"<meta.*?name=\"date\".*?content=\"(.*?)\"",
|
||||
"<time.*?datetime=\"(.*?)\""
|
||||
};
|
||||
|
||||
foreach (var pattern in patterns)
|
||||
{
|
||||
var match = Regex.Match(html, pattern, RegexOptions.IgnoreCase);
|
||||
if (match.Success && DateTime.TryParse(match.Groups[1].Value, out var dt))
|
||||
return dt.ToUniversalTime();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static HttpClient CreateHttpClient(IReadOnlyDictionary<string, object?> config)
|
||||
{
|
||||
var client = new HttpClient();
|
||||
var timeout = GetConfigInt(config, "timeoutSeconds", 15);
|
||||
client.Timeout = TimeSpan.FromSeconds(timeout);
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd(
|
||||
config.GetValueOrDefault("userAgent")?.ToString() ?? "ClawdDotNet-Agent/1.0");
|
||||
client.DefaultRequestHeaders.CacheControl = new System.Net.Http.Headers.CacheControlHeaderValue { NoCache = true, NoStore = true };
|
||||
return client;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sicheres Lesen eines int-Werts aus der Config (funktioniert mit JsonElement und primitiven Typen).
|
||||
/// </summary>
|
||||
private static int GetConfigInt(IReadOnlyDictionary<string, object?> config, string key, int defaultValue)
|
||||
{
|
||||
var val = config.GetValueOrDefault(key);
|
||||
return val switch
|
||||
{
|
||||
null => defaultValue,
|
||||
int i => i,
|
||||
long l => (int)l,
|
||||
System.Text.Json.JsonElement je when je.ValueKind == System.Text.Json.JsonValueKind.Number => je.GetInt32(),
|
||||
_ => int.TryParse(val.ToString(), out var parsed) ? parsed : defaultValue
|
||||
};
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user