RichardandClaude Opus 5.5
257ba87cf5
Jede Codeausfuehrung ausserhalb des Hauptprozesses: eigene Namensraeume (Mount, PID, Netz nur loopback, User ohne verschachtelte), nur der Arbeitsbereich unter /work beschreibbar, scripts/ darin schreibgeschuetzt, keine Umgebung und keine Deskriptoren des Hauptprozesses, alle Capabilities weg, no_new_privs, seccomp-Sperrliste (eigene BPF, x86_64 und arm64), Grenzen fuer Zeit, Speicher, Prozesse, Dateigroesse und Ausgabe. Unter Windows oder ohne bwrap: keine Ausfuehrung, mit Grund im Protokoll. Erster Nutzer: ffmpeg im SocialMediaManager (fremde Mediendateien). yt-dlp braucht Netz und folgt mit dem Egress-Proxy (#655). Die Ausbruchstests (SandboxEscapeTests) laufen nur unter Linux mit bwrap. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1294 lines
59 KiB
C#
1294 lines
59 KiB
C#
using System.Diagnostics;
|
||
using System.Net.Http.Headers;
|
||
using System.Text;
|
||
using System.Text.Json;
|
||
using ClawdDotNet.Core.State;
|
||
using ClawdDotNet.Core.Tools;
|
||
using Microsoft.Extensions.Logging;
|
||
|
||
namespace ClawdDotNet.Tools.SocialMediaManager;
|
||
|
||
public sealed class SocialMediaManagerTool : IAgentTool, IToolJobProvider, ClawdDotNet.Core.Sandbox.ISandboxUser
|
||
{
|
||
public string Name => "SocialMediaManager";
|
||
|
||
/// <summary>#715: Brings text from outside into the context — posts and comments of the public.</summary>
|
||
public bool ReadsForeignContent => true;
|
||
|
||
public string Description => """
|
||
Manager for Social Media Monitoring (X/Twitter, Reddit) and YouTube Transcripts.
|
||
|
||
Actions:
|
||
- x_search: Free-text search on X/Twitter
|
||
- x_user_tweets: Retrieve tweets from a specific user (specify username)
|
||
- reddit_search: Free-text search on Reddit
|
||
- reddit_subreddit: Retrieve new posts from a specific subreddit (specify subreddit)
|
||
- youtube_transcript: Transcribe a YouTube video
|
||
|
||
Background jobs (configured in toolJobs):
|
||
- sm_x_account_monitor: Monitors configured X accounts for new tweets (Config: xWatchAccounts)
|
||
- sm_reddit_monitor: Monitors configured subreddits for new posts (Config: redditWatchSubreddits)
|
||
""";
|
||
|
||
public JsonElement InputSchema { get; } = JsonDocument.Parse("""
|
||
{
|
||
"type": "object",
|
||
"properties": {
|
||
"action": {
|
||
"type": "string",
|
||
"enum": ["x_search", "x_user_tweets", "reddit_search", "reddit_subreddit", "youtube_transcript"],
|
||
"description": "The action to execute"
|
||
},
|
||
"query": {
|
||
"type": "string",
|
||
"description": "Search term for x_search or reddit_search"
|
||
},
|
||
"username": {
|
||
"type": "string",
|
||
"description": "X/Twitter username without @ (only for x_user_tweets)"
|
||
},
|
||
"subreddit": {
|
||
"type": "string",
|
||
"description": "Subreddit name without r/ (only for reddit_subreddit)"
|
||
},
|
||
"url": {
|
||
"type": "string",
|
||
"description": "YouTube video URL or video ID (only for youtube_transcript)"
|
||
},
|
||
"limit": {
|
||
"type": "integer",
|
||
"description": "Maximum number of results (default: 10)"
|
||
}
|
||
},
|
||
"required": ["action"]
|
||
}
|
||
""").RootElement.Clone();
|
||
|
||
private readonly HttpClient _httpClient = new();
|
||
|
||
public async Task<ToolResult> ExecuteAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||
{
|
||
var action = input.GetProperty("action").GetString() ?? throw new ArgumentException("Action is required");
|
||
|
||
// Save per-agent scoped WorkspacePath in StateStore for fallback
|
||
await context.StateStore.SetAsync($"sm_workspace_path:{context.AgentId}", context.WorkspacePath!, ct);
|
||
|
||
try
|
||
{
|
||
return action switch
|
||
{
|
||
"x_search" => await HandleXSearchAsync(input, context, ct),
|
||
"x_user_tweets" => await HandleXUserTweetsAsync(input, context, ct),
|
||
"reddit_search" => await HandleRedditSearchAsync(input, context, ct),
|
||
"reddit_subreddit" => await HandleRedditSubredditAsync(input, context, ct),
|
||
"youtube_transcript" => await HandleYoutubeTranscriptAsync(input, context, ct),
|
||
_ => ToolResult.Fail($"Unknown action: {action}")
|
||
};
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
context.Logger.LogError(ex, "Error in SocialMediaManager ExecuteAsync ({Action})", action);
|
||
return ToolResult.Fail($"Error: {ex.Message}");
|
||
}
|
||
}
|
||
|
||
private async Task<ToolResult> HandleXSearchAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||
{
|
||
var query = input.TryGetProperty("query", out var q) ? q.GetString() : "";
|
||
var limit = input.TryGetProperty("limit", out var l) ? l.GetInt32() : 10;
|
||
|
||
var apiKey = context.ToolConfig.GetValueOrDefault("xApiKey")?.ToString();
|
||
if (string.IsNullOrWhiteSpace(apiKey)) return ToolResult.Fail("X API key missing.");
|
||
|
||
using var request = new HttpRequestMessage(HttpMethod.Get, $"https://api.twitter.com/2/tweets/search/recent?query={Uri.EscapeDataString(query ?? "")}&max_results={limit}");
|
||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var content = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode) return ToolResult.Fail($"X API error: {response.StatusCode} - {content}");
|
||
|
||
return ToolResult.Ok(content);
|
||
}
|
||
|
||
private async Task<ToolResult> HandleRedditSearchAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||
{
|
||
var query = input.TryGetProperty("query", out var q) ? q.GetString() : "";
|
||
var limit = input.TryGetProperty("limit", out var l) ? l.GetInt32() : 10;
|
||
|
||
var url = $"https://www.reddit.com/search.json?q={Uri.EscapeDataString(query ?? "")}&limit={limit}";
|
||
|
||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||
request.Headers.UserAgent.ParseAdd("ClawdDotNet/1.0");
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var content = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode) return ToolResult.Fail($"Reddit API error: {response.StatusCode}");
|
||
|
||
return ToolResult.Ok(content);
|
||
}
|
||
|
||
private async Task<ToolResult> HandleXUserTweetsAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||
{
|
||
var username = input.TryGetProperty("username", out var u) ? u.GetString() : null;
|
||
|
||
if (string.IsNullOrWhiteSpace(username))
|
||
return ToolResult.Fail("'username' is required (without @).");
|
||
|
||
var cleanName = username.TrimStart('@').ToLowerInvariant();
|
||
|
||
// 1. Lokalen Cache prüfen — bereits abgerufene Tweets zurückgeben ohne API-Call
|
||
var cacheDir = Path.Combine(context.WorkspacePath!, "XMonitor");
|
||
if (Directory.Exists(cacheDir))
|
||
{
|
||
var cachedFiles = Directory.GetFiles(cacheDir, "*.json")
|
||
.OrderByDescending(f => f)
|
||
.Take(5); // Letzte 5 Cache-Dateien durchsuchen
|
||
|
||
var cachedTweets = new List<JsonElement>();
|
||
foreach (var file in cachedFiles)
|
||
{
|
||
try
|
||
{
|
||
var json = await File.ReadAllTextAsync(file, ct);
|
||
using var doc = JsonDocument.Parse(json);
|
||
foreach (var tweet in doc.RootElement.EnumerateArray())
|
||
{
|
||
if (tweet.TryGetProperty("Account", out var acc) &&
|
||
acc.GetString()?.TrimStart('@').Equals(cleanName, StringComparison.OrdinalIgnoreCase) == true)
|
||
{
|
||
cachedTweets.Add(tweet.Clone());
|
||
}
|
||
}
|
||
}
|
||
catch { /* fehlerhafte Datei ignorieren */ }
|
||
}
|
||
|
||
if (cachedTweets.Count > 0)
|
||
{
|
||
return ToolResult.Ok(
|
||
$"[From local cache — no API call needed, {cachedTweets.Count} tweets]\n" +
|
||
JsonSerializer.Serialize(cachedTweets, new JsonSerializerOptions { WriteIndented = true }));
|
||
}
|
||
}
|
||
|
||
// 2. Kein Cache vorhanden → Einmalige Abfrage über die Search API (1 API-Call)
|
||
var apiKey = context.ToolConfig.GetValueOrDefault("xApiKey")?.ToString();
|
||
if (string.IsNullOrWhiteSpace(apiKey)) return ToolResult.Fail("X API key (xApiKey) missing in tool config.");
|
||
|
||
var limit = input.TryGetProperty("limit", out var l) ? l.GetInt32() : 10;
|
||
var searchUrl = $"https://api.twitter.com/2/tweets/search/recent" +
|
||
$"?query=from:{Uri.EscapeDataString(cleanName)}" +
|
||
$"&max_results={Math.Clamp(limit, 10, 100)}" +
|
||
$"&tweet.fields=created_at,public_metrics,author_id";
|
||
|
||
using var request = new HttpRequestMessage(HttpMethod.Get, searchUrl);
|
||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var content = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode)
|
||
return ToolResult.Fail($"X API error: {response.StatusCode} - {content}");
|
||
|
||
return ToolResult.Ok($"[1 API call used]\n{content}");
|
||
}
|
||
|
||
private async Task<ToolResult> HandleRedditSubredditAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||
{
|
||
var subreddit = input.TryGetProperty("subreddit", out var s) ? s.GetString() : null;
|
||
var limit = input.TryGetProperty("limit", out var l) ? l.GetInt32() : 10;
|
||
|
||
if (string.IsNullOrWhiteSpace(subreddit))
|
||
return ToolResult.Fail("'subreddit' is required (without r/).");
|
||
|
||
var url = $"https://www.reddit.com/r/{Uri.EscapeDataString(subreddit)}/new.json?limit={limit}";
|
||
|
||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||
request.Headers.UserAgent.ParseAdd("ClawdDotNet/1.0");
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var content = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode)
|
||
return ToolResult.Fail($"Reddit API error: {response.StatusCode} - {content}");
|
||
|
||
// Kompakte Darstellung: Nur relevante Felder extrahieren
|
||
var posts = ExtractRedditPosts(content);
|
||
return ToolResult.Ok(JsonSerializer.Serialize(posts, new JsonSerializerOptions { WriteIndented = true }));
|
||
}
|
||
|
||
private async Task<ToolResult> HandleYoutubeTranscriptAsync(JsonElement input, AgentToolContext context, CancellationToken ct)
|
||
{
|
||
var url = input.TryGetProperty("url", out var u) ? u.GetString() : "";
|
||
if (string.IsNullOrWhiteSpace(url)) return ToolResult.Fail("URL missing.");
|
||
|
||
var (available, path, hasFfmpeg, ffmpegPath) = await VerifyYtDlpAsync();
|
||
if (!available) return ToolResult.Fail(GetYtDlpInstallationGuide());
|
||
|
||
var targetDir = Path.Combine(context.WorkspacePath!, "YTTranscript");
|
||
if (!Directory.Exists(targetDir)) Directory.CreateDirectory(targetDir);
|
||
|
||
// Auftragskontext speichern damit der Agent nach dem Wake weiß was zu tun ist
|
||
await SavePendingTaskAsync(context, url, ct);
|
||
|
||
// Wir starten den Prozess im Hintergrund
|
||
_ = ProcessYoutubeVideoAsync(url, context.ToolConfig, context.WorkspacePath!, context.Logger, path!, hasFfmpeg, ffmpegPath, ct);
|
||
|
||
return ToolResult.Ok($"Transcription for {url} has been started and is running in the background. " +
|
||
"Downloading, splitting, and transcribing may take 2–10 minutes depending on video length. " +
|
||
"You will be automatically notified once the transcript is ready (Transcript Notifier). " +
|
||
"Tell the user that transcription has been started and they will receive the result in a few minutes.");
|
||
}
|
||
|
||
private async Task SavePendingTaskAsync(AgentToolContext context, string url, CancellationToken ct)
|
||
{
|
||
var pendingDir = Path.Combine(context.WorkspacePath!, "YTTranscript");
|
||
if (!Directory.Exists(pendingDir)) Directory.CreateDirectory(pendingDir);
|
||
|
||
var pendingFile = Path.Combine(pendingDir, "pending_tasks.json");
|
||
|
||
List<PendingTranscriptTask> tasks;
|
||
if (File.Exists(pendingFile))
|
||
{
|
||
var existingJson = await File.ReadAllTextAsync(pendingFile, ct);
|
||
tasks = JsonSerializer.Deserialize<List<PendingTranscriptTask>>(existingJson) ?? new();
|
||
}
|
||
else
|
||
{
|
||
tasks = new();
|
||
}
|
||
|
||
tasks.Add(new PendingTranscriptTask
|
||
{
|
||
Url = url,
|
||
RequestedBy = context.AgentId,
|
||
RequestedAt = DateTime.UtcNow,
|
||
Completed = false
|
||
});
|
||
|
||
await File.WriteAllTextAsync(pendingFile, JsonSerializer.Serialize(tasks, new JsonSerializerOptions { WriteIndented = true }), ct);
|
||
}
|
||
|
||
private sealed class PendingTranscriptTask
|
||
{
|
||
public string Url { get; set; } = "";
|
||
public string RequestedBy { get; set; } = "";
|
||
public DateTime RequestedAt { get; set; }
|
||
public bool Completed { get; set; }
|
||
}
|
||
|
||
// --- IToolJobProvider ---
|
||
|
||
public IReadOnlyList<ToolJobDefinition> GetJobDefinitions() =>
|
||
[
|
||
new("sm_yt_monitor", "YouTube Monitor", "Checks channels for new videos and starts transcription"),
|
||
new("sm_transcript_notifier", "Transcript Notifier", "Informs the agent about new transcripts"),
|
||
new("sm_x_account_monitor", "X Account Monitor", "Monitors configured X accounts for new tweets (Config: xWatchAccounts)"),
|
||
new("sm_reddit_monitor", "Reddit Monitor", "Monitors configured subreddits for new posts (Config: redditWatchSubreddits)")
|
||
];
|
||
|
||
public async Task<ToolJobResult> ExecuteJobAsync(string jobTypeId, IReadOnlyDictionary<string, object?> toolConfig, IStateStore stateStore, ILogger logger, CancellationToken ct, string? agentId = null, string? workspacePath = null)
|
||
{
|
||
// Use directly passed path, fallback to per-agent scoped StateStore
|
||
if (string.IsNullOrWhiteSpace(workspacePath) && !string.IsNullOrWhiteSpace(agentId))
|
||
workspacePath = await stateStore.GetAsync($"sm_workspace_path:{agentId}", ct);
|
||
|
||
if (string.IsNullOrWhiteSpace(workspacePath))
|
||
{
|
||
return ToolJobResult.NoAction("WorkspacePath unknown. Please run tool manually once.");
|
||
}
|
||
|
||
return jobTypeId switch
|
||
{
|
||
"sm_yt_monitor" => await ExecuteYoutubeMonitorJobAsync(toolConfig, workspacePath, stateStore, logger, ct),
|
||
"sm_transcript_notifier" => await ExecuteTranscriptNotifierJobAsync(toolConfig, workspacePath, stateStore, logger, ct),
|
||
"sm_x_account_monitor" => await ExecuteXAccountMonitorJobAsync(toolConfig, workspacePath, stateStore, logger, ct),
|
||
"sm_reddit_monitor" => await ExecuteRedditMonitorJobAsync(toolConfig, workspacePath, stateStore, logger, ct),
|
||
_ => ToolJobResult.NoAction($"Unknown job: {jobTypeId}")
|
||
};
|
||
}
|
||
|
||
private async Task<ToolJobResult> ExecuteYoutubeMonitorJobAsync(IReadOnlyDictionary<string, object?> toolConfig, string workspacePath, IStateStore stateStore, ILogger logger, CancellationToken ct)
|
||
{
|
||
var (available, path, hasFfmpeg, ffmpegPath) = await VerifyYtDlpAsync();
|
||
if (!available) return ToolJobResult.NoAction("YouTube Monitor disabled: " + GetYtDlpInstallationGuide());
|
||
|
||
var channels = GetConfigArray(toolConfig, "youtubeChannels").ToList();
|
||
if (channels.Count == 0) return ToolJobResult.NoAction("No channels configured.");
|
||
|
||
var lastVidsJson = await stateStore.GetAsync("sm_yt_last_vids", ct) ?? "{}";
|
||
var lastVids = JsonSerializer.Deserialize<Dictionary<string, string>>(lastVidsJson) ?? new();
|
||
|
||
int count = 0;
|
||
foreach (var channelInput in channels)
|
||
{
|
||
if (channelInput == null) continue;
|
||
try
|
||
{
|
||
// Normalized key for state store
|
||
var stateKey = channelInput.Trim().TrimStart('@').ToLowerInvariant();
|
||
var latestId = await GetLatestVideoIdAsync(channelInput, path!, ct);
|
||
if (string.IsNullOrWhiteSpace(latestId)) continue;
|
||
|
||
if (!lastVids.TryGetValue(stateKey, out var lastId) || lastId != latestId)
|
||
{
|
||
logger.LogInformation("New video found: {VideoId} on {Channel}", latestId, channelInput);
|
||
|
||
// Start transcription
|
||
_ = ProcessYoutubeVideoAsync($"https://www.youtube.com/watch?v={latestId}", toolConfig, workspacePath, logger, path!, hasFfmpeg, ffmpegPath, ct);
|
||
|
||
lastVids[stateKey] = latestId;
|
||
count++;
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogWarning("Error checking channel {Channel}: {Msg}", channelInput, ex.Message);
|
||
}
|
||
}
|
||
|
||
await stateStore.SetAsync("sm_yt_last_vids", JsonSerializer.Serialize(lastVids), ct);
|
||
|
||
return ToolJobResult.NoAction(count > 0 ? $"{count} new video(s) started for transcription." : "No new videos.");
|
||
}
|
||
|
||
private async Task<ToolJobResult> ExecuteTranscriptNotifierJobAsync(IReadOnlyDictionary<string, object?> toolConfig, string workspacePath, IStateStore stateStore, ILogger logger, CancellationToken ct)
|
||
{
|
||
var targetDir = Path.Combine(workspacePath, "YTTranscript");
|
||
if (!Directory.Exists(targetDir))
|
||
return ToolJobResult.NoAction("YTTranscript directory does not exist yet");
|
||
|
||
var processedFilesJson = await stateStore.GetAsync("sm_processed_files", ct) ?? "[]";
|
||
var processedFiles = JsonSerializer.Deserialize<HashSet<string>>(processedFilesJson) ?? new();
|
||
|
||
var allTxtFiles = Directory.GetFiles(targetDir, "*.txt");
|
||
var newFiles = allTxtFiles
|
||
.Select(Path.GetFileName)
|
||
.Where(f => f != null
|
||
&& !processedFiles.Contains(f)
|
||
&& !f.Contains("summary", StringComparison.OrdinalIgnoreCase))
|
||
.ToList();
|
||
|
||
if (newFiles.Count == 0)
|
||
return ToolJobResult.NoAction($"No new transcripts ({allTxtFiles.Length} existing, {processedFiles.Count} already processed)");
|
||
|
||
foreach (var f in newFiles) if (f != null) processedFiles.Add(f);
|
||
await stateStore.SetAsync("sm_processed_files", JsonSerializer.Serialize(processedFiles), ct);
|
||
|
||
// Pending Tasks lesen um dem Agent mitzuteilen wer auf das Ergebnis wartet
|
||
var pendingContext = "";
|
||
var pendingFile = Path.Combine(targetDir, "pending_tasks.json");
|
||
if (File.Exists(pendingFile))
|
||
{
|
||
try
|
||
{
|
||
var pendingJson = await File.ReadAllTextAsync(pendingFile, ct);
|
||
var tasks = JsonSerializer.Deserialize<List<PendingTranscriptTask>>(pendingJson);
|
||
var openTasks = tasks?.Where(t => !t.Completed).ToList();
|
||
if (openTasks?.Count > 0)
|
||
{
|
||
pendingContext = "\n\n[TASK CONTEXT - Who is waiting for these transcripts:]\n" +
|
||
string.Join("\n", openTasks.Select(t =>
|
||
$" - Video URL: {t.Url} | Requester Agent: {t.RequestedBy} | Requested: {t.RequestedAt:yyyy-MM-dd HH:mm}"));
|
||
pendingContext += "\n\nIMPORTANT: Inform the requester agent via AgentComm about the completed transcript! " +
|
||
"Briefly summarize the content and let them know where to read the full transcript.";
|
||
|
||
// Tasks als completed markieren
|
||
foreach (var t in openTasks) t.Completed = true;
|
||
await File.WriteAllTextAsync(pendingFile, JsonSerializer.Serialize(tasks, new JsonSerializerOptions { WriteIndented = true }), ct);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogWarning(ex, "Could not read pending_tasks.json");
|
||
}
|
||
}
|
||
|
||
return ToolJobResult.WakeStateless(
|
||
$"[SYSTEM EVENT: New transcripts available]\n" +
|
||
$"There are {newFiles.Count} new video transcript(s) ready in your personal workspace under YTTranscript/:\n" +
|
||
string.Join("\n", newFiles.Select(f => $" - {f}")) +
|
||
pendingContext +
|
||
"\n\nPlease read the transcripts now using the FileRW tool (action: read, workspace: personal, path: YTTranscript/<filename>) and process them.",
|
||
logSummary: $"{newFiles.Count} new transcript(s) found"
|
||
);
|
||
}
|
||
|
||
// --- Helper ---
|
||
|
||
// Channel-Auflösung liegt jetzt in YouTubeUrl.TryResolveChannelUrl — dort wird die
|
||
// Eingabe validiert, bevor sie an einen externen Prozess geht.
|
||
|
||
private async Task<string?> GetLatestVideoIdAsync(string channelInput, string ytDlpPath, CancellationToken ct)
|
||
{
|
||
try
|
||
{
|
||
if (!YouTubeUrl.TryResolveChannelUrl(channelInput, out var channelUrl, out _))
|
||
return null;
|
||
|
||
var psi = new ProcessStartInfo(ytDlpPath)
|
||
{
|
||
RedirectStandardOutput = true,
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true
|
||
};
|
||
foreach (var arg in YouTubeUrl.BuildLatestVideoIdArgs(channelUrl))
|
||
psi.ArgumentList.Add(arg);
|
||
|
||
using var process = Process.Start(psi);
|
||
if (process == null) return null;
|
||
|
||
var output = await process.StandardOutput.ReadToEndAsync(ct);
|
||
await process.WaitForExitAsync(ct);
|
||
|
||
return output.Trim();
|
||
}
|
||
catch (Exception)
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private async Task<string> ProcessYoutubeVideoAsync(string url, IReadOnlyDictionary<string, object?> toolConfig, string workspacePath, ILogger logger, string ytDlpPath, bool hasFfmpeg, string? ffmpegPath, CancellationToken ct)
|
||
{
|
||
try
|
||
{
|
||
// Validate input before passing to external process
|
||
if (!YouTubeUrl.TryResolveChannelUrl(url, out var safeUrl, out var urlError))
|
||
throw new ArgumentException($"Invalid YouTube URL: {urlError}");
|
||
|
||
var targetDir = Path.Combine(workspacePath, "YTTranscript");
|
||
var tmpDir = Path.Combine(targetDir, "tmp");
|
||
if (!Directory.Exists(tmpDir)) Directory.CreateDirectory(tmpDir);
|
||
|
||
// 1. Video-ID ermitteln
|
||
var videoId = await GetLatestVideoIdAsync(safeUrl, ytDlpPath, ct);
|
||
if (string.IsNullOrWhiteSpace(videoId)) videoId = Guid.NewGuid().ToString();
|
||
|
||
// 2. Audio herunterladen
|
||
string audioFile;
|
||
var ffmpegDir = !string.IsNullOrEmpty(ffmpegPath) && ffmpegPath != "ffmpeg"
|
||
? Path.GetDirectoryName(ffmpegPath)
|
||
: null;
|
||
|
||
if (hasFfmpeg)
|
||
{
|
||
// With ffmpeg: yt-dlp converts directly to mp3
|
||
audioFile = Path.Combine(tmpDir, $"{videoId}.mp3");
|
||
await RunYtDlpAsync(ytDlpPath,
|
||
YouTubeUrl.BuildAudioDownloadArgs(audioFile, safeUrl, ffmpegDir, convertToMp3: true),
|
||
logger, ct);
|
||
}
|
||
else
|
||
{
|
||
// Without ffmpeg: download audio in original format
|
||
var outputTemplate = Path.Combine(tmpDir, $"{videoId}.%(ext)s");
|
||
await RunYtDlpAsync(ytDlpPath,
|
||
YouTubeUrl.BuildAudioDownloadArgs(outputTemplate, safeUrl, null, convertToMp3: false),
|
||
logger, ct);
|
||
audioFile = ""; // looked up below
|
||
}
|
||
|
||
if (!File.Exists(audioFile))
|
||
{
|
||
var foundFile = Directory.GetFiles(tmpDir, videoId + ".*")
|
||
.FirstOrDefault(f => !f.EndsWith(".part") && !f.EndsWith(".ytdl"));
|
||
|
||
if (foundFile != null)
|
||
{
|
||
audioFile = foundFile;
|
||
logger.LogInformation("Audio file found: {File} ({Size} MB)",
|
||
Path.GetFileName(foundFile), new FileInfo(foundFile).Length / 1024.0 / 1024.0);
|
||
}
|
||
else
|
||
{
|
||
throw new FileNotFoundException("Could not download audio.");
|
||
}
|
||
}
|
||
|
||
// 3. Transcribe via OpenRouter
|
||
var apiKey = toolConfig.GetValueOrDefault("openRouterApiKey")?.ToString();
|
||
var model = toolConfig.GetValueOrDefault("sttModel")?.ToString() ?? "openai/whisper-1";
|
||
|
||
if (string.IsNullOrWhiteSpace(apiKey))
|
||
{
|
||
logger.LogError("OpenRouter API key missing in SocialMediaManager tool config (openRouterApiKey). " +
|
||
"Audio was downloaded to: {AudioFile}. Please configure openRouterApiKey in agent configuration.", audioFile);
|
||
return audioFile;
|
||
}
|
||
|
||
var fileSize = new FileInfo(audioFile).Length;
|
||
logger.LogInformation("Starting transcription for {File} ({Size:F1} MB) via OpenRouter ({Model}), ffmpeg={HasFfmpeg}",
|
||
Path.GetFileName(audioFile), fileSize / 1024.0 / 1024.0, model, hasFfmpeg);
|
||
|
||
var transcript = await TranscribeAudioAsync(audioFile, apiKey, model, hasFfmpeg, ffmpegPath, ytDlpPath, logger, ct, workspacePath);
|
||
|
||
// 4. Save transcript
|
||
var transcriptFile = Path.Combine(targetDir, $"{videoId}.txt");
|
||
await File.WriteAllTextAsync(transcriptFile, transcript, ct);
|
||
logger.LogInformation("Transcript successfully saved: {File}", Path.GetFileName(transcriptFile));
|
||
|
||
return transcriptFile;
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogError(ex, "Error in ProcessYoutubeVideoAsync for {Url}", url);
|
||
throw;
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Starts yt-dlp. Arguments are passed individually (ArgumentList) so that
|
||
/// no value can accidentally be interpreted as another option.
|
||
/// </summary>
|
||
private async Task RunYtDlpAsync(string ytDlpPath, List<string> arguments, ILogger logger, CancellationToken ct)
|
||
{
|
||
try
|
||
{
|
||
var psi = new ProcessStartInfo(ytDlpPath)
|
||
{
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true,
|
||
RedirectStandardError = true
|
||
};
|
||
foreach (var arg in arguments)
|
||
psi.ArgumentList.Add(arg);
|
||
|
||
using var process = Process.Start(psi);
|
||
if (process != null)
|
||
{
|
||
var stderr = await process.StandardError.ReadToEndAsync(ct);
|
||
await process.WaitForExitAsync(ct);
|
||
if (process.ExitCode != 0)
|
||
logger.LogWarning("yt-dlp exited with code {Code}: {Stderr}", process.ExitCode, stderr);
|
||
}
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
logger.LogError(ex, "Error calling yt-dlp.");
|
||
throw;
|
||
}
|
||
}
|
||
|
||
private const long MaxChunkBytes = 24 * 1024 * 1024; // 24 MB (Whisper-Limit: 25 MB, Puffer)
|
||
|
||
/// <summary>#654: Where ffmpeg runs — set by the instance at start.</summary>
|
||
public ClawdDotNet.Core.Sandbox.ISandboxRunner? Sandbox { get; set; }
|
||
|
||
private int _warnedUnsandboxed;
|
||
|
||
private async Task<string> TranscribeAudioAsync(string filePath, string apiKey, string model,
|
||
bool hasFfmpeg, string? ffmpegPath, string ytDlpPath, ILogger logger, CancellationToken ct, string? workspacePath = null)
|
||
{
|
||
var fileInfo = new FileInfo(filePath);
|
||
if (!fileInfo.Exists)
|
||
throw new FileNotFoundException($"Audio file not found: {filePath}");
|
||
|
||
// Kleine Dateien direkt transkribieren
|
||
if (fileInfo.Length <= MaxChunkBytes)
|
||
return await TranscribeChunkAsync(filePath, apiKey, model, logger, ct);
|
||
|
||
logger.LogInformation("File is {Size:F1} MB — chunking required", fileInfo.Length / 1024.0 / 1024.0);
|
||
|
||
// Große Dateien in Chunks aufteilen
|
||
var chunkDir = Path.Combine(Path.GetDirectoryName(filePath)!, "chunks");
|
||
if (Directory.Exists(chunkDir)) Directory.Delete(chunkDir, true);
|
||
Directory.CreateDirectory(chunkDir);
|
||
|
||
try
|
||
{
|
||
List<string> chunks;
|
||
|
||
if (hasFfmpeg && workspacePath is not null && Sandbox is { IsAvailable: true } sandbox
|
||
&& ClawdDotNet.Core.Sandbox.SandboxPaths.InSandbox(workspacePath, filePath) is { } inFile
|
||
&& ClawdDotNet.Core.Sandbox.SandboxPaths.InSandbox(workspacePath, chunkDir) is { } inChunks)
|
||
{
|
||
// #654: The media comes from the internet — ffmpeg parses it in the sandbox,
|
||
// seeing nothing but the workspace, without network.
|
||
var ffmpegExe = !string.IsNullOrEmpty(ffmpegPath) ? ffmpegPath : "ffmpeg";
|
||
var result = await sandbox.RunAsync(new ClawdDotNet.Core.Sandbox.SandboxRequest
|
||
{
|
||
Executable = ffmpegExe,
|
||
Arguments = ["-hide_banner", "-loglevel", "error", "-i", inFile, "-f", "segment", "-segment_time", "1200",
|
||
"-c:a", "libmp3lame", "-q:a", "5", inChunks + "/chunk_%03d.mp3"],
|
||
WorkspacePath = workspacePath,
|
||
// A program outside /usr is made visible where it lies — read-only.
|
||
ReadOnlyPaths = Path.IsPathRooted(ffmpegExe) && !ffmpegExe.StartsWith("/usr/", StringComparison.Ordinal)
|
||
? [Path.GetDirectoryName(ffmpegExe)!]
|
||
: [],
|
||
Limits = new ClawdDotNet.Core.Sandbox.SandboxLimits
|
||
{
|
||
Timeout = TimeSpan.FromMinutes(30),
|
||
MaxFileBytes = 200L * 1024 * 1024,
|
||
MaxOutputBytes = 256 * 1024
|
||
}
|
||
}, ct);
|
||
if (!result.Succeeded)
|
||
logger.LogWarning("ffmpeg in the sandbox ended with {Code}{Timeout}: {Error}",
|
||
result.ExitCode, result.TimedOut ? " (timeout)" : "", result.StandardError.Trim());
|
||
|
||
chunks = Directory.GetFiles(chunkDir, "chunk_*.mp3").OrderBy(f => f).ToList();
|
||
}
|
||
else if (hasFfmpeg)
|
||
{
|
||
// #654: Without a sandbox (Windows, or no bwrap) as before — said once. yt-dlp,
|
||
// which needs the network, still runs this way everywhere until #655 brings the
|
||
// sandbox a network of its own.
|
||
if (OperatingSystem.IsLinux() && Interlocked.Exchange(ref _warnedUnsandboxed, 1) == 0)
|
||
logger.LogWarning("ffmpeg runs without a sandbox: {Reason}",
|
||
Sandbox?.UnavailableReason ?? "no sandbox was handed to the tool");
|
||
|
||
// ffmpeg: 20-minute segments as mp3 (under 25 MB Whisper limit at q:a 5)
|
||
var chunkPattern = Path.Combine(chunkDir, "chunk_%03d.mp3");
|
||
var ffmpegExe = !string.IsNullOrEmpty(ffmpegPath) ? ffmpegPath : "ffmpeg";
|
||
var psi = new ProcessStartInfo(ffmpegExe)
|
||
{
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true,
|
||
RedirectStandardError = true
|
||
};
|
||
foreach (var arg in new[]
|
||
{
|
||
"-i", filePath,
|
||
"-f", "segment",
|
||
"-segment_time", "1200",
|
||
"-c:a", "libmp3lame",
|
||
"-q:a", "5",
|
||
chunkPattern
|
||
})
|
||
{
|
||
psi.ArgumentList.Add(arg);
|
||
}
|
||
|
||
using var process = Process.Start(psi);
|
||
if (process is not null)
|
||
{
|
||
await process.StandardError.ReadToEndAsync(ct);
|
||
await process.WaitForExitAsync(ct);
|
||
}
|
||
|
||
chunks = Directory.GetFiles(chunkDir, "chunk_*.mp3").OrderBy(f => f).ToList();
|
||
}
|
||
else
|
||
{
|
||
// Fallback without ffmpeg
|
||
logger.LogInformation("ffmpeg not available — using yt-dlp segmentation");
|
||
chunks = await ChunkWithYtDlpAsync(filePath, chunkDir, ytDlpPath, logger, ct);
|
||
}
|
||
|
||
if (chunks.Count == 0)
|
||
throw new InvalidOperationException(
|
||
"Chunking failed — no segments produced. " +
|
||
(hasFfmpeg ? "ffmpeg error?" : "Install ffmpeg for reliable chunking: 'winget install ffmpeg'"));
|
||
|
||
logger.LogInformation("{Count} chunks produced, starting transcription...", chunks.Count);
|
||
|
||
var transcripts = new List<string>();
|
||
for (int i = 0; i < chunks.Count; i++)
|
||
{
|
||
logger.LogInformation("Transcribing chunk {Index}/{Total}: {File} ({Size:F1} MB)",
|
||
i + 1, chunks.Count, Path.GetFileName(chunks[i]), new FileInfo(chunks[i]).Length / 1024.0 / 1024.0);
|
||
var chunkText = await TranscribeChunkAsync(chunks[i], apiKey, model, logger, ct);
|
||
transcripts.Add(chunkText);
|
||
}
|
||
|
||
return string.Join("\n\n", transcripts);
|
||
}
|
||
finally
|
||
{
|
||
if (Directory.Exists(chunkDir))
|
||
Directory.Delete(chunkDir, true);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Fallback chunking without ffmpeg.
|
||
///
|
||
/// <para>yt-dlp cannot split a file that is already downloaded, and without ffmpeg
|
||
/// nothing else here can either. So a file that fits the Whisper limit is sent as a
|
||
/// single chunk; anything larger fails with an installation hint instead of a
|
||
/// truncated transcript.</para>
|
||
/// </summary>
|
||
private async Task<List<string>> ChunkWithYtDlpAsync(string filePath, string chunkDir,
|
||
string ytDlpPath, ILogger logger, CancellationToken ct)
|
||
{
|
||
var fileSize = new FileInfo(filePath).Length;
|
||
if (fileSize <= 25 * 1024 * 1024) // Whisper limit exact 25 MB
|
||
{
|
||
logger.LogInformation("File is {Size:F1} MB — attempting direct upload anyway (Whisper limit: 25 MB)", fileSize / 1024.0 / 1024.0);
|
||
return [filePath];
|
||
}
|
||
|
||
logger.LogWarning(
|
||
"Audio file is {Size:F1} MB and ffmpeg is not installed. " +
|
||
"Chunking without ffmpeg not possible. Please install ffmpeg: 'winget install ffmpeg'",
|
||
fileSize / 1024.0 / 1024.0);
|
||
|
||
throw new InvalidOperationException(
|
||
$"The audio file is {fileSize / 1024.0 / 1024.0:F1} MB and exceeds the Whisper limit of 25 MB. " +
|
||
"For large videos, ffmpeg is required for splitting. Installation: 'winget install ffmpeg'");
|
||
}
|
||
|
||
private static string GetMimeType(string filePath) => Path.GetExtension(filePath).ToLowerInvariant() switch
|
||
{
|
||
".mp3" => "audio/mpeg",
|
||
".webm" => "audio/webm",
|
||
".m4a" => "audio/mp4",
|
||
".ogg" or ".opus" => "audio/ogg",
|
||
".wav" => "audio/wav",
|
||
".flac" => "audio/flac",
|
||
_ => "application/octet-stream"
|
||
};
|
||
|
||
private async Task<string> TranscribeChunkAsync(string filePath, string apiKey, string model,
|
||
ILogger logger, CancellationToken ct)
|
||
{
|
||
var fileName = Path.GetFileName(filePath);
|
||
var fileBytes = await File.ReadAllBytesAsync(filePath, ct);
|
||
var audioBase64 = Convert.ToBase64String(fileBytes);
|
||
var audioFormat = Path.GetExtension(filePath).TrimStart('.').ToLowerInvariant();
|
||
|
||
logger.LogInformation("STT Request: Model={Model}, File={File} ({Size:F1} MB), Format={Format}",
|
||
model, fileName, fileBytes.Length / 1024.0 / 1024.0, audioFormat);
|
||
|
||
var payload = JsonSerializer.Serialize(new
|
||
{
|
||
model,
|
||
input_audio = new { data = audioBase64, format = audioFormat }
|
||
});
|
||
|
||
using var content = new StringContent(payload, Encoding.UTF8, "application/json");
|
||
using var request = new HttpRequestMessage(HttpMethod.Post, "https://openrouter.ai/api/v1/audio/transcriptions");
|
||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
||
request.Headers.Add("HTTP-Referer", "https://github.com/ClawdDotNet");
|
||
request.Headers.Add("X-Title", "ClawdDotNet Social Media Manager");
|
||
request.Content = content;
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var responseContent = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode)
|
||
{
|
||
logger.LogError("STT error for {File}: {Status} - {Response}",
|
||
fileName, response.StatusCode, responseContent);
|
||
throw new Exception($"STT error ({fileName}): {response.StatusCode} - {responseContent}");
|
||
}
|
||
|
||
var doc = JsonDocument.Parse(responseContent);
|
||
return doc.RootElement.TryGetProperty("text", out var t) ? t.GetString() ?? "" : "";
|
||
}
|
||
|
||
private IEnumerable<string> GetConfigArray(IReadOnlyDictionary<string, object?> config, string key)
|
||
{
|
||
if (!config.TryGetValue(key, out var val) || val == null) return Array.Empty<string>();
|
||
if (val is string[] arr) return arr;
|
||
if (val is List<string> list) return list;
|
||
if (val is JsonElement je && je.ValueKind == JsonValueKind.Array)
|
||
return je.EnumerateArray().Select(x => x.GetString() ?? "").Where(x => !string.IsNullOrWhiteSpace(x));
|
||
if (val is string s) return s.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||
return Array.Empty<string>();
|
||
}
|
||
|
||
private async Task<(bool available, string? path, bool hasFfmpeg, string? ffmpegPath)> VerifyYtDlpAsync()
|
||
{
|
||
// 1. Check in PATH
|
||
string? detectedPath = null;
|
||
try
|
||
{
|
||
var psi = new ProcessStartInfo("yt-dlp", "--version")
|
||
{
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true,
|
||
RedirectStandardOutput = true
|
||
};
|
||
using var process = Process.Start(psi);
|
||
if (process != null)
|
||
{
|
||
await process.WaitForExitAsync();
|
||
detectedPath = "yt-dlp";
|
||
}
|
||
}
|
||
catch { }
|
||
|
||
if (detectedPath == null)
|
||
{
|
||
// 2. Check local paths
|
||
// Unter Linux heißt das Programm ohne Endung. Ohne diese Variante fände die
|
||
// Suche im Programmverzeichnis dort nie etwas — und die PATH-Prüfung davor
|
||
// ist die einzige, die griffe.
|
||
var candidates = LocalToolCandidates("yt-dlp");
|
||
|
||
foreach (var p in candidates)
|
||
{
|
||
if (File.Exists(p))
|
||
{
|
||
detectedPath = p;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
if (detectedPath == null) return (false, null, false, null);
|
||
|
||
// 3. Check for ffmpeg (optional but recommended)
|
||
bool hasFfmpeg = false;
|
||
string? ffmpegPath = null;
|
||
try
|
||
{
|
||
var psi = new ProcessStartInfo("ffmpeg", "-version")
|
||
{
|
||
UseShellExecute = false,
|
||
CreateNoWindow = true,
|
||
RedirectStandardOutput = true
|
||
};
|
||
using var process = Process.Start(psi);
|
||
if (process != null)
|
||
{
|
||
await process.WaitForExitAsync();
|
||
hasFfmpeg = true;
|
||
ffmpegPath = "ffmpeg";
|
||
}
|
||
}
|
||
catch { }
|
||
|
||
if (!hasFfmpeg)
|
||
{
|
||
// Check local paths for ffmpeg
|
||
var ffmpegCandidates = LocalToolCandidates("ffmpeg");
|
||
|
||
foreach (var fp in ffmpegCandidates)
|
||
{
|
||
if (File.Exists(fp))
|
||
{
|
||
hasFfmpeg = true;
|
||
ffmpegPath = fp;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
return (true, detectedPath, hasFfmpeg, ffmpegPath);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Wo ein mitgeliefertes Hilfsprogramm liegen könnte — Programmverzeichnis und
|
||
/// <c>tools/</c>, je mit und ohne <c>.exe</c>.
|
||
///
|
||
/// Die Endung wird auf beiden Plattformen mitgeprüft, nicht nur auf der jeweils
|
||
/// „richtigen": Ein Deployment-Paket, das unter Windows geschnürt wurde, kann die
|
||
/// Windows-Binärdatei enthalten, und umgekehrt. Vier Aufrufe von
|
||
/// <see cref="File.Exists"/> sind billiger als eine Fallunterscheidung, die man
|
||
/// vergessen kann.
|
||
/// </summary>
|
||
private static string[] LocalToolCandidates(string name)
|
||
{
|
||
var baseDir = AppContext.BaseDirectory;
|
||
|
||
return
|
||
[
|
||
Path.Combine(baseDir, name),
|
||
Path.Combine(baseDir, name + ".exe"),
|
||
Path.Combine(baseDir, "tools", name),
|
||
Path.Combine(baseDir, "tools", name + ".exe")
|
||
];
|
||
}
|
||
|
||
private string GetYtDlpInstallationGuide()
|
||
{
|
||
return "ERROR: Dependencies missing for YouTube features.\n\n" +
|
||
"1. yt-dlp: Download 'yt-dlp.exe' from https://github.com/yt-dlp/yt-dlp/releases and place it in 'tools/yt-dlp.exe'.\n" +
|
||
"2. ffmpeg (optional): Recommended for MP3 conversion. Without ffmpeg, WebM/M4A files will be used.\n" +
|
||
" Installation via: 'winget install ffmpeg'.\n\n" +
|
||
"Tip: Use 'winget install yt-dlp ffmpeg' for a quick installation of both tools.";
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════
|
||
// X/TWITTER MONITOR JOB
|
||
//
|
||
// Cost optimisation:
|
||
// - query ALL watched accounts in ONE search query
|
||
// ("from:user1 OR from:user2 OR from:user3") → 1 API call instead of N
|
||
// - since_id tracking → only tweets newer than the last check
|
||
// - results cached locally → manual queries cost no API call
|
||
// - first run: set since_id to "now" without fetching old tweets (zero cost)
|
||
// ═══════════════════════════════════════════════════
|
||
|
||
private async Task<ToolJobResult> ExecuteXAccountMonitorJobAsync(
|
||
IReadOnlyDictionary<string, object?> toolConfig, string workspacePath,
|
||
IStateStore stateStore, ILogger logger, CancellationToken ct)
|
||
{
|
||
var apiKey = toolConfig.GetValueOrDefault("xApiKey")?.ToString();
|
||
if (string.IsNullOrWhiteSpace(apiKey))
|
||
return ToolJobResult.NoAction("X API key (xApiKey) not configured");
|
||
|
||
var accounts = GetConfigArray(toolConfig, "xWatchAccounts").ToList();
|
||
if (accounts.Count == 0)
|
||
return ToolJobResult.NoAction("No X accounts configured (xWatchAccounts)");
|
||
|
||
var cleanAccounts = accounts
|
||
.Where(a => !string.IsNullOrWhiteSpace(a))
|
||
.Select(a => a!.TrimStart('@').ToLowerInvariant())
|
||
.Distinct()
|
||
.ToList();
|
||
|
||
// Globale since_id laden (höchste bekannte Tweet-ID über alle Accounts)
|
||
var sinceId = await stateStore.GetAsync("sm_x_global_since_id", ct);
|
||
|
||
// Erster Lauf: Baseline setzen ohne alte Tweets abzurufen
|
||
if (string.IsNullOrWhiteSpace(sinceId))
|
||
{
|
||
// Eine minimale Abfrage machen um die aktuelle höchste ID zu ermitteln
|
||
var initQuery = $"from:{cleanAccounts.First()}";
|
||
var initUrl = $"https://api.twitter.com/2/tweets/search/recent" +
|
||
$"?query={Uri.EscapeDataString(initQuery)}&max_results=10&tweet.fields=created_at";
|
||
|
||
using var initReq = new HttpRequestMessage(HttpMethod.Get, initUrl);
|
||
initReq.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
||
|
||
var initResp = await _httpClient.SendAsync(initReq, ct);
|
||
if (initResp.IsSuccessStatusCode)
|
||
{
|
||
var initContent = await initResp.Content.ReadAsStringAsync(ct);
|
||
using var initDoc = JsonDocument.Parse(initContent);
|
||
|
||
// newest_id aus meta extrahieren
|
||
if (initDoc.RootElement.TryGetProperty("meta", out var meta) &&
|
||
meta.TryGetProperty("newest_id", out var newestIdEl))
|
||
{
|
||
sinceId = newestIdEl.GetString();
|
||
}
|
||
}
|
||
|
||
if (!string.IsNullOrWhiteSpace(sinceId))
|
||
{
|
||
await stateStore.SetAsync("sm_x_global_since_id", sinceId, ct);
|
||
logger.LogInformation("X Monitor: Baseline set (since_id={SinceId}). Cost: 1 API call. From now on only new tweets will be reported.", sinceId);
|
||
return ToolJobResult.NoAction($"Baseline set. Starting with next tick only new tweets will be reported. (1 API call)");
|
||
}
|
||
|
||
return ToolJobResult.NoAction("Could not set baseline — retry on next tick.");
|
||
}
|
||
|
||
// ─── Hauptlogik: 1 Search-Query für ALLE Accounts ───
|
||
|
||
// Query bauen: "from:user1 OR from:user2 OR from:user3"
|
||
// X API max Query-Länge: 512 Zeichen (Basic), 1024 (Pro)
|
||
var queryParts = cleanAccounts.Select(a => $"from:{a}");
|
||
var query = string.Join(" OR ", queryParts);
|
||
|
||
if (query.Length > 512)
|
||
{
|
||
// Bei sehr vielen Accounts: aufteilen in Batches
|
||
logger.LogWarning("X Monitor: Query too long ({Len} characters), using only the first accounts", query.Length);
|
||
var trimmedAccounts = new List<string>();
|
||
var currentLen = 0;
|
||
foreach (var acc in cleanAccounts)
|
||
{
|
||
var part = $"from:{acc}";
|
||
if (currentLen + part.Length + 4 > 500) break; // 4 = " OR "
|
||
trimmedAccounts.Add(acc);
|
||
currentLen += part.Length + 4;
|
||
}
|
||
query = string.Join(" OR ", trimmedAccounts.Select(a => $"from:{a}"));
|
||
}
|
||
|
||
var searchUrl = $"https://api.twitter.com/2/tweets/search/recent" +
|
||
$"?query={Uri.EscapeDataString(query)}" +
|
||
$"&since_id={sinceId}" +
|
||
$"&max_results=100" +
|
||
$"&tweet.fields=created_at,public_metrics,author_id" +
|
||
$"&expansions=author_id" +
|
||
$"&user.fields=username";
|
||
|
||
using var request = new HttpRequestMessage(HttpMethod.Get, searchUrl);
|
||
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", apiKey);
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var content = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode)
|
||
{
|
||
logger.LogWarning("X Monitor API error: {Status} - {Content}", response.StatusCode, content);
|
||
return ToolJobResult.NoAction($"X API error: {response.StatusCode}");
|
||
}
|
||
|
||
using var doc = JsonDocument.Parse(content);
|
||
|
||
// newest_id für nächsten Tick speichern
|
||
if (doc.RootElement.TryGetProperty("meta", out var metaEl) &&
|
||
metaEl.TryGetProperty("newest_id", out var newSinceIdEl))
|
||
{
|
||
var newSinceId = newSinceIdEl.GetString();
|
||
if (!string.IsNullOrWhiteSpace(newSinceId))
|
||
await stateStore.SetAsync("sm_x_global_since_id", newSinceId, ct);
|
||
}
|
||
|
||
// Prüfen ob es überhaupt Ergebnisse gibt
|
||
if (!doc.RootElement.TryGetProperty("data", out var dataArr) ||
|
||
dataArr.ValueKind != JsonValueKind.Array ||
|
||
dataArr.GetArrayLength() == 0)
|
||
{
|
||
return ToolJobResult.NoAction("No new tweets (1 API call)");
|
||
}
|
||
|
||
// Author-ID → Username Mapping aus "includes.users" aufbauen
|
||
var authorMap = new Dictionary<string, string>();
|
||
if (doc.RootElement.TryGetProperty("includes", out var includes) &&
|
||
includes.TryGetProperty("users", out var usersArr))
|
||
{
|
||
foreach (var user in usersArr.EnumerateArray())
|
||
{
|
||
var userId = user.TryGetProperty("id", out var uid) ? uid.GetString() : null;
|
||
var uname = user.TryGetProperty("username", out var un) ? un.GetString() : null;
|
||
if (userId != null && uname != null)
|
||
authorMap[userId] = uname;
|
||
}
|
||
}
|
||
|
||
// Tweets in strukturiertes Format umwandeln
|
||
var allNewTweets = new List<object>();
|
||
foreach (var tweet in dataArr.EnumerateArray())
|
||
{
|
||
var authorId = tweet.TryGetProperty("author_id", out var aid) ? aid.GetString() : "";
|
||
var authorName = authorId != null && authorMap.TryGetValue(authorId, out var name) ? name : authorId;
|
||
|
||
allNewTweets.Add(new
|
||
{
|
||
Account = $"@{authorName}",
|
||
Id = tweet.TryGetProperty("id", out var tid) ? tid.GetString() : "",
|
||
Text = tweet.TryGetProperty("text", out var txt) ? txt.GetString() : "",
|
||
CreatedAt = tweet.TryGetProperty("created_at", out var ca) ? ca.GetString() : "",
|
||
Metrics = tweet.TryGetProperty("public_metrics", out var pm) ? pm.Clone() : default
|
||
});
|
||
}
|
||
|
||
logger.LogInformation("X Monitor: {Count} new tweets found (1 API call for {AccountCount} accounts)",
|
||
allNewTweets.Count, cleanAccounts.Count);
|
||
|
||
// Ergebnisse im Workspace cachen
|
||
var resultDir = Path.Combine(workspacePath, "XMonitor");
|
||
Directory.CreateDirectory(resultDir);
|
||
var resultFile = Path.Combine(resultDir, $"{DateTime.Now:yyyy-MM-dd_HHmmss}_tweets.json");
|
||
await File.WriteAllTextAsync(resultFile,
|
||
JsonSerializer.Serialize(allNewTweets, new JsonSerializerOptions { WriteIndented = true }),
|
||
new UTF8Encoding(false), ct);
|
||
|
||
// Alte Cache-Dateien aufräumen (nur die letzten 20 behalten)
|
||
CleanupOldFiles(resultDir, "*.json", keepCount: 20);
|
||
|
||
var summary = string.Join("\n", allNewTweets.Take(10).Select(t =>
|
||
{
|
||
var json = JsonSerializer.SerializeToElement(t);
|
||
var account = json.TryGetProperty("Account", out var a) ? a.GetString() : "?";
|
||
var text = json.TryGetProperty("Text", out var tx) ? tx.GetString() : "";
|
||
var preview = text?.Length > 80 ? text[..80] + "..." : text;
|
||
return $" - {account}: {preview}";
|
||
}));
|
||
|
||
var extra = allNewTweets.Count > 10 ? $"\n ... and {allNewTweets.Count - 10} more" : "";
|
||
|
||
return ToolJobResult.Wake(
|
||
$"[X Monitor] {allNewTweets.Count} new tweet(s) (1 API call for {cleanAccounts.Count} accounts):\n\n{summary}{extra}\n\n" +
|
||
$"Full data saved in: personal:/XMonitor/{Path.GetFileName(resultFile)}\n" +
|
||
$"Read the file with FileRW and analyze the relevant tweets.",
|
||
logSummary: $"X: {allNewTweets.Count} new tweet(s) (1 call)"
|
||
);
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════
|
||
// REDDIT MONITOR JOB
|
||
// ═══════════════════════════════════════════════════
|
||
|
||
private async Task<ToolJobResult> ExecuteRedditMonitorJobAsync(
|
||
IReadOnlyDictionary<string, object?> toolConfig, string workspacePath,
|
||
IStateStore stateStore, ILogger logger, CancellationToken ct)
|
||
{
|
||
var subreddits = GetConfigArray(toolConfig, "redditWatchSubreddits").ToList();
|
||
if (subreddits.Count == 0)
|
||
return ToolJobResult.NoAction("No subreddits configured (redditWatchSubreddits)");
|
||
|
||
var redditLimit = 15; // Posts pro Subreddit
|
||
if (toolConfig.TryGetValue("redditPostLimit", out var rl) && rl is JsonElement rlElem)
|
||
redditLimit = rlElem.TryGetInt32(out var rlVal) ? rlVal : 15;
|
||
|
||
// Letzte bekannte Post-IDs pro Subreddit laden
|
||
var lastPostsJson = await stateStore.GetAsync("sm_reddit_last_posts", ct) ?? "{}";
|
||
var lastPosts = JsonSerializer.Deserialize<Dictionary<string, string>>(lastPostsJson) ?? new();
|
||
|
||
var allNewPosts = new List<object>();
|
||
var errors = new List<string>();
|
||
|
||
foreach (var sub in subreddits)
|
||
{
|
||
if (string.IsNullOrWhiteSpace(sub)) continue;
|
||
var cleanSub = sub.TrimStart('/').Replace("r/", "").Trim();
|
||
|
||
try
|
||
{
|
||
var url = $"https://www.reddit.com/r/{Uri.EscapeDataString(cleanSub)}/new.json?limit={redditLimit}";
|
||
using var request = new HttpRequestMessage(HttpMethod.Get, url);
|
||
request.Headers.UserAgent.ParseAdd("ClawdDotNet/1.0");
|
||
|
||
var response = await _httpClient.SendAsync(request, ct);
|
||
var content = await response.Content.ReadAsStringAsync(ct);
|
||
|
||
if (!response.IsSuccessStatusCode)
|
||
{
|
||
errors.Add($"r/{cleanSub}: HTTP {response.StatusCode}");
|
||
continue;
|
||
}
|
||
|
||
var posts = ExtractRedditPosts(content);
|
||
if (posts.Count == 0) continue;
|
||
|
||
// Filtern auf neue Posts (nach der letzten bekannten ID)
|
||
lastPosts.TryGetValue(cleanSub, out var lastKnownId);
|
||
var newPosts = lastKnownId != null
|
||
? posts.Where(p =>
|
||
{
|
||
var postId = p.TryGetValue("Id", out var id) ? id?.ToString() : null;
|
||
return postId != null && string.Compare(postId, lastKnownId, StringComparison.Ordinal) > 0;
|
||
}).ToList()
|
||
: posts; // Erster Lauf: alle Posts melden
|
||
|
||
if (newPosts.Count == 0) continue;
|
||
|
||
// Neueste ID speichern
|
||
var newestId = posts
|
||
.Select(p => p.TryGetValue("Id", out var id) ? id?.ToString() : null)
|
||
.Where(id => id != null)
|
||
.OrderByDescending(id => id)
|
||
.FirstOrDefault();
|
||
|
||
if (newestId != null)
|
||
lastPosts[cleanSub] = newestId;
|
||
|
||
foreach (var post in newPosts)
|
||
{
|
||
post["Subreddit"] = $"r/{cleanSub}";
|
||
allNewPosts.Add(post);
|
||
}
|
||
|
||
logger.LogInformation("Reddit Monitor: {Count} new posts in r/{Sub}", newPosts.Count, cleanSub);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
errors.Add($"r/{cleanSub}: {ex.Message}");
|
||
logger.LogWarning(ex, "Reddit Monitor error for r/{Sub}", cleanSub);
|
||
}
|
||
}
|
||
|
||
// State speichern
|
||
await stateStore.SetAsync("sm_reddit_last_posts", JsonSerializer.Serialize(lastPosts), ct);
|
||
|
||
if (allNewPosts.Count == 0)
|
||
{
|
||
var errMsg = errors.Count > 0 ? $" (Errors: {string.Join("; ", errors)})" : "";
|
||
return ToolJobResult.NoAction($"No new Reddit posts{errMsg}");
|
||
}
|
||
|
||
// Save results in workspace
|
||
var resultDir = Path.Combine(workspacePath, "RedditMonitor");
|
||
Directory.CreateDirectory(resultDir);
|
||
var resultFile = Path.Combine(resultDir, $"{DateTime.Now:yyyy-MM-dd_HHmmss}_posts.json");
|
||
await File.WriteAllTextAsync(resultFile,
|
||
JsonSerializer.Serialize(allNewPosts, new JsonSerializerOptions { WriteIndented = true }),
|
||
new UTF8Encoding(false), ct);
|
||
|
||
var summary = string.Join("\n", allNewPosts.Take(10).Select(p =>
|
||
{
|
||
var sub = p is Dictionary<string, object?> d ? d.GetValueOrDefault("Subreddit")?.ToString() : "?";
|
||
var title = p is Dictionary<string, object?> d2 ? d2.GetValueOrDefault("Title")?.ToString() : "";
|
||
var preview = title?.Length > 80 ? title[..80] + "..." : title;
|
||
return $" - [{sub}] {preview}";
|
||
}));
|
||
|
||
var extra = allNewPosts.Count > 10 ? $"\n ... and {allNewPosts.Count - 10} more" : "";
|
||
var errorInfo = errors.Count > 0 ? $"\n\n⚠ Errors with: {string.Join(", ", errors)}" : "";
|
||
|
||
// Cleanup old cache files
|
||
CleanupOldFiles(resultDir, "*.json", keepCount: 20);
|
||
|
||
return ToolJobResult.Wake(
|
||
$"[Reddit Monitor] {allNewPosts.Count} new post(s) from monitored subreddits:\n\n{summary}{extra}\n\n" +
|
||
$"Full data saved in: personal:/RedditMonitor/{Path.GetFileName(resultFile)}\n" +
|
||
$"Read the file with FileRW and analyze the relevant posts.{errorInfo}",
|
||
logSummary: $"Reddit Monitor: {allNewPosts.Count} new post(s)"
|
||
);
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════
|
||
// HELPER
|
||
// ═══════════════════════════════════════════════════
|
||
|
||
private static void CleanupOldFiles(string directory, string pattern, int keepCount)
|
||
{
|
||
try
|
||
{
|
||
var files = Directory.GetFiles(directory, pattern)
|
||
.OrderByDescending(f => File.GetCreationTime(f))
|
||
.Skip(keepCount)
|
||
.ToList();
|
||
|
||
foreach (var file in files)
|
||
{
|
||
try { File.Delete(file); } catch { /* best effort */ }
|
||
}
|
||
}
|
||
catch { /* ignorieren */ }
|
||
}
|
||
|
||
// ═══════════════════════════════════════════════════
|
||
// REDDIT HELPER
|
||
// ═══════════════════════════════════════════════════
|
||
|
||
private static List<Dictionary<string, object?>> ExtractRedditPosts(string jsonContent)
|
||
{
|
||
var posts = new List<Dictionary<string, object?>>();
|
||
|
||
try
|
||
{
|
||
using var doc = JsonDocument.Parse(jsonContent);
|
||
var root = doc.RootElement;
|
||
|
||
JsonElement children;
|
||
if (root.TryGetProperty("data", out var data) && data.TryGetProperty("children", out children))
|
||
{
|
||
// Standard Reddit listing format
|
||
}
|
||
else
|
||
{
|
||
return posts;
|
||
}
|
||
|
||
foreach (var child in children.EnumerateArray())
|
||
{
|
||
if (!child.TryGetProperty("data", out var postData)) continue;
|
||
|
||
posts.Add(new Dictionary<string, object?>
|
||
{
|
||
["Id"] = postData.TryGetProperty("id", out var id) ? id.GetString() : null,
|
||
["Title"] = postData.TryGetProperty("title", out var title) ? title.GetString() : null,
|
||
["Author"] = postData.TryGetProperty("author", out var author) ? author.GetString() : null,
|
||
["Score"] = postData.TryGetProperty("score", out var score) ? score.GetInt32() : 0,
|
||
["NumComments"] = postData.TryGetProperty("num_comments", out var nc) ? nc.GetInt32() : 0,
|
||
["Url"] = postData.TryGetProperty("url", out var postUrl) ? postUrl.GetString() : null,
|
||
["SelfText"] = postData.TryGetProperty("selftext", out var selfText) ? selfText.GetString() : null,
|
||
["CreatedUtc"] = postData.TryGetProperty("created_utc", out var created)
|
||
? DateTimeOffset.FromUnixTimeSeconds((long)created.GetDouble()).DateTime.ToString("yyyy-MM-dd HH:mm:ss")
|
||
: null,
|
||
["Permalink"] = postData.TryGetProperty("permalink", out var plink) ? $"https://reddit.com{plink.GetString()}" : null
|
||
});
|
||
}
|
||
}
|
||
catch
|
||
{
|
||
// Fehlerhafte JSON-Struktur ignorieren
|
||
}
|
||
|
||
return posts;
|
||
}
|
||
}
|