Richard
af398353f0
Phase 1: Modul-System formalisiert (IModule + ModuleRegistry) - Core/Modules/IModule.cs: Vertrag (Key, DisplayName, Description, Version, RegisterServices, InitializeAsync, GetWorkers, CreateWindow) - Core/Modules/ModuleRegistry.cs - CongressTradingModule auf IModule umgestellt (instanzbasiert, Platzhalter-Fenster) - Program.cs iteriert ueber IModule[] + Registry; Form1 initialisiert Module ueber Registry - Testbarkeit: WorkerBase DB-Log-Seam (virtuell); CapitolTradesScraper.ParseTradesFromHtml extrahiert (offline gegen ct_raw.html testbar) - Tests: ModuleRegistry, CongressTradingModule, WorkerBase, CapitolTradesScraper -> 15/15 gruen Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> @
343 lines
13 KiB
C#
343 lines
13 KiB
C#
using System.Text.Json;
|
||
using System.Text.Json.Serialization;
|
||
using System.Text.RegularExpressions;
|
||
using IBKRTrader.Core.Logging;
|
||
using IBKRTrader.Modules.CongressTrading.Models;
|
||
|
||
namespace IBKRTrader.Modules.CongressTrading.Scraper;
|
||
|
||
// ─── JSON-Modelle (exakte Feldnamen aus dem RSC-Stream) ──────────────────────
|
||
|
||
internal class CtApiWrapper
|
||
{
|
||
[JsonPropertyName("data")] public List<CtTrade>? Data { get; set; }
|
||
[JsonPropertyName("meta")] public CtMeta? Meta { get; set; }
|
||
}
|
||
|
||
internal class CtMeta
|
||
{
|
||
[JsonPropertyName("totalPages")] public int TotalPages { get; set; }
|
||
[JsonPropertyName("pageSize")] public int PageSize { get; set; }
|
||
[JsonPropertyName("totalCount")] public int TotalCount { get; set; }
|
||
[JsonPropertyName("page")] public int Page { get; set; }
|
||
}
|
||
|
||
internal class CtTrade
|
||
{
|
||
[JsonPropertyName("_txId")] public long TxId { get; set; }
|
||
[JsonPropertyName("txDate")] public string? TxDate { get; set; }
|
||
[JsonPropertyName("pubDate")] public string? PubDate { get; set; }
|
||
[JsonPropertyName("txType")] public string? TxType { get; set; }
|
||
[JsonPropertyName("value")] public decimal? Value { get; set; }
|
||
[JsonPropertyName("price")] public decimal? Price { get; set; }
|
||
[JsonPropertyName("owner")] public string? Owner { get; set; }
|
||
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
|
||
[JsonPropertyName("reportingGap")] public int? ReportingGap { get; set; }
|
||
[JsonPropertyName("comment")] public string? Comment { get; set; }
|
||
[JsonPropertyName("_politicianId")] public string? PoliticianId { get; set; }
|
||
[JsonPropertyName("_issuerId")] public long IssuerId { get; set; }
|
||
[JsonPropertyName("politician")] public CtPolitician? Politician { get; set; }
|
||
[JsonPropertyName("issuer")] public CtIssuer? Issuer { get; set; }
|
||
}
|
||
|
||
internal class CtPolitician
|
||
{
|
||
[JsonPropertyName("firstName")] public string? FirstName { get; set; }
|
||
[JsonPropertyName("lastName")] public string? LastName { get; set; }
|
||
[JsonPropertyName("party")] public string? Party { get; set; }
|
||
[JsonPropertyName("_stateId")] public string? StateId { get; set; }
|
||
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
|
||
}
|
||
|
||
internal class CtIssuer
|
||
{
|
||
[JsonPropertyName("issuerName")] public string? IssuerName { get; set; }
|
||
[JsonPropertyName("issuerTicker")] public string? IssuerTicker { get; set; }
|
||
[JsonPropertyName("sector")] public string? Sector { get; set; }
|
||
}
|
||
|
||
// ─── Scraper ─────────────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Scrapt capitoltrades.com über den eingebetteten Next.js RSC-Datenstrom.
|
||
///
|
||
/// Die Seite liefert Daten als escaped JSON-String innerhalb von:
|
||
/// self.__next_f.push([1,"...\"data\":[{\"_txId\":...}]..."])
|
||
///
|
||
/// Strategie:
|
||
/// 1. HTML-Seite abrufen
|
||
/// 2. Den escaped JSON-String mit dem "data"-Array extrahieren
|
||
/// 3. String un-escapen → gültiges JSON
|
||
/// 4. Als Liste von CtTrade deserialisieren
|
||
///
|
||
/// Rate-Limiting: 600ms zwischen Requests.
|
||
/// </summary>
|
||
public class CapitolTradesScraper
|
||
{
|
||
private readonly HttpClient _http;
|
||
private readonly LoggingService _logger;
|
||
|
||
private const string BaseUrl = "https://www.capitoltrades.com";
|
||
private const int RateDelayMs = 600;
|
||
|
||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||
{
|
||
PropertyNameCaseInsensitive = true,
|
||
NumberHandling = JsonNumberHandling.AllowReadingFromString
|
||
};
|
||
|
||
// Sucht das "data":[ Array im escaped RSC-String
|
||
// Das Muster ist: \"data\":[{\"_issuerId\":...,\"_txId\":...}]
|
||
private static readonly Regex DataArrayRx = new(
|
||
@"""data"":\s*\[",
|
||
RegexOptions.Compiled);
|
||
|
||
// Pagination: \"totalPages\":N
|
||
private static readonly Regex TotalPagesRx = new(
|
||
@"""totalPages""\s*:\s*(\d+)",
|
||
RegexOptions.Compiled);
|
||
|
||
public CapitolTradesScraper(LoggingService logger)
|
||
{
|
||
_logger = logger;
|
||
_http = new HttpClient();
|
||
_http.DefaultRequestHeaders.Add("User-Agent",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
|
||
_http.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9");
|
||
_http.DefaultRequestHeaders.Add("Accept",
|
||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||
_http.Timeout = TimeSpan.FromSeconds(30);
|
||
}
|
||
|
||
// ─── Öffentliche API ─────────────────────────────────────────────────────
|
||
|
||
public async Task<int> GetTotalPagesAsync(CancellationToken ct)
|
||
{
|
||
await RateDelay(ct);
|
||
var html = await _http.GetStringAsync($"{BaseUrl}/trades?pageSize=96&page=1", ct);
|
||
var unescaped = UnescapeHtmlPayload(html);
|
||
var match = TotalPagesRx.Match(unescaped);
|
||
|
||
if (match.Success)
|
||
{
|
||
var total = int.Parse(match.Groups[1].Value);
|
||
_logger.Info("CT", $"GetTotalPagesAsync: {total} Seiten gefunden.");
|
||
return total;
|
||
}
|
||
|
||
_logger.Warn("CT", "GetTotalPagesAsync: Konnte 'totalPages' im RSC-Stream nicht finden.");
|
||
_logger.Info("CT", $" Payload-Länge: {unescaped.Length}. Erste 200 Zeichen: {unescaped.Substring(0, Math.Min(200, unescaped.Length))}");
|
||
|
||
// Fallback: Wenn wir zumindest Trades gefunden haben, nehmen wir an es gibt mindestens 1 Seite.
|
||
// Falls wir totalPages nicht finden, riskieren wir hier 1, aber wir haben gewarnt.
|
||
return 1;
|
||
}
|
||
|
||
|
||
public async Task<(List<CongressTrade> Trades, HashSet<string> MemberBioIds)>
|
||
ScrapeTradesPageAsync(int page, CancellationToken ct)
|
||
{
|
||
await RateDelay(ct);
|
||
|
||
var url = $"{BaseUrl}/trades?pageSize=96&page={page}";
|
||
var html = await _http.GetStringAsync(url, ct);
|
||
|
||
return ParseTradesFromHtml(html, page);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Reine Parse-Logik: extrahiert Trades aus dem HTML (RSC-Stream) einer Trades-Seite.
|
||
/// Kein Netzwerkzugriff – dadurch offline gegen gespeicherte Fixtures testbar.
|
||
/// </summary>
|
||
public (List<CongressTrade> Trades, HashSet<string> MemberBioIds)
|
||
ParseTradesFromHtml(string html, int page = 1)
|
||
{
|
||
var trades = new List<CongressTrade>();
|
||
var memberIds = new HashSet<string>();
|
||
|
||
// HTML un-escapen → gültiges JSON
|
||
var payload = UnescapeHtmlPayload(html);
|
||
|
||
// "data":[ Block finden
|
||
var rawTrades = ExtractTradeArray(payload);
|
||
|
||
if (rawTrades == null || rawTrades.Count == 0)
|
||
{
|
||
_logger.Warn("CT", $"Seite {page}: Keine Trade-Daten im Payload gefunden.");
|
||
_logger.Info("CT", $" Payload-Länge: {payload.Length}, HTML-Länge: {html.Length}");
|
||
return (trades, memberIds);
|
||
}
|
||
|
||
foreach (var raw in rawTrades)
|
||
{
|
||
if (raw.TxId == 0) continue;
|
||
|
||
var bioId = raw.PoliticianId ?? "";
|
||
if (!string.IsNullOrEmpty(bioId))
|
||
memberIds.Add(bioId);
|
||
|
||
// Ticker: "AVGO:US" → "AVGO"
|
||
var ticker = raw.Issuer?.IssuerTicker ?? "";
|
||
if (ticker.Contains(':'))
|
||
ticker = ticker[..ticker.IndexOf(':')];
|
||
|
||
var trade = new CongressTrade
|
||
{
|
||
TradeId = raw.TxId.ToString(),
|
||
MemberBioId = bioId,
|
||
IssuerName = raw.Issuer?.IssuerName ?? "",
|
||
IssuerId = raw.IssuerId.ToString(),
|
||
Ticker = ticker,
|
||
TradeType = raw.TxType ?? "",
|
||
Chamber = raw.Chamber ?? "",
|
||
Owner = raw.Owner ?? "",
|
||
Value = raw.Value,
|
||
TradeDate = ParseDate(raw.TxDate),
|
||
PublishedDate = ParseDate(raw.PubDate),
|
||
DetailUrl = $"{BaseUrl}/trades/{raw.TxId}"
|
||
};
|
||
|
||
if (raw.Politician != null && !string.IsNullOrEmpty(bioId))
|
||
{
|
||
trade.MemberSnapshot = new CongressMember
|
||
{
|
||
BioId = bioId,
|
||
Name = $"{raw.Politician.FirstName} {raw.Politician.LastName}".Trim(),
|
||
Party = CapFirst(raw.Politician.Party ?? ""),
|
||
State = (raw.Politician.StateId ?? "").ToUpperInvariant(),
|
||
Chamber = CapFirst(raw.Politician.Chamber ?? ""),
|
||
ProfileUrl = $"{BaseUrl}/politicians/{bioId}"
|
||
};
|
||
}
|
||
|
||
trades.Add(trade);
|
||
}
|
||
|
||
_logger.Info("CT", $"Seite {page}: {trades.Count} Trades extrahiert.");
|
||
return (trades, memberIds);
|
||
}
|
||
|
||
// ─── Hilfsmethoden ────────────────────────────────────────────────────────
|
||
|
||
/// <summary>
|
||
/// Extrahiert und un-escaped den JSON-Payload aus dem RSC-Stream.
|
||
///
|
||
/// Die Seite enthält Blöcke wie:
|
||
/// self.__next_f.push([1,"...ESCAPED_JSON..."])
|
||
///
|
||
/// Die inneren Anführungszeichen sind mit \" escaped.
|
||
/// Wir konkatenieren alle Chunk-Strings und un-escapen sie.
|
||
/// </summary>
|
||
private static string UnescapeHtmlPayload(string html)
|
||
{
|
||
// Alle self.__next_f.push([1,"..."]) Chunks sammeln
|
||
// Regex: sucht [1,"(content)"] Blöcke
|
||
var sb = new System.Text.StringBuilder();
|
||
|
||
var searchFrom = 0;
|
||
const string marker = "self.__next_f.push([1,\"";
|
||
|
||
while (true)
|
||
{
|
||
var idx = html.IndexOf(marker, searchFrom, StringComparison.Ordinal);
|
||
if (idx < 0) break;
|
||
|
||
// Start des String-Inhalts
|
||
var contentStart = idx + marker.Length;
|
||
|
||
// Ende des Strings finden: suche das schließende "])
|
||
// Dabei escaped \" ignorieren
|
||
var contentEnd = FindEscapedStringEnd(html, contentStart);
|
||
if (contentEnd < 0) break;
|
||
|
||
var chunk = html[contentStart..contentEnd];
|
||
|
||
// Nur das äußere JSON-String-Escaping rückgängig machen:
|
||
// \" → " (Anführungszeichen)
|
||
// \/ → / (Forward-Slash, optional aber häufig)
|
||
// Unicode-Escapes (\u003e etc.) lässt der JSON-Deserializer selbst auf.
|
||
chunk = chunk.Replace("\\\"", "\"").Replace("\\/", "/");
|
||
|
||
sb.Append(chunk);
|
||
searchFrom = contentEnd + 2; // skip "])
|
||
}
|
||
|
||
return sb.ToString();
|
||
}
|
||
|
||
/// <summary>Findet das Ende eines JSON-Strings (schließendes " das nicht escaped ist).</summary>
|
||
private static int FindEscapedStringEnd(string s, int start)
|
||
{
|
||
for (var i = start; i < s.Length; i++)
|
||
{
|
||
if (s[i] == '\\') { i++; continue; } // überspringe escaped Zeichen
|
||
if (s[i] == '"') return i;
|
||
}
|
||
return -1;
|
||
}
|
||
|
||
/// <summary>Extrahiert das "data":[...] Array aus dem un-escaped Payload.</summary>
|
||
private static List<CtTrade>? ExtractTradeArray(string payload)
|
||
{
|
||
var match = DataArrayRx.Match(payload);
|
||
if (!match.Success) return null;
|
||
|
||
// Ab "[" das balancierte Array extrahieren
|
||
var arrayStart = match.Index + match.Length - 1; // auf dem "["
|
||
var json = ExtractBalancedBracket(payload, arrayStart, '[', ']');
|
||
if (json == null) return null;
|
||
|
||
try
|
||
{
|
||
return JsonSerializer.Deserialize<List<CtTrade>>(json, JsonOpts);
|
||
}
|
||
catch
|
||
{
|
||
return null;
|
||
}
|
||
}
|
||
|
||
private static string? ExtractBalancedBracket(string s, int start, char open, char close)
|
||
{
|
||
if (start >= s.Length || s[start] != open) return null;
|
||
|
||
var depth = 0;
|
||
var inStr = false;
|
||
var escape = false;
|
||
|
||
for (var i = start; i < s.Length; i++)
|
||
{
|
||
var c = s[i];
|
||
|
||
if (escape) { escape = false; continue; }
|
||
if (c == '\\' && inStr) { escape = true; continue; }
|
||
if (c == '"') { inStr = !inStr; continue; }
|
||
if (inStr) continue;
|
||
|
||
if (c == open) depth++;
|
||
else if (c == close)
|
||
{
|
||
depth--;
|
||
if (depth == 0) return s[start..(i + 1)];
|
||
}
|
||
}
|
||
|
||
return null;
|
||
}
|
||
|
||
private static DateOnly? ParseDate(string? s)
|
||
{
|
||
if (string.IsNullOrEmpty(s)) return null;
|
||
var datePart = s.Length > 10 ? s[..10] : s;
|
||
return DateOnly.TryParse(datePart, out var d) ? d : null;
|
||
}
|
||
|
||
private static string CapFirst(string s)
|
||
=> string.IsNullOrEmpty(s) ? s
|
||
: char.ToUpperInvariant(s[0]) + s[1..].ToLowerInvariant();
|
||
|
||
private static Task RateDelay(CancellationToken ct)
|
||
=> Task.Delay(RateDelayMs, ct);
|
||
}
|