RichardandClaude Opus 4.8
ebeb035e92
.NET WinForms-Anwendung (Core, Modules/CongressTrading, UI). Enthaelt .gitignore und settings.example.json als Konfigurationsvorlage. Echte settings.json mit Zugangsdaten ist bewusst ausgeschlossen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
333 lines
13 KiB
C#
333 lines
13 KiB
C#
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
using System.Text.RegularExpressions;
|
|
using IBKRTrader.Core.Logging;
|
|
using IBKRTrader.Modules.CongressTrading.Models;
|
|
|
|
namespace IBKRTrader.Modules.CongressTrading.Scraper;
|
|
|
|
// ─── JSON-Modelle (exakte Feldnamen aus dem RSC-Stream) ──────────────────────
|
|
|
|
internal class CtApiWrapper
|
|
{
|
|
[JsonPropertyName("data")] public List<CtTrade>? Data { get; set; }
|
|
[JsonPropertyName("meta")] public CtMeta? Meta { get; set; }
|
|
}
|
|
|
|
internal class CtMeta
|
|
{
|
|
[JsonPropertyName("totalPages")] public int TotalPages { get; set; }
|
|
[JsonPropertyName("pageSize")] public int PageSize { get; set; }
|
|
[JsonPropertyName("totalCount")] public int TotalCount { get; set; }
|
|
[JsonPropertyName("page")] public int Page { get; set; }
|
|
}
|
|
|
|
internal class CtTrade
|
|
{
|
|
[JsonPropertyName("_txId")] public long TxId { get; set; }
|
|
[JsonPropertyName("txDate")] public string? TxDate { get; set; }
|
|
[JsonPropertyName("pubDate")] public string? PubDate { get; set; }
|
|
[JsonPropertyName("txType")] public string? TxType { get; set; }
|
|
[JsonPropertyName("value")] public decimal? Value { get; set; }
|
|
[JsonPropertyName("price")] public decimal? Price { get; set; }
|
|
[JsonPropertyName("owner")] public string? Owner { get; set; }
|
|
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
|
|
[JsonPropertyName("reportingGap")] public int? ReportingGap { get; set; }
|
|
[JsonPropertyName("comment")] public string? Comment { get; set; }
|
|
[JsonPropertyName("_politicianId")] public string? PoliticianId { get; set; }
|
|
[JsonPropertyName("_issuerId")] public long IssuerId { get; set; }
|
|
[JsonPropertyName("politician")] public CtPolitician? Politician { get; set; }
|
|
[JsonPropertyName("issuer")] public CtIssuer? Issuer { get; set; }
|
|
}
|
|
|
|
internal class CtPolitician
|
|
{
|
|
[JsonPropertyName("firstName")] public string? FirstName { get; set; }
|
|
[JsonPropertyName("lastName")] public string? LastName { get; set; }
|
|
[JsonPropertyName("party")] public string? Party { get; set; }
|
|
[JsonPropertyName("_stateId")] public string? StateId { get; set; }
|
|
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
|
|
}
|
|
|
|
internal class CtIssuer
|
|
{
|
|
[JsonPropertyName("issuerName")] public string? IssuerName { get; set; }
|
|
[JsonPropertyName("issuerTicker")] public string? IssuerTicker { get; set; }
|
|
[JsonPropertyName("sector")] public string? Sector { get; set; }
|
|
}
|
|
|
|
// ─── Scraper ─────────────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Scrapt capitoltrades.com über den eingebetteten Next.js RSC-Datenstrom.
|
|
///
|
|
/// Die Seite liefert Daten als escaped JSON-String innerhalb von:
|
|
/// self.__next_f.push([1,"...\"data\":[{\"_txId\":...}]..."])
|
|
///
|
|
/// Strategie:
|
|
/// 1. HTML-Seite abrufen
|
|
/// 2. Den escaped JSON-String mit dem "data"-Array extrahieren
|
|
/// 3. String un-escapen → gültiges JSON
|
|
/// 4. Als Liste von CtTrade deserialisieren
|
|
///
|
|
/// Rate-Limiting: 600ms zwischen Requests.
|
|
/// </summary>
|
|
public class CapitolTradesScraper
|
|
{
|
|
private readonly HttpClient _http;
|
|
private readonly LoggingService _logger;
|
|
|
|
private const string BaseUrl = "https://www.capitoltrades.com";
|
|
private const int RateDelayMs = 600;
|
|
|
|
private static readonly JsonSerializerOptions JsonOpts = new()
|
|
{
|
|
PropertyNameCaseInsensitive = true,
|
|
NumberHandling = JsonNumberHandling.AllowReadingFromString
|
|
};
|
|
|
|
// Sucht das "data":[ Array im escaped RSC-String
|
|
// Das Muster ist: \"data\":[{\"_issuerId\":...,\"_txId\":...}]
|
|
private static readonly Regex DataArrayRx = new(
|
|
@"""data"":\s*\[",
|
|
RegexOptions.Compiled);
|
|
|
|
// Pagination: \"totalPages\":N
|
|
private static readonly Regex TotalPagesRx = new(
|
|
@"""totalPages""\s*:\s*(\d+)",
|
|
RegexOptions.Compiled);
|
|
|
|
public CapitolTradesScraper(LoggingService logger)
|
|
{
|
|
_logger = logger;
|
|
_http = new HttpClient();
|
|
_http.DefaultRequestHeaders.Add("User-Agent",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
|
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
|
|
_http.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9");
|
|
_http.DefaultRequestHeaders.Add("Accept",
|
|
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
|
_http.Timeout = TimeSpan.FromSeconds(30);
|
|
}
|
|
|
|
// ─── Öffentliche API ─────────────────────────────────────────────────────
|
|
|
|
public async Task<int> GetTotalPagesAsync(CancellationToken ct)
|
|
{
|
|
await RateDelay(ct);
|
|
var html = await _http.GetStringAsync($"{BaseUrl}/trades?pageSize=96&page=1", ct);
|
|
var unescaped = UnescapeHtmlPayload(html);
|
|
var match = TotalPagesRx.Match(unescaped);
|
|
|
|
if (match.Success)
|
|
{
|
|
var total = int.Parse(match.Groups[1].Value);
|
|
_logger.Info("CT", $"GetTotalPagesAsync: {total} Seiten gefunden.");
|
|
return total;
|
|
}
|
|
|
|
_logger.Warn("CT", "GetTotalPagesAsync: Konnte 'totalPages' im RSC-Stream nicht finden.");
|
|
_logger.Info("CT", $" Payload-Länge: {unescaped.Length}. Erste 200 Zeichen: {unescaped.Substring(0, Math.Min(200, unescaped.Length))}");
|
|
|
|
// Fallback: Wenn wir zumindest Trades gefunden haben, nehmen wir an es gibt mindestens 1 Seite.
|
|
// Falls wir totalPages nicht finden, riskieren wir hier 1, aber wir haben gewarnt.
|
|
return 1;
|
|
}
|
|
|
|
|
|
public async Task<(List<CongressTrade> Trades, HashSet<string> MemberBioIds)>
|
|
ScrapeTradesPageAsync(int page, CancellationToken ct)
|
|
{
|
|
await RateDelay(ct);
|
|
|
|
var url = $"{BaseUrl}/trades?pageSize=96&page={page}";
|
|
var html = await _http.GetStringAsync(url, ct);
|
|
|
|
var trades = new List<CongressTrade>();
|
|
var memberIds = new HashSet<string>();
|
|
|
|
// HTML un-escapen → gültiges JSON
|
|
var payload = UnescapeHtmlPayload(html);
|
|
|
|
// "data":[ Block finden
|
|
var rawTrades = ExtractTradeArray(payload);
|
|
|
|
if (rawTrades == null || rawTrades.Count == 0)
|
|
{
|
|
_logger.Warn("CT", $"Seite {page}: Keine Trade-Daten im Payload gefunden.");
|
|
_logger.Info("CT", $" Payload-Länge: {payload.Length}, HTML-Länge: {html.Length}");
|
|
return (trades, memberIds);
|
|
}
|
|
|
|
foreach (var raw in rawTrades)
|
|
{
|
|
if (raw.TxId == 0) continue;
|
|
|
|
var bioId = raw.PoliticianId ?? "";
|
|
if (!string.IsNullOrEmpty(bioId))
|
|
memberIds.Add(bioId);
|
|
|
|
// Ticker: "AVGO:US" → "AVGO"
|
|
var ticker = raw.Issuer?.IssuerTicker ?? "";
|
|
if (ticker.Contains(':'))
|
|
ticker = ticker[..ticker.IndexOf(':')];
|
|
|
|
var trade = new CongressTrade
|
|
{
|
|
TradeId = raw.TxId.ToString(),
|
|
MemberBioId = bioId,
|
|
IssuerName = raw.Issuer?.IssuerName ?? "",
|
|
IssuerId = raw.IssuerId.ToString(),
|
|
Ticker = ticker,
|
|
TradeType = raw.TxType ?? "",
|
|
Chamber = raw.Chamber ?? "",
|
|
Owner = raw.Owner ?? "",
|
|
Value = raw.Value,
|
|
TradeDate = ParseDate(raw.TxDate),
|
|
PublishedDate = ParseDate(raw.PubDate),
|
|
DetailUrl = $"{BaseUrl}/trades/{raw.TxId}"
|
|
};
|
|
|
|
if (raw.Politician != null && !string.IsNullOrEmpty(bioId))
|
|
{
|
|
trade.MemberSnapshot = new CongressMember
|
|
{
|
|
BioId = bioId,
|
|
Name = $"{raw.Politician.FirstName} {raw.Politician.LastName}".Trim(),
|
|
Party = CapFirst(raw.Politician.Party ?? ""),
|
|
State = (raw.Politician.StateId ?? "").ToUpperInvariant(),
|
|
Chamber = CapFirst(raw.Politician.Chamber ?? ""),
|
|
ProfileUrl = $"{BaseUrl}/politicians/{bioId}"
|
|
};
|
|
}
|
|
|
|
trades.Add(trade);
|
|
}
|
|
|
|
_logger.Info("CT", $"Seite {page}: {trades.Count} Trades extrahiert.");
|
|
return (trades, memberIds);
|
|
}
|
|
|
|
// ─── Hilfsmethoden ────────────────────────────────────────────────────────
|
|
|
|
/// <summary>
|
|
/// Extrahiert und un-escaped den JSON-Payload aus dem RSC-Stream.
|
|
///
|
|
/// Die Seite enthält Blöcke wie:
|
|
/// self.__next_f.push([1,"...ESCAPED_JSON..."])
|
|
///
|
|
/// Die inneren Anführungszeichen sind mit \" escaped.
|
|
/// Wir konkatenieren alle Chunk-Strings und un-escapen sie.
|
|
/// </summary>
|
|
private static string UnescapeHtmlPayload(string html)
|
|
{
|
|
// Alle self.__next_f.push([1,"..."]) Chunks sammeln
|
|
// Regex: sucht [1,"(content)"] Blöcke
|
|
var sb = new System.Text.StringBuilder();
|
|
|
|
var searchFrom = 0;
|
|
const string marker = "self.__next_f.push([1,\"";
|
|
|
|
while (true)
|
|
{
|
|
var idx = html.IndexOf(marker, searchFrom, StringComparison.Ordinal);
|
|
if (idx < 0) break;
|
|
|
|
// Start des String-Inhalts
|
|
var contentStart = idx + marker.Length;
|
|
|
|
// Ende des Strings finden: suche das schließende "])
|
|
// Dabei escaped \" ignorieren
|
|
var contentEnd = FindEscapedStringEnd(html, contentStart);
|
|
if (contentEnd < 0) break;
|
|
|
|
var chunk = html[contentStart..contentEnd];
|
|
|
|
// Nur das äußere JSON-String-Escaping rückgängig machen:
|
|
// \" → " (Anführungszeichen)
|
|
// \/ → / (Forward-Slash, optional aber häufig)
|
|
// Unicode-Escapes (\u003e etc.) lässt der JSON-Deserializer selbst auf.
|
|
chunk = chunk.Replace("\\\"", "\"").Replace("\\/", "/");
|
|
|
|
sb.Append(chunk);
|
|
searchFrom = contentEnd + 2; // skip "])
|
|
}
|
|
|
|
return sb.ToString();
|
|
}
|
|
|
|
/// <summary>Findet das Ende eines JSON-Strings (schließendes " das nicht escaped ist).</summary>
|
|
private static int FindEscapedStringEnd(string s, int start)
|
|
{
|
|
for (var i = start; i < s.Length; i++)
|
|
{
|
|
if (s[i] == '\\') { i++; continue; } // überspringe escaped Zeichen
|
|
if (s[i] == '"') return i;
|
|
}
|
|
return -1;
|
|
}
|
|
|
|
/// <summary>Extrahiert das "data":[...] Array aus dem un-escaped Payload.</summary>
|
|
private static List<CtTrade>? ExtractTradeArray(string payload)
|
|
{
|
|
var match = DataArrayRx.Match(payload);
|
|
if (!match.Success) return null;
|
|
|
|
// Ab "[" das balancierte Array extrahieren
|
|
var arrayStart = match.Index + match.Length - 1; // auf dem "["
|
|
var json = ExtractBalancedBracket(payload, arrayStart, '[', ']');
|
|
if (json == null) return null;
|
|
|
|
try
|
|
{
|
|
return JsonSerializer.Deserialize<List<CtTrade>>(json, JsonOpts);
|
|
}
|
|
catch
|
|
{
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private static string? ExtractBalancedBracket(string s, int start, char open, char close)
|
|
{
|
|
if (start >= s.Length || s[start] != open) return null;
|
|
|
|
var depth = 0;
|
|
var inStr = false;
|
|
var escape = false;
|
|
|
|
for (var i = start; i < s.Length; i++)
|
|
{
|
|
var c = s[i];
|
|
|
|
if (escape) { escape = false; continue; }
|
|
if (c == '\\' && inStr) { escape = true; continue; }
|
|
if (c == '"') { inStr = !inStr; continue; }
|
|
if (inStr) continue;
|
|
|
|
if (c == open) depth++;
|
|
else if (c == close)
|
|
{
|
|
depth--;
|
|
if (depth == 0) return s[start..(i + 1)];
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
private static DateOnly? ParseDate(string? s)
|
|
{
|
|
if (string.IsNullOrEmpty(s)) return null;
|
|
var datePart = s.Length > 10 ? s[..10] : s;
|
|
return DateOnly.TryParse(datePart, out var d) ? d : null;
|
|
}
|
|
|
|
private static string CapFirst(string s)
|
|
=> string.IsNullOrEmpty(s) ? s
|
|
: char.ToUpperInvariant(s[0]) + s[1..].ToLowerInvariant();
|
|
|
|
private static Task RateDelay(CancellationToken ct)
|
|
=> Task.Delay(RateDelayMs, ct);
|
|
}
|