Richard/IBKRTrader
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

@

Browse Source 
R1: Umbau auf Multi-Projekt-Struktur (Vorbild PolytraderSharp)

Kurskorrektur: Vorbild ist PolytraderSharp (C#), nicht die veraltete Python-Version.
Reiner Strukturumbau, Verhalten unveraendert.

- src/IBKRTrader.Core (classlib): Core-Code + UI-Contract (ModuleFormBase, WindowManager)
- src/IBKRTrader.Modules.CongressTrading (classlib, referenziert nur Core)
- Root: IBKRTrader.App (WinExe) referenziert Core + Modul
- tests/IBKRTrader.Tests: Referenzen auf Core + Modul
- Neue .slnx; ungenutztes HtmlAgilityPack entfernt
- docs/ARCHITECTURE.md: korrigierter Ziel-Stand + R-Phasenplan
- Build + 38/38 Tests gruen; App startet

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
@
This commit is contained in:
Richard
2026-07-27 20:26:25 +02:00
parent 2ad4b55db1
commit c20538c8d8
61 changed files with 171 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/IBKRTrader.Modules.CongressTrading/Scraper/CapitolTradesScraper.cs
+342
View File
@@ -0,0 +1,342 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using IBKRTrader.Core.Logging;
using IBKRTrader.Modules.CongressTrading.Models;
namespace IBKRTrader.Modules.CongressTrading.Scraper;
// ─── JSON-Modelle (exakte Feldnamen aus dem RSC-Stream) ──────────────────────
internal class CtApiWrapper
{
[JsonPropertyName("data")] public List<CtTrade>? Data { get; set; }
[JsonPropertyName("meta")] public CtMeta? Meta { get; set; }
}
internal class CtMeta
{
[JsonPropertyName("totalPages")] public int TotalPages { get; set; }
[JsonPropertyName("pageSize")] public int PageSize { get; set; }
[JsonPropertyName("totalCount")] public int TotalCount { get; set; }
[JsonPropertyName("page")] public int Page { get; set; }
}
internal class CtTrade
{
[JsonPropertyName("_txId")] public long TxId { get; set; }
[JsonPropertyName("txDate")] public string? TxDate { get; set; }
[JsonPropertyName("pubDate")] public string? PubDate { get; set; }
[JsonPropertyName("txType")] public string? TxType { get; set; }
[JsonPropertyName("value")] public decimal? Value { get; set; }
[JsonPropertyName("price")] public decimal? Price { get; set; }
[JsonPropertyName("owner")] public string? Owner { get; set; }
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
[JsonPropertyName("reportingGap")] public int? ReportingGap { get; set; }
[JsonPropertyName("comment")] public string? Comment { get; set; }
[JsonPropertyName("_politicianId")] public string? PoliticianId { get; set; }
[JsonPropertyName("_issuerId")] public long IssuerId { get; set; }
[JsonPropertyName("politician")] public CtPolitician? Politician { get; set; }
[JsonPropertyName("issuer")] public CtIssuer? Issuer { get; set; }
}
internal class CtPolitician
{
[JsonPropertyName("firstName")] public string? FirstName { get; set; }
[JsonPropertyName("lastName")] public string? LastName { get; set; }
[JsonPropertyName("party")] public string? Party { get; set; }
[JsonPropertyName("_stateId")] public string? StateId { get; set; }
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
}
internal class CtIssuer
{
[JsonPropertyName("issuerName")] public string? IssuerName { get; set; }
[JsonPropertyName("issuerTicker")] public string? IssuerTicker { get; set; }
[JsonPropertyName("sector")] public string? Sector { get; set; }
}
// ─── Scraper ─────────────────────────────────────────────────────────────────
/// <summary>
/// Scrapt capitoltrades.com über den eingebetteten Next.js RSC-Datenstrom.
///
/// Die Seite liefert Daten als escaped JSON-String innerhalb von:
/// self.__next_f.push([1,"...\"data\":[{\"_txId\":...}]..."])
///
/// Strategie:
/// 1. HTML-Seite abrufen
/// 2. Den escaped JSON-String mit dem "data"-Array extrahieren
/// 3. String un-escapen → gültiges JSON
/// 4. Als Liste von CtTrade deserialisieren
///
/// Rate-Limiting: 600ms zwischen Requests.
/// </summary>
public class CapitolTradesScraper
{
private readonly HttpClient _http;
private readonly LoggingService _logger;
private const string BaseUrl = "https://www.capitoltrades.com";
private const int RateDelayMs = 600;
private static readonly JsonSerializerOptions JsonOpts = new()
{
PropertyNameCaseInsensitive = true,
NumberHandling = JsonNumberHandling.AllowReadingFromString
};
// Sucht das "data":[ Array im escaped RSC-String
// Das Muster ist: \"data\":[{\"_issuerId\":...,\"_txId\":...}]
private static readonly Regex DataArrayRx = new(
@"""data"":\s*\[",
RegexOptions.Compiled);
// Pagination: \"totalPages\":N
private static readonly Regex TotalPagesRx = new(
@"""totalPages""\s*:\s*(\d+)",
RegexOptions.Compiled);
public CapitolTradesScraper(LoggingService logger)
{
_logger = logger;
_http = new HttpClient();
_http.DefaultRequestHeaders.Add("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
_http.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9");
_http.DefaultRequestHeaders.Add("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
_http.Timeout = TimeSpan.FromSeconds(30);
}
// ─── Öffentliche API ─────────────────────────────────────────────────────
public async Task<int> GetTotalPagesAsync(CancellationToken ct)
{
await RateDelay(ct);
var html = await _http.GetStringAsync($"{BaseUrl}/trades?pageSize=96&page=1", ct);
var unescaped = UnescapeHtmlPayload(html);
var match = TotalPagesRx.Match(unescaped);
if (match.Success)
{
var total = int.Parse(match.Groups[1].Value);
_logger.Info("CT", $"GetTotalPagesAsync: {total} Seiten gefunden.");
return total;
}
_logger.Warn("CT", "GetTotalPagesAsync: Konnte 'totalPages' im RSC-Stream nicht finden.");
_logger.Info("CT", $" Payload-Länge: {unescaped.Length}. Erste 200 Zeichen: {unescaped.Substring(0, Math.Min(200, unescaped.Length))}");
// Fallback: Wenn wir zumindest Trades gefunden haben, nehmen wir an es gibt mindestens 1 Seite.
// Falls wir totalPages nicht finden, riskieren wir hier 1, aber wir haben gewarnt.
return 1;
}
public async Task<(List<CongressTrade> Trades, HashSet<string> MemberBioIds)>
ScrapeTradesPageAsync(int page, CancellationToken ct)
{
await RateDelay(ct);
var url = $"{BaseUrl}/trades?pageSize=96&page={page}";
var html = await _http.GetStringAsync(url, ct);
return ParseTradesFromHtml(html, page);
}
/// <summary>
/// Reine Parse-Logik: extrahiert Trades aus dem HTML (RSC-Stream) einer Trades-Seite.
/// Kein Netzwerkzugriff – dadurch offline gegen gespeicherte Fixtures testbar.
/// </summary>
public (List<CongressTrade> Trades, HashSet<string> MemberBioIds)
ParseTradesFromHtml(string html, int page = 1)
{
var trades = new List<CongressTrade>();
var memberIds = new HashSet<string>();
// HTML un-escapen → gültiges JSON
var payload = UnescapeHtmlPayload(html);
// "data":[ Block finden
var rawTrades = ExtractTradeArray(payload);
if (rawTrades == null || rawTrades.Count == 0)
{
_logger.Warn("CT", $"Seite {page}: Keine Trade-Daten im Payload gefunden.");
_logger.Info("CT", $" Payload-Länge: {payload.Length}, HTML-Länge: {html.Length}");
return (trades, memberIds);
}
foreach (var raw in rawTrades)
{
if (raw.TxId == 0) continue;
var bioId = raw.PoliticianId ?? "";
if (!string.IsNullOrEmpty(bioId))
memberIds.Add(bioId);
// Ticker: "AVGO:US" → "AVGO"
var ticker = raw.Issuer?.IssuerTicker ?? "";
if (ticker.Contains(':'))
ticker = ticker[..ticker.IndexOf(':')];
var trade = new CongressTrade
{
TradeId = raw.TxId.ToString(),
MemberBioId = bioId,
IssuerName = raw.Issuer?.IssuerName ?? "",
IssuerId = raw.IssuerId.ToString(),
Ticker = ticker,
TradeType = raw.TxType ?? "",
Chamber = raw.Chamber ?? "",
Owner = raw.Owner ?? "",
Value = raw.Value,
TradeDate = ParseDate(raw.TxDate),
PublishedDate = ParseDate(raw.PubDate),
DetailUrl = $"{BaseUrl}/trades/{raw.TxId}"
};
if (raw.Politician != null && !string.IsNullOrEmpty(bioId))
{
trade.MemberSnapshot = new CongressMember
{
BioId = bioId,
Name = $"{raw.Politician.FirstName} {raw.Politician.LastName}".Trim(),
Party = CapFirst(raw.Politician.Party ?? ""),
State = (raw.Politician.StateId ?? "").ToUpperInvariant(),
Chamber = CapFirst(raw.Politician.Chamber ?? ""),
ProfileUrl = $"{BaseUrl}/politicians/{bioId}"
};
}
trades.Add(trade);
}
_logger.Info("CT", $"Seite {page}: {trades.Count} Trades extrahiert.");
return (trades, memberIds);
}
// ─── Hilfsmethoden ────────────────────────────────────────────────────────
/// <summary>
/// Extrahiert und un-escaped den JSON-Payload aus dem RSC-Stream.
///
/// Die Seite enthält Blöcke wie:
/// self.__next_f.push([1,"...ESCAPED_JSON..."])
///
/// Die inneren Anführungszeichen sind mit \" escaped.
/// Wir konkatenieren alle Chunk-Strings und un-escapen sie.
/// </summary>
private static string UnescapeHtmlPayload(string html)
{
// Alle self.__next_f.push([1,"..."]) Chunks sammeln
// Regex: sucht [1,"(content)"] Blöcke
var sb = new System.Text.StringBuilder();
var searchFrom = 0;
const string marker = "self.__next_f.push([1,\"";
while (true)
{
var idx = html.IndexOf(marker, searchFrom, StringComparison.Ordinal);
if (idx < 0) break;
// Start des String-Inhalts
var contentStart = idx + marker.Length;
// Ende des Strings finden: suche das schließende "])
// Dabei escaped \" ignorieren
var contentEnd = FindEscapedStringEnd(html, contentStart);
if (contentEnd < 0) break;
var chunk = html[contentStart..contentEnd];
// Nur das äußere JSON-String-Escaping rückgängig machen:
// \" → " (Anführungszeichen)
// \/ → / (Forward-Slash, optional aber häufig)
// Unicode-Escapes (\u003e etc.) lässt der JSON-Deserializer selbst auf.
chunk = chunk.Replace("\\\"", "\"").Replace("\\/", "/");
sb.Append(chunk);
searchFrom = contentEnd + 2; // skip "])
}
return sb.ToString();
}
/// <summary>Findet das Ende eines JSON-Strings (schließendes " das nicht escaped ist).</summary>
private static int FindEscapedStringEnd(string s, int start)
{
for (var i = start; i < s.Length; i++)
{
if (s[i] == '\\') { i++; continue; } // überspringe escaped Zeichen
if (s[i] == '"') return i;
}
return -1;
}
/// <summary>Extrahiert das "data":[...] Array aus dem un-escaped Payload.</summary>
private static List<CtTrade>? ExtractTradeArray(string payload)
{
var match = DataArrayRx.Match(payload);
if (!match.Success) return null;
// Ab "[" das balancierte Array extrahieren
var arrayStart = match.Index + match.Length - 1; // auf dem "["
var json = ExtractBalancedBracket(payload, arrayStart, '[', ']');
if (json == null) return null;
try
{
return JsonSerializer.Deserialize<List<CtTrade>>(json, JsonOpts);
}
catch
{
return null;
}
}
private static string? ExtractBalancedBracket(string s, int start, char open, char close)
{
if (start >= s.Length || s[start] != open) return null;
var depth = 0;
var inStr = false;
var escape = false;
for (var i = start; i < s.Length; i++)
{
var c = s[i];
if (escape) { escape = false; continue; }
if (c == '\\' && inStr) { escape = true; continue; }
if (c == '"') { inStr = !inStr; continue; }
if (inStr) continue;
if (c == open) depth++;
else if (c == close)
{
depth--;
if (depth == 0) return s[start..(i + 1)];
}
}
return null;
}
private static DateOnly? ParseDate(string? s)
{
if (string.IsNullOrEmpty(s)) return null;
var datePart = s.Length > 10 ? s[..10] : s;
return DateOnly.TryParse(datePart, out var d) ? d : null;
}
private static string CapFirst(string s)
=> string.IsNullOrEmpty(s) ? s
: char.ToUpperInvariant(s[0]) + s[1..].ToLowerInvariant();
private static Task RateDelay(CancellationToken ct)
=> Task.Delay(RateDelayMs, ct);
}