IBKRTrader/src/IBKRTrader.Modules.CongressTrading/Scraper/CapitolTradesScraper.cs

using System.Globalization;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using IBKRTrader.Core.Logging;
using IBKRTrader.Modules.CongressTrading.Models;

namespace IBKRTrader.Modules.CongressTrading.Scraper;

// ─── JSON-Modelle (exakte Feldnamen aus dem RSC-Stream) ──────────────────────

internal class CtApiWrapper
{
    [JsonPropertyName("data")]        public List<CtTrade>? Data       { get; set; }
    [JsonPropertyName("meta")]        public CtMeta?        Meta       { get; set; }
}

internal class CtMeta
{
    [JsonPropertyName("totalPages")]  public int TotalPages  { get; set; }
    [JsonPropertyName("pageSize")]    public int PageSize    { get; set; }
    [JsonPropertyName("totalCount")]  public int TotalCount  { get; set; }
    [JsonPropertyName("page")]        public int Page        { get; set; }
}

internal class CtTrade
{
    [JsonPropertyName("_txId")]         public long          TxId         { get; set; }
    [JsonPropertyName("txDate")]        public string?       TxDate       { get; set; }
    [JsonPropertyName("pubDate")]       public string?       PubDate      { get; set; }
    [JsonPropertyName("txType")]        public string?       TxType       { get; set; }
    [JsonPropertyName("value")]         public decimal?      Value        { get; set; }
    [JsonPropertyName("price")]         public decimal?      Price        { get; set; }
    [JsonPropertyName("owner")]         public string?       Owner        { get; set; }
    [JsonPropertyName("chamber")]       public string?       Chamber      { get; set; }
    [JsonPropertyName("reportingGap")]  public int?          ReportingGap { get; set; }
    [JsonPropertyName("comment")]       public string?       Comment      { get; set; }
    [JsonPropertyName("_politicianId")] public string?       PoliticianId { get; set; }
    [JsonPropertyName("_issuerId")]     public long          IssuerId     { get; set; }
    [JsonPropertyName("politician")]    public CtPolitician? Politician   { get; set; }
    [JsonPropertyName("issuer")]        public CtIssuer?     Issuer       { get; set; }
}

internal class CtPolitician
{
    [JsonPropertyName("firstName")]  public string? FirstName { get; set; }
    [JsonPropertyName("lastName")]   public string? LastName  { get; set; }
    [JsonPropertyName("party")]      public string? Party     { get; set; }
    [JsonPropertyName("_stateId")]   public string? StateId   { get; set; }
    [JsonPropertyName("chamber")]    public string? Chamber   { get; set; }
}

internal class CtIssuer
{
    [JsonPropertyName("issuerName")]    public string? IssuerName   { get; set; }
    [JsonPropertyName("issuerTicker")]  public string? IssuerTicker { get; set; }
    [JsonPropertyName("sector")]        public string? Sector       { get; set; }
}

// ─── Scraper ─────────────────────────────────────────────────────────────────

/// <summary>
/// Scrapt capitoltrades.com über den eingebetteten Next.js RSC-Datenstrom.
///
/// Die Seite liefert Daten als escaped JSON-String innerhalb von:
///   self.__next_f.push([1,"...\"data\":[{\"_txId\":...}]..."])
///
/// Strategie:
///   1. HTML-Seite abrufen
///   2. Den escaped JSON-String mit dem "data"-Array extrahieren
///   3. String un-escapen → gültiges JSON
///   4. Als Liste von CtTrade deserialisieren
///
/// Rate-Limiting: 600ms zwischen Requests.
/// </summary>
public class CapitolTradesScraper
{
    private readonly HttpClient     _http;
    private readonly LoggingService _logger;

    private const string BaseUrl     = "https://www.capitoltrades.com";
    private const int    RateDelayMs = 600;

    private static readonly JsonSerializerOptions JsonOpts = new()
    {
        PropertyNameCaseInsensitive = true,
        NumberHandling = JsonNumberHandling.AllowReadingFromString
    };

    // Sucht das "data":[ Array im escaped RSC-String
    // Das Muster ist: \"data\":[{\"_issuerId\":...,\"_txId\":...}]
    private static readonly Regex DataArrayRx = new(
        @"""data"":\s*\[",
        RegexOptions.Compiled);

    // Pagination: \"totalPages\":N
    private static readonly Regex TotalPagesRx = new(
        @"""totalPages""\s*:\s*(\d+)",
        RegexOptions.Compiled);

    public CapitolTradesScraper(LoggingService logger)
    {
        _logger = logger;
        _http   = new HttpClient();
        _http.DefaultRequestHeaders.Add("User-Agent",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
            "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
        _http.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9");
        _http.DefaultRequestHeaders.Add("Accept",
            "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
        _http.Timeout = TimeSpan.FromSeconds(30);
    }

    // ─── Öffentliche API ─────────────────────────────────────────────────────

    public async Task<int> GetTotalPagesAsync(CancellationToken ct)
    {
        await RateDelay(ct);
        var html      = await _http.GetStringAsync($"{BaseUrl}/trades?pageSize=96&page=1", ct);
        var unescaped = UnescapeHtmlPayload(html);
        var match     = TotalPagesRx.Match(unescaped);
        
        if (match.Success)
        {
            var total = int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture);
            _logger.Info("CT", $"GetTotalPagesAsync: {total} Seiten gefunden.");
            return total;
        }

        _logger.Warn("CT", "GetTotalPagesAsync: Konnte 'totalPages' im RSC-Stream nicht finden.");
        _logger.Info("CT", $"  Payload-Länge: {unescaped.Length}. Erste 200 Zeichen: {unescaped.Substring(0, Math.Min(200, unescaped.Length))}");
        
        // Fallback: Wenn wir zumindest Trades gefunden haben, nehmen wir an es gibt mindestens 1 Seite.
        // Falls wir totalPages nicht finden, riskieren wir hier 1, aber wir haben gewarnt.
        return 1;
    }


    public async Task<(List<CongressTrade> Trades, HashSet<string> MemberBioIds)>
        ScrapeTradesPageAsync(int page, CancellationToken ct)
    {
        await RateDelay(ct);

        var url  = $"{BaseUrl}/trades?pageSize=96&page={page}";
        var html = await _http.GetStringAsync(url, ct);

        return ParseTradesFromHtml(html, page);
    }

    /// <summary>
    /// Reine Parse-Logik: extrahiert Trades aus dem HTML (RSC-Stream) einer Trades-Seite.
    /// Kein Netzwerkzugriff – dadurch offline gegen gespeicherte Fixtures testbar.
    /// </summary>
    public (List<CongressTrade> Trades, HashSet<string> MemberBioIds)
        ParseTradesFromHtml(string html, int page = 1)
    {
        var trades    = new List<CongressTrade>();
        var memberIds = new HashSet<string>();

        // HTML un-escapen → gültiges JSON
        var payload = UnescapeHtmlPayload(html);

        // "data":[ Block finden
        var rawTrades = ExtractTradeArray(payload);

        if (rawTrades == null || rawTrades.Count == 0)
        {
            _logger.Warn("CT", $"Seite {page}: Keine Trade-Daten im Payload gefunden.");
            _logger.Info("CT", $"  Payload-Länge: {payload.Length}, HTML-Länge: {html.Length}");
            return (trades, memberIds);
        }

        foreach (var raw in rawTrades)
        {
            if (raw.TxId == 0) continue;

            var bioId = raw.PoliticianId ?? "";
            if (!string.IsNullOrEmpty(bioId))
                memberIds.Add(bioId);

            // Ticker: "AVGO:US" → "AVGO"
            var ticker = raw.Issuer?.IssuerTicker ?? "";
            if (ticker.Contains(':'))
                ticker = ticker[..ticker.IndexOf(':')];

            var trade = new CongressTrade
            {
                TradeId       = raw.TxId.ToString(),
                MemberBioId   = bioId,
                IssuerName    = raw.Issuer?.IssuerName ?? "",
                IssuerId      = raw.IssuerId.ToString(),
                Ticker        = ticker,
                TradeType     = raw.TxType ?? "",
                Chamber       = raw.Chamber ?? "",
                Owner         = raw.Owner ?? "",
                Value         = raw.Value,
                TradeDate     = ParseDate(raw.TxDate),
                PublishedDate = ParseDate(raw.PubDate),
                DetailUrl     = $"{BaseUrl}/trades/{raw.TxId}"
            };

            if (raw.Politician != null && !string.IsNullOrEmpty(bioId))
            {
                trade.MemberSnapshot = new CongressMember
                {
                    BioId      = bioId,
                    Name       = $"{raw.Politician.FirstName} {raw.Politician.LastName}".Trim(),
                    Party      = CapFirst(raw.Politician.Party ?? ""),
                    State      = (raw.Politician.StateId ?? "").ToUpperInvariant(),
                    Chamber    = CapFirst(raw.Politician.Chamber ?? ""),
                    ProfileUrl = $"{BaseUrl}/politicians/{bioId}"
                };
            }

            trades.Add(trade);
        }

        _logger.Info("CT", $"Seite {page}: {trades.Count} Trades extrahiert.");
        return (trades, memberIds);
    }

    // ─── Hilfsmethoden ────────────────────────────────────────────────────────

    /// <summary>
    /// Extrahiert und un-escaped den JSON-Payload aus dem RSC-Stream.
    ///
    /// Die Seite enthält Blöcke wie:
    ///   self.__next_f.push([1,"...ESCAPED_JSON..."])
    ///
    /// Die inneren Anführungszeichen sind mit \" escaped.
    /// Wir konkatenieren alle Chunk-Strings und un-escapen sie.
    /// </summary>
    private static string UnescapeHtmlPayload(string html)
    {
        // Alle self.__next_f.push([1,"..."]) Chunks sammeln
        // Regex: sucht [1,"(content)"] Blöcke
        var sb = new System.Text.StringBuilder();

        var searchFrom = 0;
        const string marker = "self.__next_f.push([1,\"";

        while (true)
        {
            var idx = html.IndexOf(marker, searchFrom, StringComparison.Ordinal);
            if (idx < 0) break;

            // Start des String-Inhalts
            var contentStart = idx + marker.Length;

            // Ende des Strings finden: suche das schließende "])
            // Dabei escaped \" ignorieren
            var contentEnd = FindEscapedStringEnd(html, contentStart);
            if (contentEnd < 0) break;

            var chunk = html[contentStart..contentEnd];

            // Nur das äußere JSON-String-Escaping rückgängig machen:
            //   \" → "   (Anführungszeichen)
            //   \/ → /   (Forward-Slash, optional aber häufig)
            // Unicode-Escapes (\u003e etc.) lässt der JSON-Deserializer selbst auf.
            chunk = chunk.Replace("\\\"", "\"").Replace("\\/", "/");

            sb.Append(chunk);
            searchFrom = contentEnd + 2; // skip "])
        }

        return sb.ToString();
    }

    /// <summary>Findet das Ende eines JSON-Strings (schließendes " das nicht escaped ist).</summary>
    private static int FindEscapedStringEnd(string s, int start)
    {
        for (var i = start; i < s.Length; i++)
        {
            if (s[i] == '\\') { i++; continue; } // überspringe escaped Zeichen
            if (s[i] == '"')  return i;
        }
        return -1;
    }

    /// <summary>Extrahiert das "data":[...] Array aus dem un-escaped Payload.</summary>
    private static List<CtTrade>? ExtractTradeArray(string payload)
    {
        var match = DataArrayRx.Match(payload);
        if (!match.Success) return null;

        // Ab "[" das balancierte Array extrahieren
        var arrayStart = match.Index + match.Length - 1; // auf dem "["
        var json       = ExtractBalancedBracket(payload, arrayStart, '[', ']');
        if (json == null) return null;

        try
        {
            return JsonSerializer.Deserialize<List<CtTrade>>(json, JsonOpts);
        }
        catch
        {
            return null;
        }
    }

    private static string? ExtractBalancedBracket(string s, int start, char open, char close)
    {
        if (start >= s.Length || s[start] != open) return null;

        var depth  = 0;
        var inStr  = false;
        var escape = false;

        for (var i = start; i < s.Length; i++)
        {
            var c = s[i];

            if (escape)           { escape = false; continue; }
            if (c == '\\' && inStr) { escape = true; continue; }
            if (c == '"')         { inStr = !inStr; continue; }
            if (inStr)            continue;

            if (c == open)  depth++;
            else if (c == close)
            {
                depth--;
                if (depth == 0) return s[start..(i + 1)];
            }
        }

        return null;
    }

    /// <summary>
    /// Datum aus dem RSC-Stream ("2026-08-04T…"). Bewusst <c>TryParseExact</c> mit fester
    /// Invariant-Kultur: <c>TryParse</c> ohne Formatanbieter hätte die Kultur des Rechners benutzt
    /// und damit je nach Host anders geraten. Ändert die Quelle ihr Format, soll das <b>auffallen</b>
    /// (null → Satz wird verworfen) statt still zu einem falschen Datum zu werden.
    /// </summary>
    private static DateOnly? ParseDate(string? s)
    {
        if (string.IsNullOrEmpty(s)) return null;
        var datePart = s.Length > 10 ? s[..10] : s;
        return DateOnly.TryParseExact(datePart, "yyyy-MM-dd", CultureInfo.InvariantCulture,
            DateTimeStyles.None, out var d) ? d : null;
    }

    private static string CapFirst(string s)
        => string.IsNullOrEmpty(s) ? s
        : char.ToUpperInvariant(s[0]) + s[1..].ToLowerInvariant();

    private static Task RateDelay(CancellationToken ct)
        => Task.Delay(RateDelayMs, ct);
}