Initial commit: IBKRTrader
.NET WinForms-Anwendung (Core, Modules/CongressTrading, UI). Enthaelt .gitignore und settings.example.json als Konfigurationsvorlage. Echte settings.json mit Zugangsdaten ist bewusst ausgeschlossen. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Richard
@@ -0,0 +1,55 @@
|
||||
using IBKRTrader.Core.Workers;
|
||||
using IBKRTrader.Modules.CongressTrading.Database;
|
||||
using IBKRTrader.Modules.CongressTrading.Scraper;
|
||||
using IBKRTrader.Modules.CongressTrading.Workers;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
|
||||
namespace IBKRTrader.Modules.CongressTrading;
|
||||
|
||||
/// <summary>
|
||||
/// Entry-Point des CongressTrading-Moduls.
|
||||
/// Registriert alle modulspezifischen Services in den DI-Container.
|
||||
///
|
||||
/// Architektur-Regel: Dieser Code wird von Program.cs aufgerufen.
|
||||
/// Der Core kennt dieses Modul NICHT – die Abhängigkeit geht nur von Modul → Core.
|
||||
/// </summary>
|
||||
public static class CongressTradingModule
|
||||
{
|
||||
public const string ModuleKey = "CT";
|
||||
|
||||
/// <summary>Registriert alle Modul-Services im DI-Container.</summary>
|
||||
public static void Register(IServiceCollection services)
|
||||
{
|
||||
// Datenbank
|
||||
services.AddSingleton<CongressMigrations>();
|
||||
services.AddSingleton<CongressRepository>();
|
||||
|
||||
// Scraper
|
||||
services.AddSingleton<CapitolTradesScraper>();
|
||||
|
||||
// Worker
|
||||
services.AddSingleton<CongressHistoryImportWorker>();
|
||||
services.AddSingleton<CongressScrapeWorker>();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Führt die Modul-Migrationen aus und resettet ggf. den Fortschritt.
|
||||
/// </summary>
|
||||
public static async Task InitializeAsync(IServiceProvider provider)
|
||||
{
|
||||
var migrations = provider.GetRequiredService<CongressMigrations>();
|
||||
await migrations.RunAsync();
|
||||
|
||||
// Einmaliger Reset: löscht alle alten (fehlerhaften) Fortschritts-Daten
|
||||
// damit der Import sauber von Seite 1 neu startet.
|
||||
var repo = provider.GetRequiredService<CongressRepository>();
|
||||
// await repo.ResetHistoryImportAsync();
|
||||
}
|
||||
|
||||
/// <summary>Gibt die Modul-Worker für die WorkerEngine zurück.</summary>
|
||||
public static IEnumerable<IWorker> GetWorkers(IServiceProvider provider) =>
|
||||
[
|
||||
provider.GetRequiredService<CongressHistoryImportWorker>(),
|
||||
provider.GetRequiredService<CongressScrapeWorker>()
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
using IBKRTrader.Core.Database;
|
||||
using IBKRTrader.Core.Logging;
|
||||
|
||||
namespace IBKRTrader.Modules.CongressTrading.Database;
|
||||
|
||||
/// <summary>
|
||||
/// Legt die ct_xxx-Tabellen idempotent an.
|
||||
/// Namensschema: ct_{tabellenname} gemäß Architekturregeln.
|
||||
/// </summary>
|
||||
public class CongressMigrations
|
||||
{
|
||||
private readonly DatabaseService _db;
|
||||
private readonly LoggingService _logger;
|
||||
|
||||
public CongressMigrations(DatabaseService db, LoggingService logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task RunAsync()
|
||||
{
|
||||
_logger.Info("CT", "Starte CongressTrading-Datenbankmigrationen...");
|
||||
await CreateCongressMemberTableAsync();
|
||||
await CreateCongressTradeTableAsync();
|
||||
_logger.Info("CT", "CongressTrading-Migrationen abgeschlossen.");
|
||||
}
|
||||
|
||||
private Task CreateCongressMemberTableAsync() => _db.ExecuteAsync(@"
|
||||
CREATE TABLE IF NOT EXISTS `ct_congressMember` (
|
||||
`id` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
`bio_id` VARCHAR(20) NOT NULL UNIQUE,
|
||||
`name` VARCHAR(200),
|
||||
`party` VARCHAR(50),
|
||||
`state` VARCHAR(50),
|
||||
`chamber` VARCHAR(10),
|
||||
`profile_url` VARCHAR(500),
|
||||
`first_seen` DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
`last_updated` DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX `idx_bio_id` (`bio_id`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;");
|
||||
|
||||
private async Task CreateCongressTradeTableAsync()
|
||||
{
|
||||
// Tabelle anlegen
|
||||
await _db.ExecuteAsync(@"
|
||||
CREATE TABLE IF NOT EXISTS `ct_trade` (
|
||||
`id` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,
|
||||
`trade_id` VARCHAR(20) NOT NULL UNIQUE,
|
||||
`member_bio_id` VARCHAR(20) NOT NULL,
|
||||
`issuer_name` VARCHAR(500),
|
||||
`issuer_id` VARCHAR(20),
|
||||
`ticker` VARCHAR(20),
|
||||
`trade_type` VARCHAR(50),
|
||||
`chamber` VARCHAR(20),
|
||||
`owner` VARCHAR(50),
|
||||
`value` DECIMAL(18,2),
|
||||
`size_range_low` BIGINT,
|
||||
`size_range_high` BIGINT,
|
||||
`trade_date` DATE,
|
||||
`published_date` DATE,
|
||||
`detail_url` VARCHAR(500),
|
||||
`details_fetched` TINYINT(1) NOT NULL DEFAULT 1,
|
||||
`scraped_at` DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
INDEX `idx_member` (`member_bio_id`),
|
||||
INDEX `idx_trade_date` (`trade_date`),
|
||||
INDEX `idx_ticker` (`ticker`),
|
||||
INDEX `idx_details` (`details_fetched`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;");
|
||||
|
||||
// Idempotente Spalten-Migrationen (für bestehende DBs)
|
||||
await _db.ExecuteAsync("ALTER TABLE `ct_trade` ADD COLUMN IF NOT EXISTS `details_fetched` TINYINT(1) NOT NULL DEFAULT 1;");
|
||||
await _db.ExecuteAsync("ALTER TABLE `ct_trade` ADD COLUMN IF NOT EXISTS `chamber` VARCHAR(20);");
|
||||
await _db.ExecuteAsync("ALTER TABLE `ct_trade` ADD COLUMN IF NOT EXISTS `owner` VARCHAR(50);");
|
||||
await _db.ExecuteAsync("ALTER TABLE `ct_trade` ADD COLUMN IF NOT EXISTS `value` DECIMAL(18,2);");
|
||||
await _db.ExecuteAsync("ALTER TABLE `ct_trade` ADD INDEX IF NOT EXISTS `idx_details` (`details_fetched`);");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
using Dapper;
|
||||
using IBKRTrader.Core.Database;
|
||||
using IBKRTrader.Core.Logging;
|
||||
using IBKRTrader.Modules.CongressTrading.Models;
|
||||
|
||||
namespace IBKRTrader.Modules.CongressTrading.Database;
|
||||
|
||||
/// <summary>
|
||||
/// Datenbankzugriff für das CongressTrading-Modul.
|
||||
/// Kapselt alle INSERT/SELECT auf ct_congressMember und ct_trade.
|
||||
/// </summary>
|
||||
public class CongressRepository
|
||||
{
|
||||
private readonly DatabaseService _db;
|
||||
private readonly LoggingService _logger;
|
||||
|
||||
public CongressRepository(DatabaseService db, LoggingService logger)
|
||||
{
|
||||
_db = db;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
// ─── CongressMember ───────────────────────────────────────────────────────
|
||||
|
||||
public async Task<bool> MemberExistsAsync(string bioId)
|
||||
{
|
||||
var count = await _db.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM `ct_congressMember` WHERE bio_id = @bioId",
|
||||
new { bioId });
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
public async Task UpsertMemberAsync(CongressMember member)
|
||||
{
|
||||
await _db.ExecuteAsync(@"
|
||||
INSERT INTO `ct_congressMember` (bio_id, name, party, state, chamber, profile_url)
|
||||
VALUES (@BioId, @Name, @Party, @State, @Chamber, @ProfileUrl)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
party = VALUES(party),
|
||||
state = VALUES(state),
|
||||
chamber = VALUES(chamber),
|
||||
profile_url = VALUES(profile_url),
|
||||
last_updated = CURRENT_TIMESTAMP", member);
|
||||
}
|
||||
|
||||
public async Task<HashSet<string>> GetAllMemberBioIdsAsync()
|
||||
{
|
||||
var ids = await _db.QueryAsync<string>(
|
||||
"SELECT bio_id FROM `ct_congressMember`");
|
||||
return ids.ToHashSet();
|
||||
}
|
||||
|
||||
// ─── CongressTrade ────────────────────────────────────────────────────────
|
||||
|
||||
public async Task<bool> TradeExistsAsync(string tradeId)
|
||||
{
|
||||
var count = await _db.ExecuteScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM `ct_trade` WHERE trade_id = @tradeId",
|
||||
new { tradeId });
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
public async Task<HashSet<string>> GetAllTradeIdsAsync()
|
||||
{
|
||||
var ids = await _db.QueryAsync<string>(
|
||||
"SELECT trade_id FROM `ct_trade`");
|
||||
return ids.ToHashSet();
|
||||
}
|
||||
|
||||
/// <summary>Fügt einen Trade ein. Ignoriert Duplikate (INSERT IGNORE).</summary>
|
||||
public async Task InsertTradeAsync(CongressTrade trade, bool detailsFetched = true)
|
||||
{
|
||||
await _db.ExecuteAsync(@"
|
||||
INSERT IGNORE INTO `ct_trade`
|
||||
(trade_id, member_bio_id, issuer_name, issuer_id, ticker,
|
||||
trade_type, chamber, owner, value,
|
||||
trade_date, published_date, detail_url, details_fetched)
|
||||
VALUES
|
||||
(@TradeId, @MemberBioId, @IssuerName, @IssuerId, @Ticker,
|
||||
@TradeType, @Chamber, @Owner, @Value,
|
||||
@TradeDate, @PublishedDate, @DetailUrl, @DetailsFetched)",
|
||||
new
|
||||
{
|
||||
trade.TradeId,
|
||||
trade.MemberBioId,
|
||||
trade.IssuerName,
|
||||
trade.IssuerId,
|
||||
trade.Ticker,
|
||||
trade.TradeType,
|
||||
trade.Chamber,
|
||||
trade.Owner,
|
||||
trade.Value,
|
||||
TradeDate = trade.TradeDate?.ToString("yyyy-MM-dd"),
|
||||
PublishedDate = trade.PublishedDate?.ToString("yyyy-MM-dd"),
|
||||
trade.DetailUrl,
|
||||
DetailsFetched = detailsFetched ? 1 : 0
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Aktualisiert die Detail-Felder eines bereits vorhandenen Trades.</summary>
|
||||
public async Task UpdateTradeDetailsAsync(CongressTrade trade)
|
||||
{
|
||||
await _db.ExecuteAsync(@"
|
||||
UPDATE `ct_trade`
|
||||
SET ticker = @Ticker,
|
||||
trade_type = @TradeType,
|
||||
trade_date = @TradeDate,
|
||||
published_date = @PublishedDate,
|
||||
details_fetched = 1
|
||||
WHERE trade_id = @TradeId
|
||||
AND details_fetched = 0",
|
||||
new
|
||||
{
|
||||
trade.TradeId,
|
||||
trade.Ticker,
|
||||
trade.TradeType,
|
||||
TradeDate = trade.TradeDate?.ToString("yyyy-MM-dd"),
|
||||
PublishedDate = trade.PublishedDate?.ToString("yyyy-MM-dd")
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Gibt Trade-IDs zurück für die noch keine Details geholt wurden.</summary>
|
||||
public async Task<IEnumerable<string>> GetTradesWithoutDetailsAsync(int limit = 500)
|
||||
=> await _db.QueryAsync<string>(
|
||||
"SELECT trade_id FROM `ct_trade` WHERE details_fetched = 0 LIMIT @limit",
|
||||
new { limit });
|
||||
|
||||
public async Task<int> GetTradeCountAsync()
|
||||
=> await _db.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM `ct_trade`");
|
||||
|
||||
public async Task<int> GetMemberCountAsync()
|
||||
=> await _db.ExecuteScalarAsync<int>("SELECT COUNT(*) FROM `ct_congressMember`");
|
||||
|
||||
// ─── core_settings Flags ──────────────────────────────────────────────────
|
||||
|
||||
public async Task<string?> GetSettingAsync(string key)
|
||||
{
|
||||
return await _db.QueryFirstOrDefaultAsync<string>(
|
||||
"SELECT `value` FROM `core_settings` WHERE `key` = @key",
|
||||
new { key });
|
||||
}
|
||||
|
||||
public async Task SetSettingAsync(string key, string value)
|
||||
{
|
||||
await _db.ExecuteAsync(@"
|
||||
INSERT INTO `core_settings` (`key`, `value`)
|
||||
VALUES (@key, @value)
|
||||
ON DUPLICATE KEY UPDATE `value` = @value",
|
||||
new { key, value });
|
||||
}
|
||||
|
||||
public async Task DeleteSettingAsync(string key)
|
||||
{
|
||||
await _db.ExecuteAsync(
|
||||
"DELETE FROM `core_settings` WHERE `key` = @key", new { key });
|
||||
}
|
||||
|
||||
// ─── Reset ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Löscht alle CT-Fortschritts-Flags und alle CT-Daten aus der DB.
|
||||
/// Danach startet der History-Import beim nächsten Worker-Run von Seite 1.
|
||||
/// </summary>
|
||||
public async Task ResetHistoryImportAsync()
|
||||
{
|
||||
await DeleteSettingAsync("ct.history_import_done");
|
||||
await DeleteSettingAsync("ct.history_import_page");
|
||||
await DeleteSettingAsync("ct.history_import_total");
|
||||
await _db.ExecuteAsync("DELETE FROM `ct_trade`");
|
||||
await _db.ExecuteAsync("DELETE FROM `ct_congressMember`");
|
||||
_logger.Warn("CT", "History-Import-Reset durchgeführt – alle CT-Daten gelöscht.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace IBKRTrader.Modules.CongressTrading.Models;
|
||||
|
||||
/// <summary>Kongressmitglied aus capitoltrades.com</summary>
|
||||
public class CongressMember
|
||||
{
|
||||
public long Id { get; set; }
|
||||
public string BioId { get; set; } = ""; // z.B. "W000805"
|
||||
public string Name { get; set; } = "";
|
||||
public string Party { get; set; } = ""; // Democrat/Republican/Independent
|
||||
public string State { get; set; } = "";
|
||||
public string Chamber { get; set; } = ""; // House/Senate
|
||||
public string ProfileUrl { get; set; } = "";
|
||||
public DateTime FirstSeen { get; set; } = DateTime.UtcNow;
|
||||
public DateTime LastUpdated { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace IBKRTrader.Modules.CongressTrading.Models;
|
||||
|
||||
/// <summary>Ein einzelner Kongress-Trade aus capitoltrades.com</summary>
|
||||
public class CongressTrade
|
||||
{
|
||||
public long Id { get; set; }
|
||||
public string TradeId { get; set; } = "";
|
||||
public string MemberBioId { get; set; } = "";
|
||||
public string IssuerName { get; set; } = "";
|
||||
public string IssuerId { get; set; } = "";
|
||||
public string Ticker { get; set; } = "";
|
||||
public string TradeType { get; set; } = ""; // buy/sell/exchange
|
||||
public string Chamber { get; set; } = ""; // house/senate
|
||||
public string Owner { get; set; } = ""; // self/spouse/child/trust
|
||||
public decimal? Value { get; set; } // exakter Handelswert in USD
|
||||
public DateOnly? TradeDate { get; set; }
|
||||
public DateOnly? PublishedDate { get; set; }
|
||||
public string DetailUrl { get; set; } = "";
|
||||
public DateTime ScrapedAt { get; set; } = DateTime.UtcNow;
|
||||
|
||||
/// <summary>Transient – wird direkt in ct_congressMember gespeichert.</summary>
|
||||
public CongressMember? MemberSnapshot { get; set; }
|
||||
}
|
||||
@@ -0,0 +1,332 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Text.RegularExpressions;
|
||||
using IBKRTrader.Core.Logging;
|
||||
using IBKRTrader.Modules.CongressTrading.Models;
|
||||
|
||||
namespace IBKRTrader.Modules.CongressTrading.Scraper;
|
||||
|
||||
// ─── JSON-Modelle (exakte Feldnamen aus dem RSC-Stream) ──────────────────────
|
||||
|
||||
internal class CtApiWrapper
|
||||
{
|
||||
[JsonPropertyName("data")] public List<CtTrade>? Data { get; set; }
|
||||
[JsonPropertyName("meta")] public CtMeta? Meta { get; set; }
|
||||
}
|
||||
|
||||
internal class CtMeta
|
||||
{
|
||||
[JsonPropertyName("totalPages")] public int TotalPages { get; set; }
|
||||
[JsonPropertyName("pageSize")] public int PageSize { get; set; }
|
||||
[JsonPropertyName("totalCount")] public int TotalCount { get; set; }
|
||||
[JsonPropertyName("page")] public int Page { get; set; }
|
||||
}
|
||||
|
||||
internal class CtTrade
|
||||
{
|
||||
[JsonPropertyName("_txId")] public long TxId { get; set; }
|
||||
[JsonPropertyName("txDate")] public string? TxDate { get; set; }
|
||||
[JsonPropertyName("pubDate")] public string? PubDate { get; set; }
|
||||
[JsonPropertyName("txType")] public string? TxType { get; set; }
|
||||
[JsonPropertyName("value")] public decimal? Value { get; set; }
|
||||
[JsonPropertyName("price")] public decimal? Price { get; set; }
|
||||
[JsonPropertyName("owner")] public string? Owner { get; set; }
|
||||
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
|
||||
[JsonPropertyName("reportingGap")] public int? ReportingGap { get; set; }
|
||||
[JsonPropertyName("comment")] public string? Comment { get; set; }
|
||||
[JsonPropertyName("_politicianId")] public string? PoliticianId { get; set; }
|
||||
[JsonPropertyName("_issuerId")] public long IssuerId { get; set; }
|
||||
[JsonPropertyName("politician")] public CtPolitician? Politician { get; set; }
|
||||
[JsonPropertyName("issuer")] public CtIssuer? Issuer { get; set; }
|
||||
}
|
||||
|
||||
internal class CtPolitician
|
||||
{
|
||||
[JsonPropertyName("firstName")] public string? FirstName { get; set; }
|
||||
[JsonPropertyName("lastName")] public string? LastName { get; set; }
|
||||
[JsonPropertyName("party")] public string? Party { get; set; }
|
||||
[JsonPropertyName("_stateId")] public string? StateId { get; set; }
|
||||
[JsonPropertyName("chamber")] public string? Chamber { get; set; }
|
||||
}
|
||||
|
||||
internal class CtIssuer
|
||||
{
|
||||
[JsonPropertyName("issuerName")] public string? IssuerName { get; set; }
|
||||
[JsonPropertyName("issuerTicker")] public string? IssuerTicker { get; set; }
|
||||
[JsonPropertyName("sector")] public string? Sector { get; set; }
|
||||
}
|
||||
|
||||
// ─── Scraper ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Scrapt capitoltrades.com über den eingebetteten Next.js RSC-Datenstrom.
|
||||
///
|
||||
/// Die Seite liefert Daten als escaped JSON-String innerhalb von:
|
||||
/// self.__next_f.push([1,"...\"data\":[{\"_txId\":...}]..."])
|
||||
///
|
||||
/// Strategie:
|
||||
/// 1. HTML-Seite abrufen
|
||||
/// 2. Den escaped JSON-String mit dem "data"-Array extrahieren
|
||||
/// 3. String un-escapen → gültiges JSON
|
||||
/// 4. Als Liste von CtTrade deserialisieren
|
||||
///
|
||||
/// Rate-Limiting: 600ms zwischen Requests.
|
||||
/// </summary>
|
||||
public class CapitolTradesScraper
|
||||
{
|
||||
private readonly HttpClient _http;
|
||||
private readonly LoggingService _logger;
|
||||
|
||||
private const string BaseUrl = "https://www.capitoltrades.com";
|
||||
private const int RateDelayMs = 600;
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOpts = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
NumberHandling = JsonNumberHandling.AllowReadingFromString
|
||||
};
|
||||
|
||||
// Sucht das "data":[ Array im escaped RSC-String
|
||||
// Das Muster ist: \"data\":[{\"_issuerId\":...,\"_txId\":...}]
|
||||
private static readonly Regex DataArrayRx = new(
|
||||
@"""data"":\s*\[",
|
||||
RegexOptions.Compiled);
|
||||
|
||||
// Pagination: \"totalPages\":N
|
||||
private static readonly Regex TotalPagesRx = new(
|
||||
@"""totalPages""\s*:\s*(\d+)",
|
||||
RegexOptions.Compiled);
|
||||
|
||||
public CapitolTradesScraper(LoggingService logger)
|
||||
{
|
||||
_logger = logger;
|
||||
_http = new HttpClient();
|
||||
_http.DefaultRequestHeaders.Add("User-Agent",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " +
|
||||
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36");
|
||||
_http.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9");
|
||||
_http.DefaultRequestHeaders.Add("Accept",
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||||
_http.Timeout = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
// ─── Öffentliche API ─────────────────────────────────────────────────────
|
||||
|
||||
public async Task<int> GetTotalPagesAsync(CancellationToken ct)
|
||||
{
|
||||
await RateDelay(ct);
|
||||
var html = await _http.GetStringAsync($"{BaseUrl}/trades?pageSize=96&page=1", ct);
|
||||
var unescaped = UnescapeHtmlPayload(html);
|
||||
var match = TotalPagesRx.Match(unescaped);
|
||||
|
||||
if (match.Success)
|
||||
{
|
||||
var total = int.Parse(match.Groups[1].Value);
|
||||
_logger.Info("CT", $"GetTotalPagesAsync: {total} Seiten gefunden.");
|
||||
return total;
|
||||
}
|
||||
|
||||
_logger.Warn("CT", "GetTotalPagesAsync: Konnte 'totalPages' im RSC-Stream nicht finden.");
|
||||
_logger.Info("CT", $" Payload-Länge: {unescaped.Length}. Erste 200 Zeichen: {unescaped.Substring(0, Math.Min(200, unescaped.Length))}");
|
||||
|
||||
// Fallback: Wenn wir zumindest Trades gefunden haben, nehmen wir an es gibt mindestens 1 Seite.
|
||||
// Falls wir totalPages nicht finden, riskieren wir hier 1, aber wir haben gewarnt.
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
public async Task<(List<CongressTrade> Trades, HashSet<string> MemberBioIds)>
|
||||
ScrapeTradesPageAsync(int page, CancellationToken ct)
|
||||
{
|
||||
await RateDelay(ct);
|
||||
|
||||
var url = $"{BaseUrl}/trades?pageSize=96&page={page}";
|
||||
var html = await _http.GetStringAsync(url, ct);
|
||||
|
||||
var trades = new List<CongressTrade>();
|
||||
var memberIds = new HashSet<string>();
|
||||
|
||||
// HTML un-escapen → gültiges JSON
|
||||
var payload = UnescapeHtmlPayload(html);
|
||||
|
||||
// "data":[ Block finden
|
||||
var rawTrades = ExtractTradeArray(payload);
|
||||
|
||||
if (rawTrades == null || rawTrades.Count == 0)
|
||||
{
|
||||
_logger.Warn("CT", $"Seite {page}: Keine Trade-Daten im Payload gefunden.");
|
||||
_logger.Info("CT", $" Payload-Länge: {payload.Length}, HTML-Länge: {html.Length}");
|
||||
return (trades, memberIds);
|
||||
}
|
||||
|
||||
foreach (var raw in rawTrades)
|
||||
{
|
||||
if (raw.TxId == 0) continue;
|
||||
|
||||
var bioId = raw.PoliticianId ?? "";
|
||||
if (!string.IsNullOrEmpty(bioId))
|
||||
memberIds.Add(bioId);
|
||||
|
||||
// Ticker: "AVGO:US" → "AVGO"
|
||||
var ticker = raw.Issuer?.IssuerTicker ?? "";
|
||||
if (ticker.Contains(':'))
|
||||
ticker = ticker[..ticker.IndexOf(':')];
|
||||
|
||||
var trade = new CongressTrade
|
||||
{
|
||||
TradeId = raw.TxId.ToString(),
|
||||
MemberBioId = bioId,
|
||||
IssuerName = raw.Issuer?.IssuerName ?? "",
|
||||
IssuerId = raw.IssuerId.ToString(),
|
||||
Ticker = ticker,
|
||||
TradeType = raw.TxType ?? "",
|
||||
Chamber = raw.Chamber ?? "",
|
||||
Owner = raw.Owner ?? "",
|
||||
Value = raw.Value,
|
||||
TradeDate = ParseDate(raw.TxDate),
|
||||
PublishedDate = ParseDate(raw.PubDate),
|
||||
DetailUrl = $"{BaseUrl}/trades/{raw.TxId}"
|
||||
};
|
||||
|
||||
if (raw.Politician != null && !string.IsNullOrEmpty(bioId))
|
||||
{
|
||||
trade.MemberSnapshot = new CongressMember
|
||||
{
|
||||
BioId = bioId,
|
||||
Name = $"{raw.Politician.FirstName} {raw.Politician.LastName}".Trim(),
|
||||
Party = CapFirst(raw.Politician.Party ?? ""),
|
||||
State = (raw.Politician.StateId ?? "").ToUpperInvariant(),
|
||||
Chamber = CapFirst(raw.Politician.Chamber ?? ""),
|
||||
ProfileUrl = $"{BaseUrl}/politicians/{bioId}"
|
||||
};
|
||||
}
|
||||
|
||||
trades.Add(trade);
|
||||
}
|
||||
|
||||
_logger.Info("CT", $"Seite {page}: {trades.Count} Trades extrahiert.");
|
||||
return (trades, memberIds);
|
||||
}
|
||||
|
||||
// ─── Hilfsmethoden ────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Extrahiert und un-escaped den JSON-Payload aus dem RSC-Stream.
|
||||
///
|
||||
/// Die Seite enthält Blöcke wie:
|
||||
/// self.__next_f.push([1,"...ESCAPED_JSON..."])
|
||||
///
|
||||
/// Die inneren Anführungszeichen sind mit \" escaped.
|
||||
/// Wir konkatenieren alle Chunk-Strings und un-escapen sie.
|
||||
/// </summary>
|
||||
private static string UnescapeHtmlPayload(string html)
|
||||
{
|
||||
// Alle self.__next_f.push([1,"..."]) Chunks sammeln
|
||||
// Regex: sucht [1,"(content)"] Blöcke
|
||||
var sb = new System.Text.StringBuilder();
|
||||
|
||||
var searchFrom = 0;
|
||||
const string marker = "self.__next_f.push([1,\"";
|
||||
|
||||
while (true)
|
||||
{
|
||||
var idx = html.IndexOf(marker, searchFrom, StringComparison.Ordinal);
|
||||
if (idx < 0) break;
|
||||
|
||||
// Start des String-Inhalts
|
||||
var contentStart = idx + marker.Length;
|
||||
|
||||
// Ende des Strings finden: suche das schließende "])
|
||||
// Dabei escaped \" ignorieren
|
||||
var contentEnd = FindEscapedStringEnd(html, contentStart);
|
||||
if (contentEnd < 0) break;
|
||||
|
||||
var chunk = html[contentStart..contentEnd];
|
||||
|
||||
// Nur das äußere JSON-String-Escaping rückgängig machen:
|
||||
// \" → " (Anführungszeichen)
|
||||
// \/ → / (Forward-Slash, optional aber häufig)
|
||||
// Unicode-Escapes (\u003e etc.) lässt der JSON-Deserializer selbst auf.
|
||||
chunk = chunk.Replace("\\\"", "\"").Replace("\\/", "/");
|
||||
|
||||
sb.Append(chunk);
|
||||
searchFrom = contentEnd + 2; // skip "])
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>Findet das Ende eines JSON-Strings (schließendes " das nicht escaped ist).</summary>
|
||||
private static int FindEscapedStringEnd(string s, int start)
|
||||
{
|
||||
for (var i = start; i < s.Length; i++)
|
||||
{
|
||||
if (s[i] == '\\') { i++; continue; } // überspringe escaped Zeichen
|
||||
if (s[i] == '"') return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/// <summary>Extrahiert das "data":[...] Array aus dem un-escaped Payload.</summary>
|
||||
private static List<CtTrade>? ExtractTradeArray(string payload)
|
||||
{
|
||||
var match = DataArrayRx.Match(payload);
|
||||
if (!match.Success) return null;
|
||||
|
||||
// Ab "[" das balancierte Array extrahieren
|
||||
var arrayStart = match.Index + match.Length - 1; // auf dem "["
|
||||
var json = ExtractBalancedBracket(payload, arrayStart, '[', ']');
|
||||
if (json == null) return null;
|
||||
|
||||
try
|
||||
{
|
||||
return JsonSerializer.Deserialize<List<CtTrade>>(json, JsonOpts);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractBalancedBracket(string s, int start, char open, char close)
|
||||
{
|
||||
if (start >= s.Length || s[start] != open) return null;
|
||||
|
||||
var depth = 0;
|
||||
var inStr = false;
|
||||
var escape = false;
|
||||
|
||||
for (var i = start; i < s.Length; i++)
|
||||
{
|
||||
var c = s[i];
|
||||
|
||||
if (escape) { escape = false; continue; }
|
||||
if (c == '\\' && inStr) { escape = true; continue; }
|
||||
if (c == '"') { inStr = !inStr; continue; }
|
||||
if (inStr) continue;
|
||||
|
||||
if (c == open) depth++;
|
||||
else if (c == close)
|
||||
{
|
||||
depth--;
|
||||
if (depth == 0) return s[start..(i + 1)];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static DateOnly? ParseDate(string? s)
|
||||
{
|
||||
if (string.IsNullOrEmpty(s)) return null;
|
||||
var datePart = s.Length > 10 ? s[..10] : s;
|
||||
return DateOnly.TryParse(datePart, out var d) ? d : null;
|
||||
}
|
||||
|
||||
private static string CapFirst(string s)
|
||||
=> string.IsNullOrEmpty(s) ? s
|
||||
: char.ToUpperInvariant(s[0]) + s[1..].ToLowerInvariant();
|
||||
|
||||
private static Task RateDelay(CancellationToken ct)
|
||||
=> Task.Delay(RateDelayMs, ct);
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
using IBKRTrader.Core.Database;
|
||||
using IBKRTrader.Core.Logging;
|
||||
using IBKRTrader.Modules.CongressTrading.Database;
|
||||
using IBKRTrader.Modules.CongressTrading.Scraper;
|
||||
using IBKRTrader.Core.Workers;
|
||||
|
||||
namespace IBKRTrader.Modules.CongressTrading.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Einmaliger History-Import – übersteht beliebig viele Neustarts.
|
||||
///
|
||||
/// Fortschritts-Persistenz via core_settings:
|
||||
/// ct.history_import_done = "true" → abgeschlossen, Worker skippt sich selbst
|
||||
/// ct.history_import_page = "N" → nächste zu verarbeitende Seite
|
||||
/// ct.history_import_total = "N" → Gesamtseitenanzahl (gecacht)
|
||||
///
|
||||
/// Alle Trade-Daten (Datum, Typ, Value, Ticker, Mitglied) kommen direkt
|
||||
/// aus dem Listen-JSON – kein separates Detail-Scraping notwendig.
|
||||
/// </summary>
|
||||
public class CongressHistoryImportWorker : WorkerBase
|
||||
{
|
||||
private readonly CapitolTradesScraper _scraper;
|
||||
private readonly CongressRepository _repo;
|
||||
|
||||
private const string DoneFlag = "ct.history_import_done";
|
||||
private const string ProgressKey = "ct.history_import_page";
|
||||
private const string TotalKey = "ct.history_import_total";
|
||||
|
||||
public override string Name => "CT-HistoryImport";
|
||||
public override string Module => "CT";
|
||||
public override WorkerType Type => WorkerType.Worker;
|
||||
|
||||
protected override TimeSpan? Interval => TimeSpan.FromDays(365);
|
||||
|
||||
public CongressHistoryImportWorker(
|
||||
LoggingService logger,
|
||||
DatabaseService db,
|
||||
CapitolTradesScraper scraper,
|
||||
CongressRepository repo)
|
||||
: base(logger, db)
|
||||
{
|
||||
_scraper = scraper;
|
||||
_repo = repo;
|
||||
Info.Info = "Bereit – prüfe Fortschritt beim Start...";
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
// ── 1. Bereits vollständig abgeschlossen? ────────────────────────────
|
||||
if (await _repo.GetSettingAsync(DoneFlag) == "true")
|
||||
{
|
||||
Logger.Info(Module, "CT History-Import: bereits abgeschlossen.");
|
||||
Info.Info = "✓ Abgeschlossen";
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 2. Fortschritt aus DB laden ──────────────────────────────────────
|
||||
var pageStr = await _repo.GetSettingAsync(ProgressKey);
|
||||
var startPage = pageStr != null && int.TryParse(pageStr, out var sp) ? sp : 1;
|
||||
|
||||
var totalStr = await _repo.GetSettingAsync(TotalKey);
|
||||
int totalPages;
|
||||
|
||||
if (totalStr != null && int.TryParse(totalStr, out var tp) && tp > 0)
|
||||
{
|
||||
totalPages = tp;
|
||||
Logger.Info(Module,
|
||||
$"▶ Fortsetze History-Import: Seite {startPage} von {totalPages} (Gesamtanzahl aus Cache)");
|
||||
}
|
||||
else
|
||||
{
|
||||
Logger.Info(Module, "Ermittle Gesamtseitenanzahl von capitoltrades.com...");
|
||||
totalPages = await _scraper.GetTotalPagesAsync(ct);
|
||||
await _repo.SetSettingAsync(TotalKey, totalPages.ToString());
|
||||
Logger.Info(Module,
|
||||
$"▶ Starte History-Import: {totalPages} Seiten gefunden. Beginne bei Seite {startPage}.");
|
||||
}
|
||||
|
||||
|
||||
// ── 3. Seiten paginieren ─────────────────────────────────────────────
|
||||
var knownMembers = await _repo.GetAllMemberBioIdsAsync();
|
||||
var importedTotal = 0;
|
||||
|
||||
for (var page = startPage; page <= totalPages; page++)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
Info.Info = $"Seite {page}/{totalPages} – {importedTotal} neue Trades";
|
||||
|
||||
var (trades, memberIds) = await _scraper.ScrapeTradesPageAsync(page, ct);
|
||||
|
||||
if (trades.Count == 0)
|
||||
{
|
||||
Logger.Warn(Module,
|
||||
$"[{page}/{totalPages}] Leere Seite – Seite wird übersprungen.");
|
||||
await _repo.SetSettingAsync(ProgressKey, (page + 1).ToString());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Neue Kongressmitglieder anlegen (Daten kommen aus MemberSnapshot)
|
||||
foreach (var trade in trades.Where(t => t.MemberSnapshot != null))
|
||||
{
|
||||
var snapshot = trade.MemberSnapshot!;
|
||||
if (knownMembers.Contains(snapshot.BioId)) continue;
|
||||
|
||||
await _repo.UpsertMemberAsync(snapshot);
|
||||
knownMembers.Add(snapshot.BioId);
|
||||
Logger.Info(Module,
|
||||
$" + {snapshot.Name} ({snapshot.BioId}) {snapshot.Party}/{snapshot.State}");
|
||||
}
|
||||
|
||||
// Trades einfügen (INSERT IGNORE – Duplikate werden automatisch ignoriert)
|
||||
foreach (var trade in trades)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (await _repo.TradeExistsAsync(trade.TradeId)) continue;
|
||||
await _repo.InsertTradeAsync(trade, detailsFetched: true);
|
||||
importedTotal++;
|
||||
}
|
||||
|
||||
// ─── Fortschritt nach jeder erfolgreichen Seite persistieren ───
|
||||
await _repo.SetSettingAsync(ProgressKey, (page + 1).ToString());
|
||||
|
||||
Logger.Info(Module,
|
||||
$"[{page}/{totalPages}] {trades.Count} Trades, {importedTotal} neu gesamt.");
|
||||
}
|
||||
|
||||
// ── 4. Abschluss ─────────────────────────────────────────────────────
|
||||
await _repo.SetSettingAsync(DoneFlag, "true");
|
||||
|
||||
var totalInDb = await _repo.GetTradeCountAsync();
|
||||
var membersInDb = await _repo.GetMemberCountAsync();
|
||||
|
||||
Logger.Info(Module,
|
||||
$"=== History-Import abgeschlossen: {totalInDb} Trades, {membersInDb} Mitglieder ===");
|
||||
Info.Info = $"✓ Fertig – {totalInDb} Trades, {membersInDb} Mitglieder";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
using IBKRTrader.Core.Database;
|
||||
using IBKRTrader.Core.Logging;
|
||||
using IBKRTrader.Core.Settings;
|
||||
using IBKRTrader.Modules.CongressTrading.Database;
|
||||
using IBKRTrader.Modules.CongressTrading.Scraper;
|
||||
using IBKRTrader.Core.Workers;
|
||||
|
||||
namespace IBKRTrader.Modules.CongressTrading.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// Periodischer Scrape-Worker: alle 30 Minuten.
|
||||
/// Lädt nur Seite 1 (neueste 96 Trades) und prüft auf neue Einträge.
|
||||
/// Alle Daten (Datum, Typ, Value, Ticker, Mitglied) kommen direkt aus dem RSC-JSON.
|
||||
/// </summary>
|
||||
public class CongressScrapeWorker : WorkerBase
|
||||
{
|
||||
private readonly CapitolTradesScraper _scraper;
|
||||
private readonly CongressRepository _repo;
|
||||
|
||||
public override string Name => "CT-ScrapeWorker";
|
||||
public override string Module => "CT";
|
||||
public override WorkerType Type => WorkerType.Worker;
|
||||
|
||||
protected override TimeSpan? Interval => TimeSpan.FromMinutes(30);
|
||||
|
||||
public CongressScrapeWorker(
|
||||
LoggingService logger,
|
||||
DatabaseService db,
|
||||
CapitolTradesScraper scraper,
|
||||
CongressRepository repo,
|
||||
SettingsService settings)
|
||||
: base(logger, db)
|
||||
{
|
||||
_scraper = scraper;
|
||||
_repo = repo;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken ct)
|
||||
{
|
||||
Logger.Info(Module, "Suche neue Kongress-Trades (Seite 1)...");
|
||||
|
||||
var knownMembers = await _repo.GetAllMemberBioIdsAsync();
|
||||
var (trades, _) = await _scraper.ScrapeTradesPageAsync(1, ct);
|
||||
|
||||
if (trades.Count == 0)
|
||||
{
|
||||
Logger.Warn(Module, "Scrape: Keine Trades auf Seite 1 gefunden.");
|
||||
Info.Info = $"Letzter Lauf {DateTime.Now:HH:mm}: 0 Trades";
|
||||
return;
|
||||
}
|
||||
|
||||
// Neue Kongressmitglieder
|
||||
var newMembers = 0;
|
||||
foreach (var trade in trades.Where(t => t.MemberSnapshot != null))
|
||||
{
|
||||
var snapshot = trade.MemberSnapshot!;
|
||||
if (knownMembers.Contains(snapshot.BioId)) continue;
|
||||
|
||||
await _repo.UpsertMemberAsync(snapshot);
|
||||
knownMembers.Add(snapshot.BioId);
|
||||
Logger.Info(Module, $"Neues Mitglied: {snapshot.Name} ({snapshot.BioId})");
|
||||
newMembers++;
|
||||
}
|
||||
|
||||
// Neue Trades
|
||||
var newTrades = 0;
|
||||
foreach (var trade in trades)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
if (await _repo.TradeExistsAsync(trade.TradeId)) continue;
|
||||
|
||||
await _repo.InsertTradeAsync(trade, detailsFetched: true);
|
||||
Logger.Info(Module,
|
||||
$" + {trade.IssuerName} ({trade.TradeType}) [{trade.Ticker}] von {trade.MemberBioId}");
|
||||
newTrades++;
|
||||
}
|
||||
|
||||
var summary = $"{newTrades} neue Trades, {newMembers} neue Mitglieder";
|
||||
Logger.Info(Module, $"Scrape abgeschlossen: {summary}");
|
||||
Info.Info = $"Letzter Lauf {DateTime.Now:HH:mm}: {summary}";
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user