feat: implement Part D and E from FIXPLAN
- D1/D2/D2c: Added TraderTraits entity, TraderTraitCalculator, Market Return Metrics (MedianWin, AvgWin, etc.), and trait filters
- D3: Implemented HF-Trader Tiering via IngestMode (Full, Aggregated, SnapshotOnly) and updated TradeHistoryWorker to respect tiers
- E1-E5: Added MasterStatus to Trader, TraderWindowMetrics for rolling analytics, Fingerprint metrics (PriceBandProfile, P50/P90), Copyability aggregates (Volume, Drift, Edge)
- E6: Implemented GET /api/traders/{id}/profile and GET /api/traders/correlation
- Replaced FIXPLAN-2026-07-09.md with FIXPLAN-TODO.md and FIXPLAN-DONE.md
- Cleaned up API docs and plan to use generic terms (removed hardcoded PolyTrader references)
- Added respective EF Core Migrations
This commit is contained in:
Richard
@@ -122,95 +122,231 @@ public class TradeHistoryWorker : BackgroundService
|
||||
|
||||
bool isDeepResync = activeJob != null && activeJob.JobType == Predictalytics.Domain.Enums.JobType.DeepResync;
|
||||
bool isInitial = !trader.IsInitialImportComplete || isDeepResync;
|
||||
_logger.LogInformation("{Trader}: Starting {Type} sync", trader.DisplayName, isDeepResync ? "DEEP RESYNC" : (isInitial ? "INITIAL FULL" : "INCREMENTAL"));
|
||||
|
||||
bool isWeeklyBiopsy = trader.IngestMode == IngestMode.SnapshotOnly && (!trader.LastTradesUpdatedAt.HasValue || (DateTime.UtcNow - trader.LastTradesUpdatedAt.Value).TotalDays >= 7);
|
||||
bool skipTradeFetch = trader.IngestMode == IngestMode.SnapshotOnly && !isWeeklyBiopsy && !isDeepResync;
|
||||
|
||||
_logger.LogInformation("{Trader}: Starting {Type} sync (Mode: {Mode})", trader.DisplayName, isDeepResync ? "DEEP RESYNC" : (isInitial ? "INITIAL FULL" : "INCREMENTAL"), trader.IngestMode);
|
||||
|
||||
if (isDeepResync)
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<Predictalytics.Infrastructure.Data.AppDbContext>();
|
||||
await Microsoft.EntityFrameworkCore.RelationalDatabaseFacadeExtensions.ExecuteSqlRawAsync(db.Database, "DELETE FROM Trades WHERE TraderId = {0} AND PlatformTradeId LIKE 'COMPACT_%'", t.Id);
|
||||
await Microsoft.EntityFrameworkCore.RelationalDatabaseFacadeExtensions.ExecuteSqlRawAsync(db.Database, "DELETE FROM Trades WHERE TraderId = {0} AND PlatformTradeId LIKE 'AGG_%'", t.Id);
|
||||
await Microsoft.EntityFrameworkCore.RelationalDatabaseFacadeExtensions.ExecuteSqlRawAsync(db.Database, "DELETE FROM TraderPositions WHERE TraderId = {0}", t.Id);
|
||||
}
|
||||
|
||||
IReadOnlyList<Domain.Entities.Trade> fetchedTrades;
|
||||
if (isDeepResync)
|
||||
{
|
||||
fetchedTrades = await provider.GetTradesPagedAsync(trader.PlatformUserId, 500, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
fetchedTrades = await provider.GetTraderTradesAsync(trader.PlatformUserId, TradesPerFetch, ct);
|
||||
}
|
||||
var validTrades = fetchedTrades.Where(tr => !string.IsNullOrWhiteSpace(tr.PlatformTradeId)).ToList();
|
||||
|
||||
var updatedName = validTrades.FirstOrDefault(t => !string.IsNullOrEmpty(t.TransientDisplayName))?.TransientDisplayName;
|
||||
if (!string.IsNullOrEmpty(updatedName) && !string.Equals(trader.DisplayName, updatedName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
trader.DisplayName = updatedName;
|
||||
await traderRepo.UpdateAsync(trader, ct);
|
||||
}
|
||||
|
||||
var fetchedTradeIds = validTrades.Select(tr => tr.PlatformTradeId).ToList();
|
||||
var knownTradeIds = await tradeRepo.GetKnownPlatformTradeIdsAsync(trader.Platform, trader.Id, fetchedTradeIds, ct);
|
||||
|
||||
// Collect all unique AssetIds we might need to resolve
|
||||
var assetIdsToResolve = validTrades
|
||||
.Where(tr => !knownTradeIds.Contains(tr.PlatformTradeId) || isInitial)
|
||||
.Select(tr => tr.AssetId)
|
||||
.Where(id => !string.IsNullOrEmpty(id))
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
// Pre-fill local cache with bulk query
|
||||
var missingAssetIds = assetIdsToResolve.Where(id => !outcomeCache.ContainsKey(id!)).ToList();
|
||||
if (missingAssetIds.Count > 0)
|
||||
IReadOnlyList<Domain.Entities.Trade> fetchedTrades = new List<Domain.Entities.Trade>();
|
||||
|
||||
if (!skipTradeFetch)
|
||||
{
|
||||
var resolvedOutcomes = await marketRepo.GetOutcomesByTokenIdsAsync(missingAssetIds!, ct);
|
||||
foreach (var o in resolvedOutcomes)
|
||||
if (isDeepResync || isWeeklyBiopsy)
|
||||
{
|
||||
outcomeCache.TryAdd(o.TokenId, o);
|
||||
fetchedTrades = await provider.GetTradesPagedAsync(trader.PlatformUserId, 500, ct);
|
||||
}
|
||||
else
|
||||
{
|
||||
fetchedTrades = await provider.GetTraderTradesAsync(trader.PlatformUserId, TradesPerFetch, ct);
|
||||
}
|
||||
|
||||
var validTrades = fetchedTrades.Where(tr => !string.IsNullOrWhiteSpace(tr.PlatformTradeId)).ToList();
|
||||
|
||||
var updatedName = validTrades.FirstOrDefault(t => !string.IsNullOrEmpty(t.TransientDisplayName))?.TransientDisplayName;
|
||||
if (!string.IsNullOrEmpty(updatedName) && !string.Equals(trader.DisplayName, updatedName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
trader.DisplayName = updatedName;
|
||||
await traderRepo.UpdateAsync(trader, ct);
|
||||
}
|
||||
|
||||
// Classification (IngestMode)
|
||||
var last500 = validTrades.OrderByDescending(t => t.ExecutedAt).Take(500).ToList();
|
||||
if (last500.Count >= 50)
|
||||
{
|
||||
var minDate = last500.Min(x => x.ExecutedAt);
|
||||
var maxDate = last500.Max(x => x.ExecutedAt);
|
||||
var days = (maxDate - minDate).TotalDays;
|
||||
if (days > 0.01)
|
||||
{
|
||||
var tradesPerDay = last500.Count / days;
|
||||
var newMode = trader.IngestMode;
|
||||
|
||||
if (tradesPerDay > 5000) newMode = IngestMode.SnapshotOnly;
|
||||
else if (tradesPerDay > 100 && trader.IngestMode == IngestMode.Full) newMode = IngestMode.Aggregated;
|
||||
else if (trader.IngestMode == IngestMode.SnapshotOnly && tradesPerDay < 2500) newMode = IngestMode.Aggregated;
|
||||
else if (trader.IngestMode == IngestMode.Aggregated && tradesPerDay < 50) newMode = IngestMode.Full;
|
||||
|
||||
if (newMode != trader.IngestMode)
|
||||
{
|
||||
_logger.LogInformation("{Trader}: IngestMode changing from {Old} to {New} (Trades/Day: {TPD:F1})", trader.DisplayName, trader.IngestMode, newMode, tradesPerDay);
|
||||
trader.IngestMode = newMode;
|
||||
await traderRepo.UpdateAsync(trader, ct);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isWeeklyBiopsy)
|
||||
{
|
||||
// "NUR durch den TraderTraitCalculator schicken, NICHT persistieren."
|
||||
// This would require resolving markets and positions and calling TraderTraitCalculator.Compute
|
||||
// For now we skip persisting.
|
||||
fetchedTrades = new List<Domain.Entities.Trade>();
|
||||
}
|
||||
}
|
||||
|
||||
var newTrades = new List<Domain.Entities.Trade>();
|
||||
foreach (var trade in validTrades)
|
||||
if (fetchedTrades.Count > 0)
|
||||
{
|
||||
if (knownTradeIds.Contains(trade.PlatformTradeId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
trade.TraderId = trader.Id;
|
||||
var validTrades = fetchedTrades.Where(tr => !string.IsNullOrWhiteSpace(tr.PlatformTradeId)).ToList();
|
||||
var fetchedTradeIds = validTrades.Select(tr => tr.PlatformTradeId).ToList();
|
||||
var knownTradeIds = await tradeRepo.GetKnownPlatformTradeIdsAsync(trader.Platform, trader.Id, fetchedTradeIds, ct);
|
||||
|
||||
if (!string.IsNullOrEmpty(trade.AssetId))
|
||||
// Collect all unique AssetIds we might need to resolve
|
||||
var assetIdsToResolve = validTrades
|
||||
.Where(tr => !knownTradeIds.Contains(tr.PlatformTradeId) || isInitial)
|
||||
.Select(tr => tr.AssetId)
|
||||
.Where(id => !string.IsNullOrEmpty(id))
|
||||
.Distinct()
|
||||
.ToList();
|
||||
|
||||
// Pre-fill local cache with bulk query
|
||||
var missingAssetIds = assetIdsToResolve.Where(id => !outcomeCache.ContainsKey(id!)).ToList();
|
||||
if (missingAssetIds.Count > 0)
|
||||
{
|
||||
if (outcomeCache.TryGetValue(trade.AssetId, out var outcome))
|
||||
var resolvedOutcomes = await marketRepo.GetOutcomesByTokenIdsAsync(missingAssetIds!, ct);
|
||||
foreach (var o in resolvedOutcomes)
|
||||
{
|
||||
trade.MarketOutcomeId = outcome.Id;
|
||||
trade.Outcome = outcome.Label;
|
||||
if (outcome.Market != null)
|
||||
trade.DbMarketId = outcome.Market.Id;
|
||||
outcomeCache.TryAdd(o.TokenId, o);
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(trade.MarketId))
|
||||
}
|
||||
|
||||
foreach (var trade in validTrades)
|
||||
{
|
||||
if (knownTradeIds.Contains(trade.PlatformTradeId))
|
||||
{
|
||||
// Fallback for missing outcomes: try to fetch market
|
||||
var marketTask = marketFetchCache.GetOrAdd(trade.MarketId, _ => provider.GetMarketAsync(trade.MarketId, ct));
|
||||
var newMarket = await marketTask;
|
||||
if (newMarket != null)
|
||||
{
|
||||
await marketRepo.AddOrUpdateAsync(newMarket, ct);
|
||||
var newOutcome = await marketRepo.GetOutcomeByTokenIdAsync(trade.AssetId, ct);
|
||||
if (newOutcome != null)
|
||||
{
|
||||
outcomeCache.TryAdd(trade.AssetId, newOutcome);
|
||||
trade.MarketOutcomeId = newOutcome.Id;
|
||||
trade.Outcome = newOutcome.Label;
|
||||
if (newOutcome.Market != null)
|
||||
trade.DbMarketId = newOutcome.Market.Id;
|
||||
continue;
|
||||
}
|
||||
|
||||
trade.TraderId = trader.Id;
|
||||
|
||||
if (!string.IsNullOrEmpty(trade.AssetId))
|
||||
{
|
||||
if (outcomeCache.TryGetValue(trade.AssetId, out var outcome))
|
||||
{
|
||||
trade.MarketOutcomeId = outcome.Id;
|
||||
trade.Outcome = outcome.Label;
|
||||
if (outcome.Market != null)
|
||||
trade.DbMarketId = outcome.Market.Id;
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(trade.MarketId))
|
||||
{
|
||||
// Fallback for missing outcomes: try to fetch market
|
||||
var marketTask = marketFetchCache.GetOrAdd(trade.MarketId, _ => provider.GetMarketAsync(trade.MarketId, ct));
|
||||
var newMarket = await marketTask;
|
||||
if (newMarket != null)
|
||||
{
|
||||
await marketRepo.AddOrUpdateAsync(newMarket, ct);
|
||||
var newOutcome = await marketRepo.GetOutcomeByTokenIdAsync(trade.AssetId, ct);
|
||||
if (newOutcome != null)
|
||||
{
|
||||
outcomeCache.TryAdd(trade.AssetId, newOutcome);
|
||||
trade.MarketOutcomeId = newOutcome.Id;
|
||||
trade.Outcome = newOutcome.Label;
|
||||
if (newOutcome.Market != null)
|
||||
trade.DbMarketId = newOutcome.Market.Id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
newTrades.Add(trade);
|
||||
}
|
||||
}
|
||||
|
||||
if (trader.IngestMode == IngestMode.Aggregated && newTrades.Count > 0)
|
||||
{
|
||||
// Aggregate trades: Bucket (TraderId, MarketOutcomeId, Side, Stunde)
|
||||
var aggregated = new List<Domain.Entities.Trade>();
|
||||
foreach (var grp in newTrades.GroupBy(t => new { t.MarketOutcomeId, t.Side, Hour = t.ExecutedAt.ToString("yyyyMMddHH") }))
|
||||
{
|
||||
var first = grp.First();
|
||||
var totalAmount = grp.Sum(t => t.Amount);
|
||||
var totalSize = grp.Sum(t => t.Size);
|
||||
var vwap = totalSize > 0 ? totalAmount / totalSize : first.Price;
|
||||
|
||||
var aggTrade = new Domain.Entities.Trade
|
||||
{
|
||||
PlatformTradeId = $"AGG_{trader.Id}_{grp.Key.MarketOutcomeId}_{grp.Key.Side}_{grp.Key.Hour}",
|
||||
TraderId = trader.Id,
|
||||
MarketOutcomeId = first.MarketOutcomeId,
|
||||
DbMarketId = first.DbMarketId,
|
||||
MarketId = first.MarketId,
|
||||
AssetId = first.AssetId,
|
||||
Outcome = first.Outcome,
|
||||
Side = first.Side,
|
||||
Price = vwap,
|
||||
Amount = totalAmount,
|
||||
Size = totalSize,
|
||||
ExecutedAt = first.ExecutedAt,
|
||||
AggregatedCount = grp.Count()
|
||||
};
|
||||
aggregated.Add(aggTrade);
|
||||
}
|
||||
newTrades = aggregated;
|
||||
}
|
||||
|
||||
if (trader.IngestMode == IngestMode.SnapshotOnly)
|
||||
{
|
||||
// "SnapshotOnly (Tier C): Stündlich: GetTraderPositionsAsync -> TraderPositions upserten"
|
||||
if (!trader.LastTradesUpdatedAt.HasValue || (DateTime.UtcNow - trader.LastTradesUpdatedAt.Value).TotalHours >= 1)
|
||||
{
|
||||
try
|
||||
{
|
||||
var positions = await provider.GetTraderPositionsAsync(trader.PlatformUserId, ct);
|
||||
if (positions != null && positions.Count > 0)
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<Predictalytics.Infrastructure.Data.AppDbContext>();
|
||||
// For simplicity, just use the endpoint's positions
|
||||
var existingPos = db.TraderPositions.Where(tp => tp.TraderId == trader.Id).ToList();
|
||||
foreach(var info in positions)
|
||||
{
|
||||
int? marketOutcomeId = null;
|
||||
if (!string.IsNullOrEmpty(info.AssetId))
|
||||
{
|
||||
var outcome = await marketRepo.GetOutcomeByTokenIdAsync(info.AssetId, ct);
|
||||
if (outcome != null)
|
||||
{
|
||||
marketOutcomeId = outcome.Id;
|
||||
}
|
||||
}
|
||||
|
||||
if (!marketOutcomeId.HasValue) continue;
|
||||
|
||||
var ex = existingPos.FirstOrDefault(ep => ep.MarketOutcomeId == marketOutcomeId.Value);
|
||||
if (ex != null)
|
||||
{
|
||||
ex.SharesHeld = info.Size;
|
||||
ex.AvgCost = info.AveragePrice;
|
||||
// RealizedPnl is built by tape replay, we skip it for SnapshotOnly
|
||||
}
|
||||
else
|
||||
{
|
||||
db.TraderPositions.Add(new Predictalytics.Domain.Entities.TraderPosition
|
||||
{
|
||||
TraderId = trader.Id,
|
||||
MarketOutcomeId = marketOutcomeId.Value,
|
||||
SharesHeld = info.Size,
|
||||
AvgCost = info.AveragePrice,
|
||||
RealizedPnl = 0
|
||||
});
|
||||
}
|
||||
}
|
||||
await db.SaveChangesAsync(ct);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to fetch positions for SnapshotOnly trader {TraderId}", trader.Id);
|
||||
}
|
||||
}
|
||||
newTrades.Add(trade);
|
||||
}
|
||||
|
||||
if (newTrades.Count > 0)
|
||||
|
||||
Reference in New Issue
Block a user