Fix Estimator scoring and PnL engine cashflow

This commit is contained in:
Richard
2026-07-07 18:30:20 +02:00
parent 8d056653f9
commit 4eb0d99b4e
17 changed files with 2594 additions and 97 deletions
@@ -155,7 +155,7 @@ public class AnalyticsService : IAnalyticsService
if (highlyCopyable) if (highlyCopyable)
{ {
traders = traders.Where(t => t.CurrentScore != null && t.CurrentScore.CopytradingScore >= 60).ToList(); traders = traders.Where(t => t.Analytics != null && t.Analytics.CopytradingScore >= 60).ToList();
} }
traders = traders.Skip(skip).Take(take).ToList(); traders = traders.Skip(skip).Take(take).ToList();
@@ -201,6 +201,7 @@ public class AnalyticsService : IAnalyticsService
var trades = await _tradeRepo.GetByTraderIdAsync(traderId, 0, 50, ct); var trades = await _tradeRepo.GetByTraderIdAsync(traderId, 0, 50, ct);
var wl = await _watchlistRepo.GetByTraderIdAsync(traderId, ct); var wl = await _watchlistRepo.GetByTraderIdAsync(traderId, ct);
var s = trader.CurrentScore; var s = trader.CurrentScore;
var a = trader.Analytics;
var perfs = trader.CategoryPerformances.Select(p => new TraderCategoryPerformanceDto( var perfs = trader.CategoryPerformances.Select(p => new TraderCategoryPerformanceDto(
p.Category.ToString(), p.Category.ToString(),
p.TotalVolume, p.TotalVolume,
@@ -213,7 +214,7 @@ public class AnalyticsService : IAnalyticsService
trader.Notes, trader.Tier.ToString(), trader.Strategy.ToString(), trader.IsSuspectedBot, trader.ManualPriorityOverride, trader.Notes, trader.Tier.ToString(), trader.Strategy.ToString(), trader.IsSuspectedBot, trader.ManualPriorityOverride,
trader.WinRate, trader.TotalPnl, trader.TotalTrades, trader.WinRate, trader.TotalPnl, trader.TotalTrades,
s?.ActivityScore ?? 0, s?.QualityScore ?? 0, s?.VolumeScore ?? 0, s?.TimingScore ?? 0, s?.ActivityScore ?? 0, s?.QualityScore ?? 0, s?.VolumeScore ?? 0, s?.TimingScore ?? 0,
s?.CombinedScore ?? 0, s?.CopytradingScore ?? 0, s?.CopytradingQualityScore ?? 0, s?.CopytradingCopyabilityScore ?? 0, s?.Rank ?? 0, wl != null, trader.CreatedAt, trader.LastPolledAt, s?.CombinedScore ?? 0, a?.CopytradingScore ?? 0, a?.CopytradingQualityScore ?? 0, a?.CopytradingCopyabilityScore ?? 0, s?.Rank ?? 0, wl != null, trader.CreatedAt, trader.LastPolledAt,
trader.AiStrategySummary, trader.AiStrategySummary,
trades.Select(MapTradeDto).ToList(), trades.Select(MapTradeDto).ToList(),
perfs); perfs);
@@ -451,8 +452,8 @@ public class AnalyticsService : IAnalyticsService
private static TraderDto MapTraderDto(Trader t, HashSet<int> wIds) => new( private static TraderDto MapTraderDto(Trader t, HashSet<int> wIds) => new(
t.Id, t.Platform.ToString(), t.PlatformUserId, t.DisplayName, t.Tier.ToString(), t.Strategy.ToString(), t.Id, t.Platform.ToString(), t.PlatformUserId, t.DisplayName, t.Tier.ToString(), t.Strategy.ToString(),
t.CurrentScore?.CombinedScore ?? 0, t.CurrentScore?.CopytradingScore ?? 0, t.CurrentScore?.CombinedScore ?? 0, t.Analytics?.CopytradingScore ?? 0,
t.CurrentScore?.CopytradingQualityScore ?? 0, t.CurrentScore?.CopytradingCopyabilityScore ?? 0, t.Analytics?.CopytradingQualityScore ?? 0, t.Analytics?.CopytradingCopyabilityScore ?? 0,
t.WinRate, t.TotalPnl, t.TotalTrades, t.WinRate, t.TotalPnl, t.TotalTrades,
wIds.Contains(t.Id), t.IsSuspectedBot, t.LastPolledAt); wIds.Contains(t.Id), t.IsSuspectedBot, t.LastPolledAt);
@@ -70,22 +70,27 @@ public class ScoringService : IScoringService
if (intervals.Average() < 10) botIndicators.Add("Sub-10s trade frequency"); if (intervals.Average() < 10) botIndicators.Add("Sub-10s trade frequency");
} }
trader.IsSuspectedBot = botIndicators.Count > 0; bool aiControlsStrategy = trader.AiStrategyUpdatedAt.HasValue && trader.AiStrategyUpdatedAt.Value > DateTime.UtcNow.AddDays(-7);
var marketKeys = trades if (!aiControlsStrategy)
.Select(t => t.DbMarketId.HasValue ? t.DbMarketId.Value.ToString() : t.MarketId) {
.Where(k => !string.IsNullOrEmpty(k)) trader.IsSuspectedBot = botIndicators.Count > 0;
.ToList();
var marketsTraded = marketKeys.Distinct().Count();
var hedgeGroups = trades
.GroupBy(t => t.DbMarketId.HasValue ? t.DbMarketId.Value.ToString() : t.MarketId)
.Where(g => !string.IsNullOrEmpty(g.Key) && g.Select(t => t.Outcome).Distinct().Count() > 1);
var hedgingRate = marketsTraded > 0 ? (decimal)hedgeGroups.Count() / marketsTraded * 100 : 0;
trader.Strategy = avgSize > 10000 ? Predictalytics.Domain.Enums.StrategyType.Whale : var marketKeys = trades
hedgingRate > 30 ? Predictalytics.Domain.Enums.StrategyType.Hedger : .Select(t => t.DbMarketId.HasValue ? t.DbMarketId.Value.ToString() : t.MarketId)
trader.IsSuspectedBot ? Predictalytics.Domain.Enums.StrategyType.Bot : .Where(k => !string.IsNullOrEmpty(k))
Predictalytics.Domain.Enums.StrategyType.Unknown; .ToList();
var marketsTraded = marketKeys.Distinct().Count();
var hedgeGroups = trades
.GroupBy(t => t.DbMarketId.HasValue ? t.DbMarketId.Value.ToString() : t.MarketId)
.Where(g => !string.IsNullOrEmpty(g.Key) && g.Select(t => t.Outcome).Distinct().Count() > 1);
var hedgingRate = marketsTraded > 0 ? (decimal)hedgeGroups.Count() / marketsTraded * 100 : 0;
trader.Strategy = avgSize > 10000 ? Predictalytics.Domain.Enums.StrategyType.Whale :
hedgingRate > 30 ? Predictalytics.Domain.Enums.StrategyType.Hedger :
trader.IsSuspectedBot ? Predictalytics.Domain.Enums.StrategyType.Bot :
Predictalytics.Domain.Enums.StrategyType.Unknown;
}
var score = new PriorityScore(activityScore, qualityScore, volumeScore, timingScore, combined, trader.ManualPriorityOverride); var score = new PriorityScore(activityScore, qualityScore, volumeScore, timingScore, combined, trader.ManualPriorityOverride);
@@ -25,6 +25,15 @@ public class TraderAnalytics
public decimal EstimatedBankroll { get; set; } public decimal EstimatedBankroll { get; set; }
public decimal CurrentBalance { get; set; } public decimal CurrentBalance { get; set; }
/// <summary>Copytrading suitability score (0-100). Generated by the Estimator.</summary>
public decimal CopytradingScore { get; set; }
/// <summary>The pure skill/edge dimension of the copytrading score (0-100). Generated by the Estimator.</summary>
public decimal CopytradingQualityScore { get; set; }
/// <summary>The copyability dimension (alpha-decay, sizing consistency) of the copytrading score (0-100). Generated by the Estimator.</summary>
public decimal CopytradingCopyabilityScore { get; set; }
// Navigation // Navigation
public virtual Trader Trader { get; set; } = null!; public virtual Trader Trader { get; set; } = null!;
} }
@@ -0,0 +1,19 @@
using System;
namespace Predictalytics.Domain.Entities;
/// <summary>
/// Daily snapshot of a trader's performance metrics.
/// Used to calculate rolling time windows (e.g. 24h, 7d, 30d PnL).
/// </summary>
public class TraderDailySnapshot
{
public int Id { get; set; }
public int TraderId { get; set; }
public DateTime Date { get; set; }
public decimal TotalPnl { get; set; }
public decimal CurrentBalance { get; set; }
public Trader Trader { get; set; } = null!;
}
@@ -28,6 +28,12 @@ public class TraderPosition
/// <summary>ID of the last trade applied to this position.</summary> /// <summary>ID of the last trade applied to this position.</summary>
public long LastAppliedTradeId { get; set; } public long LastAppliedTradeId { get; set; }
/// <summary>Timestamp of the last trade applied. Used to detect out-of-order inserts.</summary>
public DateTime? LastTradeExecutedAt { get; set; }
/// <summary>True if old trades for this position have been pruned/deleted. Prevents resetting.</summary>
public bool IsHistoryPruned { get; set; }
/// <summary>When this position was last updated.</summary> /// <summary>When this position was last updated.</summary>
public DateTime LastUpdatedAt { get; set; } = DateTime.UtcNow; public DateTime LastUpdatedAt { get; set; } = DateTime.UtcNow;
@@ -29,15 +29,6 @@ public class TraderScore
/// <summary>Overall rank among all tracked traders.</summary> /// <summary>Overall rank among all tracked traders.</summary>
public int Rank { get; set; } public int Rank { get; set; }
/// <summary>Copytrading suitability score (0-100).</summary>
public decimal CopytradingScore { get; set; }
/// <summary>The pure skill/edge dimension of the copytrading score (0-100).</summary>
public decimal CopytradingQualityScore { get; set; }
/// <summary>The copyability dimension (alpha-decay, sizing consistency) of the copytrading score (0-100).</summary>
public decimal CopytradingCopyabilityScore { get; set; }
/// <summary>When this score was last calculated.</summary> /// <summary>When this score was last calculated.</summary>
public DateTime CalculatedAt { get; set; } = DateTime.UtcNow; public DateTime CalculatedAt { get; set; } = DateTime.UtcNow;
@@ -18,6 +18,7 @@ public class AppDbContext : DbContext
public DbSet<MarketAnalytics> MarketAnalytics => Set<MarketAnalytics>(); public DbSet<MarketAnalytics> MarketAnalytics => Set<MarketAnalytics>();
public DbSet<TraderPosition> TraderPositions => Set<TraderPosition>(); public DbSet<TraderPosition> TraderPositions => Set<TraderPosition>();
public DbSet<MarketOutcomePriceSnapshot> MarketOutcomePriceSnapshots => Set<MarketOutcomePriceSnapshot>(); public DbSet<MarketOutcomePriceSnapshot> MarketOutcomePriceSnapshots => Set<MarketOutcomePriceSnapshot>();
public DbSet<TraderDailySnapshot> TraderDailySnapshots => Set<TraderDailySnapshot>();
public DbSet<TraderCategoryPerformance> TraderCategoryPerformances => Set<TraderCategoryPerformance>(); public DbSet<TraderCategoryPerformance> TraderCategoryPerformances => Set<TraderCategoryPerformance>();
public DbSet<TradeContext> TradeContexts => Set<TradeContext>(); public DbSet<TradeContext> TradeContexts => Set<TradeContext>();
public DbSet<BackgroundJob> BackgroundJobs => Set<BackgroundJob>(); public DbSet<BackgroundJob> BackgroundJobs => Set<BackgroundJob>();
@@ -119,12 +120,13 @@ public class AppDbContext : DbContext
mb.Entity<TraderScore>(e => mb.Entity<TraderScore>(e =>
{ {
e.HasKey(s => s.Id); e.HasKey(s => s.Id);
e.Property(s => s.ActivityScore).HasPrecision(8, 4); e.HasIndex(s => s.TraderId).IsUnique();
e.Property(s => s.QualityScore).HasPrecision(8, 4); e.Property(s => s.ActivityScore).HasPrecision(5, 2);
e.Property(s => s.CombinedScore).HasPrecision(8, 4); e.Property(s => s.QualityScore).HasPrecision(5, 2);
e.Property(s => s.VolumeScore).HasPrecision(8, 4); e.Property(s => s.CombinedScore).HasPrecision(5, 2);
e.Property(s => s.TimingScore).HasPrecision(8, 4); e.Property(s => s.VolumeScore).HasPrecision(5, 2);
e.Property(s => s.CopytradingScore).HasPrecision(8, 4); e.Property(s => s.TimingScore).HasPrecision(5, 2);
e.HasOne(s => s.Trader).WithOne(t => t.CurrentScore).HasForeignKey<TraderScore>(s => s.TraderId).OnDelete(DeleteBehavior.Cascade);
}); });
// WatchlistEntry // WatchlistEntry
@@ -213,6 +215,16 @@ public class AppDbContext : DbContext
e.HasOne(tp => tp.MarketOutcome).WithMany().HasForeignKey(tp => tp.MarketOutcomeId).OnDelete(DeleteBehavior.Cascade); e.HasOne(tp => tp.MarketOutcome).WithMany().HasForeignKey(tp => tp.MarketOutcomeId).OnDelete(DeleteBehavior.Cascade);
}); });
// TraderDailySnapshot
mb.Entity<TraderDailySnapshot>(e =>
{
e.HasKey(s => s.Id);
e.HasIndex(s => new { s.TraderId, s.Date }).IsUnique();
e.Property(s => s.TotalPnl).HasPrecision(18, 4);
e.Property(s => s.CurrentBalance).HasPrecision(18, 4);
e.HasOne(s => s.Trader).WithMany().HasForeignKey(s => s.TraderId).OnDelete(DeleteBehavior.Cascade);
});
// MarketOutcomePriceSnapshot // MarketOutcomePriceSnapshot
mb.Entity<MarketOutcomePriceSnapshot>(e => mb.Entity<MarketOutcomePriceSnapshot>(e =>
{ {
@@ -2,13 +2,20 @@ using Predictalytics.Domain.Entities;
using Predictalytics.Domain.Enums; using Predictalytics.Domain.Enums;
using Predictalytics.Domain.Interfaces; using Predictalytics.Domain.Interfaces;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace Predictalytics.Infrastructure.Data.Repositories; namespace Predictalytics.Infrastructure.Data.Repositories;
public class TradeRepository : ITradeRepository public class TradeRepository : ITradeRepository
{ {
private readonly AppDbContext _db; private readonly AppDbContext _db;
public TradeRepository(AppDbContext db) => _db = db; private readonly Microsoft.Extensions.Logging.ILogger<TradeRepository> _logger;
public TradeRepository(AppDbContext db, Microsoft.Extensions.Logging.ILogger<TradeRepository> logger)
{
_db = db;
_logger = logger;
}
public async Task<Trade?> GetByPlatformTradeIdAsync(PlatformType platform, string platformTradeId, CancellationToken ct = default) public async Task<Trade?> GetByPlatformTradeIdAsync(PlatformType platform, string platformTradeId, CancellationToken ct = default)
=> await _db.Trades.FirstOrDefaultAsync(t => t.Platform == platform && t.PlatformTradeId == platformTradeId, ct); => await _db.Trades.FirstOrDefaultAsync(t => t.Platform == platform && t.PlatformTradeId == platformTradeId, ct);
@@ -92,7 +99,8 @@ public class TradeRepository : ITradeRepository
parameters.Add(t.IsContextEnriched); parameters.Add(t.IsContextEnriched);
} }
await _db.Database.ExecuteSqlRawAsync(sb.ToString(), parameters.ToArray(), ct); var rowsInserted = await _db.Database.ExecuteSqlRawAsync(sb.ToString(), parameters.ToArray(), ct);
_logger.LogInformation("Inserted {RowsInserted} trades into the database.", rowsInserted);
} }
} }
@@ -141,10 +149,11 @@ public class TradeRepository : ITradeRepository
.Include(t => t.Trader) .Include(t => t.Trader)
.Include(t => t.Trader.CurrentScore) .Include(t => t.Trader.CurrentScore)
.Include(t => t.Trader.WatchlistEntries) .Include(t => t.Trader.WatchlistEntries)
.Include(t => t.Trader.Analytics)
.Where(t => !t.IsContextEnriched .Where(t => !t.IsContextEnriched
&& t.Platform == PlatformType.Polymarket && t.Platform == PlatformType.Polymarket
&& t.AssetId != "") && t.AssetId != "")
.Where(t => t.Trader.WatchlistEntries.Any() || (t.Trader.CurrentScore != null && t.Trader.CurrentScore.CopytradingScore > 50)) .Where(t => t.Trader.WatchlistEntries.Any() || (t.Trader.Analytics != null && t.Trader.Analytics.CopytradingScore > 50))
.OrderByDescending(t => t.ExecutedAt) .OrderByDescending(t => t.ExecutedAt)
.Take(limit) .Take(limit)
.ToListAsync(ct); .ToListAsync(ct);
@@ -0,0 +1,43 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Predictalytics.Infrastructure.Migrations
{
/// <inheritdoc />
public partial class AddTraderPositionHistoryGuards : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "IsHistoryPruned",
table: "TraderPositions",
type: "tinyint(1)",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<DateTime>(
name: "LastTradeExecutedAt",
table: "TraderPositions",
type: "datetime(6)",
nullable: true);
// Step 2.1: Global Reset to force re-calculation of all positions and fix corrupted PnL data
migrationBuilder.Sql("UPDATE TraderPositions SET LastAppliedTradeId = 0;");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "IsHistoryPruned",
table: "TraderPositions");
migrationBuilder.DropColumn(
name: "LastTradeExecutedAt",
table: "TraderPositions");
}
}
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,52 @@
using System;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Predictalytics.Infrastructure.Migrations
{
/// <inheritdoc />
public partial class AddTraderDailySnapshots : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "TraderDailySnapshots",
columns: table => new
{
Id = table.Column<int>(type: "int", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
TraderId = table.Column<int>(type: "int", nullable: false),
Date = table.Column<DateTime>(type: "datetime(6)", nullable: false),
TotalPnl = table.Column<decimal>(type: "decimal(18,4)", precision: 18, scale: 4, nullable: false),
CurrentBalance = table.Column<decimal>(type: "decimal(18,4)", precision: 18, scale: 4, nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_TraderDailySnapshots", x => x.Id);
table.ForeignKey(
name: "FK_TraderDailySnapshots_Traders_TraderId",
column: x => x.TraderId,
principalTable: "Traders",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateIndex(
name: "IX_TraderDailySnapshots_TraderId_Date",
table: "TraderDailySnapshots",
columns: new[] { "TraderId", "Date" },
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "TraderDailySnapshots");
}
}
}
@@ -718,6 +718,36 @@ namespace Predictalytics.Infrastructure.Migrations
b.ToTable("TraderCategoryPerformances"); b.ToTable("TraderCategoryPerformances");
}); });
modelBuilder.Entity("Predictalytics.Domain.Entities.TraderDailySnapshot", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<int>("Id"));
b.Property<decimal>("CurrentBalance")
.HasPrecision(18, 4)
.HasColumnType("decimal(18,4)");
b.Property<DateTime>("Date")
.HasColumnType("datetime(6)");
b.Property<decimal>("TotalPnl")
.HasPrecision(18, 4)
.HasColumnType("decimal(18,4)");
b.Property<int>("TraderId")
.HasColumnType("int");
b.HasKey("Id");
b.HasIndex("TraderId", "Date")
.IsUnique();
b.ToTable("TraderDailySnapshots");
});
modelBuilder.Entity("Predictalytics.Domain.Entities.TraderPosition", b => modelBuilder.Entity("Predictalytics.Domain.Entities.TraderPosition", b =>
{ {
b.Property<int>("Id") b.Property<int>("Id")
@@ -730,9 +760,15 @@ namespace Predictalytics.Infrastructure.Migrations
.HasPrecision(10, 6) .HasPrecision(10, 6)
.HasColumnType("decimal(10,6)"); .HasColumnType("decimal(10,6)");
b.Property<bool>("IsHistoryPruned")
.HasColumnType("tinyint(1)");
b.Property<long>("LastAppliedTradeId") b.Property<long>("LastAppliedTradeId")
.HasColumnType("bigint"); .HasColumnType("bigint");
b.Property<DateTime?>("LastTradeExecutedAt")
.HasColumnType("datetime(6)");
b.Property<DateTime>("LastUpdatedAt") b.Property<DateTime>("LastUpdatedAt")
.HasColumnType("datetime(6)"); .HasColumnType("datetime(6)");
@@ -970,6 +1006,17 @@ namespace Predictalytics.Infrastructure.Migrations
b.Navigation("Trader"); b.Navigation("Trader");
}); });
modelBuilder.Entity("Predictalytics.Domain.Entities.TraderDailySnapshot", b =>
{
b.HasOne("Predictalytics.Domain.Entities.Trader", "Trader")
.WithMany()
.HasForeignKey("TraderId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Trader");
});
modelBuilder.Entity("Predictalytics.Domain.Entities.TraderPosition", b => modelBuilder.Entity("Predictalytics.Domain.Entities.TraderPosition", b =>
{ {
b.HasOne("Predictalytics.Domain.Entities.MarketOutcome", "MarketOutcome") b.HasOne("Predictalytics.Domain.Entities.MarketOutcome", "MarketOutcome")
@@ -48,6 +48,30 @@ public class PositionPnLEngine : IPositionPnLEngine
.Where(tp => tp.TraderId == traderId) .Where(tp => tp.TraderId == traderId)
.ToDictionaryAsync(tp => tp.MarketOutcomeId, ct); .ToDictionaryAsync(tp => tp.MarketOutcomeId, ct);
// Pre-flight check: Detect out-of-order unapplied trades (Bug 3) or manually reset checkpoints (Bug 2a)
foreach (var pos in existingPositions.Values)
{
if (pos.IsHistoryPruned) continue; // Cannot reset if history is pruned
var outcomeTrades = trades.Where(t => t.MarketOutcomeId == pos.MarketOutcomeId).ToList();
bool needsReset = pos.LastAppliedTradeId == 0 ||
outcomeTrades.Any(t => t.Id > pos.LastAppliedTradeId && pos.LastTradeExecutedAt.HasValue && t.ExecutedAt < pos.LastTradeExecutedAt.Value);
if (needsReset)
{
if (pos.LastAppliedTradeId > 0)
{
_logger.LogInformation("Out-of-order unapplied trade detected for Trader {TraderId} Outcome {OutcomeId}. Resetting position.", traderId, pos.MarketOutcomeId);
}
pos.SharesHeld = 0;
pos.AvgCost = 0;
pos.RealizedPnl = 0;
pos.LastAppliedTradeId = 0;
pos.LastTradeExecutedAt = null;
}
}
var analytics = trader.Analytics; var analytics = trader.Analytics;
if (analytics == null) if (analytics == null)
{ {
@@ -61,10 +85,6 @@ public class PositionPnLEngine : IPositionPnLEngine
var cutoff7d = DateTime.UtcNow.AddDays(-7); var cutoff7d = DateTime.UtcNow.AddDays(-7);
var cutoff24h = DateTime.UtcNow.AddHours(-24); var cutoff24h = DateTime.UtcNow.AddHours(-24);
var realizedPnl30d = 0m;
var realizedPnl7d = 0m;
var realizedPnl24h = 0m;
decimal currentBalance = analytics.CurrentBalance; decimal currentBalance = analytics.CurrentBalance;
decimal estimatedBankroll = analytics.EstimatedBankroll; decimal estimatedBankroll = analytics.EstimatedBankroll;
@@ -97,10 +117,13 @@ public class PositionPnLEngine : IPositionPnLEngine
{ {
TraderId = traderId, TraderId = traderId,
MarketOutcomeId = outcomeId, MarketOutcomeId = outcomeId,
MarketOutcome = trade.MarketOutcome, // Bug 8 Fix: Assign immediately so virtual payout runs
SharesHeld = 0, SharesHeld = 0,
AvgCost = 0, AvgCost = 0,
RealizedPnl = 0, RealizedPnl = 0,
LastAppliedTradeId = 0 LastAppliedTradeId = 0,
LastTradeExecutedAt = null,
IsHistoryPruned = false
}; };
} }
pos.LastUpdatedAt = DateTime.UtcNow; pos.LastUpdatedAt = DateTime.UtcNow;
@@ -160,6 +183,25 @@ public class PositionPnLEngine : IPositionPnLEngine
case TradeSide.Split: case TradeSide.Split:
case TradeSide.Merge: case TradeSide.Merge:
var cashEquivalent = Math.Abs(trade.Size) * trade.Price;
if (trade.Size > 0)
{
currentBalance -= cashEquivalent;
var totalCost = (pos.SharesHeld * pos.AvgCost) + cashEquivalent;
var totalShares = pos.SharesHeld + trade.Size;
pos.AvgCost = totalShares > 0 ? totalCost / totalShares : 0;
pos.SharesHeld = totalShares;
}
else if (trade.Size < 0)
{
var absSize = Math.Abs(trade.Size);
currentBalance += cashEquivalent;
var splitSizeToSell = Math.Min(absSize, pos.SharesHeld);
pos.RealizedPnl += splitSizeToSell * (trade.Price - pos.AvgCost);
pos.SharesHeld -= absSize;
if (pos.SharesHeld < 0) pos.SharesHeld = 0;
}
break;
case TradeSide.AddLiquidity: case TradeSide.AddLiquidity:
case TradeSide.RemoveLiquidity: case TradeSide.RemoveLiquidity:
case TradeSide.Unknown: case TradeSide.Unknown:
@@ -174,15 +216,12 @@ public class PositionPnLEngine : IPositionPnLEngine
} }
pos.LastAppliedTradeId = Math.Max(pos.LastAppliedTradeId, trade.Id); pos.LastAppliedTradeId = Math.Max(pos.LastAppliedTradeId, trade.Id);
if (pos.LastTradeExecutedAt == null || trade.ExecutedAt > pos.LastTradeExecutedAt.Value)
{
pos.LastTradeExecutedAt = trade.ExecutedAt;
}
var realizedPnlDelta = pos.RealizedPnl - previousRealizedPnl; var realizedPnlDelta = pos.RealizedPnl - previousRealizedPnl;
if (realizedPnlDelta != 0)
{
// This will only accumulate deltas for NEW trades.
if (trade.ExecutedAt >= cutoff30d) realizedPnl30d += realizedPnlDelta;
if (trade.ExecutedAt >= cutoff7d) realizedPnl7d += realizedPnlDelta;
if (trade.ExecutedAt >= cutoff24h) realizedPnl24h += realizedPnlDelta;
}
} }
// Bug 6: Virtual payout for unredeemed winning positions // Bug 6: Virtual payout for unredeemed winning positions
@@ -200,10 +239,6 @@ public class PositionPnLEngine : IPositionPnLEngine
pos.RealizedPnl += virtualPnlDelta; pos.RealizedPnl += virtualPnlDelta;
pos.SharesHeld = 0; pos.SharesHeld = 0;
pos.AvgCost = 0; pos.AvgCost = 0;
if (market.ClosedAt.HasValue && market.ClosedAt.Value >= cutoff30d) realizedPnl30d += virtualPnlDelta;
if (market.ClosedAt.HasValue && market.ClosedAt.Value >= cutoff7d) realizedPnl7d += virtualPnlDelta;
if (market.ClosedAt.HasValue && market.ClosedAt.Value >= cutoff24h) realizedPnl24h += virtualPnlDelta;
} }
} }
} }
@@ -211,9 +246,6 @@ public class PositionPnLEngine : IPositionPnLEngine
// Persist new / updated positions and calculate total values // Persist new / updated positions and calculate total values
decimal totalRealizedPnl = 0; decimal totalRealizedPnl = 0;
decimal totalUnrealizedPnl = 0; decimal totalUnrealizedPnl = 0;
decimal unrealizedPnl30d = 0;
decimal unrealizedPnl7d = 0;
decimal unrealizedPnl24h = 0;
foreach (var pos in tempPositions.Values) foreach (var pos in tempPositions.Values)
{ {
@@ -222,10 +254,6 @@ public class PositionPnLEngine : IPositionPnLEngine
{ {
var unrealized = pos.SharesHeld * (outcome.CurrentPrice - pos.AvgCost); var unrealized = pos.SharesHeld * (outcome.CurrentPrice - pos.AvgCost);
totalUnrealizedPnl += unrealized; totalUnrealizedPnl += unrealized;
if (tradedOutcomes30d.Contains(pos.MarketOutcomeId)) unrealizedPnl30d += unrealized;
if (tradedOutcomes7d.Contains(pos.MarketOutcomeId)) unrealizedPnl7d += unrealized;
if (tradedOutcomes24h.Contains(pos.MarketOutcomeId)) unrealizedPnl24h += unrealized;
} }
totalRealizedPnl += pos.RealizedPnl; totalRealizedPnl += pos.RealizedPnl;
@@ -239,27 +267,60 @@ public class PositionPnLEngine : IPositionPnLEngine
} }
} }
// Remove positions for outcomes that have no trades anymore // Bug 7: Positions are intentionally kept even if their trades are pruned by retention policies.
foreach (var outcomeId in existingPositions.Keys)
{
if (!tempPositions.ContainsKey(outcomeId))
{
_db.TraderPositions.Remove(existingPositions[outcomeId]);
}
}
// Update analytics record // Update analytics record
// (Analytics instance is already retrieved at the top of this method) // (Analytics instance is already retrieved at the top of this method)
var overallPnl = totalRealizedPnl + totalUnrealizedPnl; var overallPnl = totalRealizedPnl + totalUnrealizedPnl;
analytics.OverallPnL = overallPnl; analytics.OverallPnL = overallPnl;
analytics.PnL30d = realizedPnl30d + unrealizedPnl30d;
analytics.PnL7d = realizedPnl7d + unrealizedPnl7d;
analytics.PnL24h = realizedPnl24h + unrealizedPnl24h;
analytics.CurrentBalance = currentBalance; analytics.CurrentBalance = currentBalance;
analytics.EstimatedBankroll = estimatedBankroll; analytics.EstimatedBankroll = estimatedBankroll;
// Bug 1: Calculate time windows based on historical snapshots
var today = DateTime.UtcNow.Date;
// Save today's snapshot
var todaySnapshot = await _db.TraderDailySnapshots
.FirstOrDefaultAsync(s => s.TraderId == traderId && s.Date == today, ct);
if (todaySnapshot == null)
{
todaySnapshot = new TraderDailySnapshot
{
TraderId = traderId,
Date = today,
TotalPnl = overallPnl,
CurrentBalance = currentBalance
};
_db.TraderDailySnapshots.Add(todaySnapshot);
}
else
{
todaySnapshot.TotalPnl = overallPnl;
todaySnapshot.CurrentBalance = currentBalance;
}
// Fetch historical snapshots
var snapshot24h = await _db.TraderDailySnapshots
.Where(s => s.TraderId == traderId && s.Date <= today.AddDays(-1))
.OrderByDescending(s => s.Date)
.FirstOrDefaultAsync(ct);
var snapshot7d = await _db.TraderDailySnapshots
.Where(s => s.TraderId == traderId && s.Date <= today.AddDays(-7))
.OrderByDescending(s => s.Date)
.FirstOrDefaultAsync(ct);
var snapshot30d = await _db.TraderDailySnapshots
.Where(s => s.TraderId == traderId && s.Date <= today.AddDays(-30))
.OrderByDescending(s => s.Date)
.FirstOrDefaultAsync(ct);
analytics.PnL24h = overallPnl - (snapshot24h?.TotalPnl ?? 0);
analytics.PnL7d = overallPnl - (snapshot7d?.TotalPnl ?? 0);
analytics.PnL30d = overallPnl - (snapshot30d?.TotalPnl ?? 0);
// Calculate Win Rate on Market level // Calculate Win Rate on Market level
var (winRateOverall, winRate30d, winRate7d, winRate24h) = CalculateMarketWinRates(trades, tempPositions, cutoff30d, cutoff7d, cutoff24h); var (winRateOverall, winRate30d, winRate7d, winRate24h) = CalculateMarketWinRates(trades, tempPositions, cutoff30d, cutoff7d, cutoff24h);
@@ -102,6 +102,12 @@ public class TradeReconciliationWorker : BackgroundService
} }
// 2. Bulk update orphaned trades // 2. Bulk update orphaned trades
var pairsToReset = await db.Trades
.Where(t => t.MarketOutcomeId == null && t.AssetId != "")
.Join(db.MarketOutcomes, t => t.AssetId, o => o.TokenId, (t, o) => new { t.TraderId, MarketOutcomeId = o.Id })
.Distinct()
.ToListAsync(ct);
reconciledCount = await Microsoft.EntityFrameworkCore.RelationalDatabaseFacadeExtensions.ExecuteSqlRawAsync(db.Database, @" reconciledCount = await Microsoft.EntityFrameworkCore.RelationalDatabaseFacadeExtensions.ExecuteSqlRawAsync(db.Database, @"
UPDATE Trades t UPDATE Trades t
INNER JOIN MarketOutcomes o ON t.AssetId = o.TokenId INNER JOIN MarketOutcomes o ON t.AssetId = o.TokenId
@@ -112,21 +118,35 @@ public class TradeReconciliationWorker : BackgroundService
WHERE t.MarketOutcomeId IS NULL AND t.AssetId != ''; WHERE t.MarketOutcomeId IS NULL AND t.AssetId != '';
", ct); ", ct);
if (reconciledCount > 0) if (reconciledCount > 0 && pairsToReset.Any())
{ {
// Force re-analysis of traders who now have new linked trades that haven't been applied // Reset checkpoints for positions that received newly linked older trades
await Microsoft.EntityFrameworkCore.RelationalDatabaseFacadeExtensions.ExecuteSqlRawAsync(db.Database, @" var posIdsToReset = await db.TraderPositions
UPDATE Traders t .Where(tp => pairsToReset.Select(p => p.TraderId).Contains(tp.TraderId) &&
SET LastAnalyzedAt = NULL pairsToReset.Select(p => p.MarketOutcomeId).Contains(tp.MarketOutcomeId))
WHERE LastAnalyzedAt IS NOT NULL .Select(tp => tp.Id)
AND EXISTS ( .ToListAsync(ct);
SELECT 1 FROM Trades tr
LEFT JOIN TraderPositions tp ON tr.TraderId = tp.TraderId AND tr.MarketOutcomeId = tp.MarketOutcomeId // Need to filter client side since EF core can't translate tuple Contains
WHERE tr.TraderId = t.Id var actualPosIdsToReset = (await db.TraderPositions
AND tr.MarketOutcomeId IS NOT NULL .Where(tp => posIdsToReset.Contains(tp.Id))
AND (tp.Id IS NULL OR tr.Id > tp.LastAppliedTradeId) .ToListAsync(ct))
); .Where(tp => pairsToReset.Any(p => p.TraderId == tp.TraderId && p.MarketOutcomeId == tp.MarketOutcomeId))
", ct); .Select(tp => tp.Id)
.ToList();
if (actualPosIdsToReset.Any())
{
await db.TraderPositions
.Where(tp => actualPosIdsToReset.Contains(tp.Id))
.ExecuteUpdateAsync(s => s.SetProperty(p => p.LastAppliedTradeId, 0), ct);
}
// Force re-analysis of traders who now have new linked trades
var traderIds = pairsToReset.Select(p => p.TraderId).Distinct().ToList();
await db.Traders
.Where(t => traderIds.Contains(t.Id))
.ExecuteUpdateAsync(s => s.SetProperty(p => p.LastAnalyzedAt, (DateTime?)null), ct);
} }
} }
catch (Exception ex) catch (Exception ex)
@@ -71,13 +71,47 @@ public class TradeRetentionWorker : BackgroundService
var retentionCutoff = utcNow.Date.AddDays(-retentionDays); var retentionCutoff = utcNow.Date.AddDays(-retentionDays);
var compactionCutoff = utcNow.Date.AddDays(-compactionDays); var compactionCutoff = utcNow.Date.AddDays(-compactionDays);
// 1. Prune Old Trades (C1 & C2)
// Exclude trades if the trader is on any active Watchlist // Exclude trades if the trader is on any active Watchlist
_logger.LogInformation("Pruning trades older than {Cutoff}...", retentionCutoff); _logger.LogInformation("Pruning trades older than {Cutoff}...", retentionCutoff);
var deletedTrades = await db.Trades
.Where(t => t.ExecutedAt < retentionCutoff && !t.Trader.WatchlistEntries.Any())
.Select(t => new { t.TraderId, t.MarketOutcomeId })
.Distinct()
.ToListAsync(ct);
var deletedCount = await db.Trades var deletedCount = await db.Trades
.Where(t => t.ExecutedAt < retentionCutoff && !t.Trader.WatchlistEntries.Any()) .Where(t => t.ExecutedAt < retentionCutoff && !t.Trader.WatchlistEntries.Any())
.ExecuteDeleteAsync(ct); .ExecuteDeleteAsync(ct);
if (deletedCount > 0 && deletedTrades.Any())
{
var validDeletedTrades = deletedTrades.Where(d => d.MarketOutcomeId.HasValue).ToList();
if (validDeletedTrades.Any())
{
var posIdsToUpdate = await db.TraderPositions
.Where(tp => validDeletedTrades.Select(d => d.TraderId).Contains(tp.TraderId) &&
validDeletedTrades.Select(d => d.MarketOutcomeId!.Value).Contains(tp.MarketOutcomeId))
.Select(tp => tp.Id)
.ToListAsync(ct);
// Filter on client side due to EF Core limitation with tuple Contains
var actualPosIdsToUpdate = (await db.TraderPositions
.Where(tp => posIdsToUpdate.Contains(tp.Id))
.ToListAsync(ct))
.Where(tp => validDeletedTrades.Any(d => d.TraderId == tp.TraderId && d.MarketOutcomeId == tp.MarketOutcomeId))
.Select(tp => tp.Id)
.ToList();
if (actualPosIdsToUpdate.Any())
{
await db.TraderPositions
.Where(tp => actualPosIdsToUpdate.Contains(tp.Id))
.ExecuteUpdateAsync(s => s.SetProperty(p => p.IsHistoryPruned, true), ct);
}
}
}
_logger.LogInformation("Pruned {Count} old trades from the database.", deletedCount); _logger.LogInformation("Pruned {Count} old trades from the database.", deletedCount);
// 2. Compact Bot Trades (C3) // 2. Compact Bot Trades (C3)
@@ -100,14 +134,25 @@ public class TradeRetentionWorker : BackgroundService
{ {
if (ct.IsCancellationRequested) break; if (ct.IsCancellationRequested) break;
// Fetch positions to ensure we only compact applied trades and can bump checkpoints
var positions = await db.TraderPositions
.Where(tp => tp.TraderId == traderId)
.ToDictionaryAsync(tp => tp.MarketOutcomeId, ct);
// Load candidate trades to compact (older than compactionCutoff, newer than retentionCutoff) // Load candidate trades to compact (older than compactionCutoff, newer than retentionCutoff)
var tradesToCompact = await db.Trades var tradesToCompact = await db.Trades
.Where(t => t.TraderId == traderId && .Where(t => t.TraderId == traderId &&
t.ExecutedAt >= retentionCutoff && t.ExecutedAt >= retentionCutoff &&
t.ExecutedAt < compactionCutoff && t.ExecutedAt < compactionCutoff &&
!t.PlatformTradeId.StartsWith("COMPACT_")) !t.PlatformTradeId.StartsWith("COMPACT_") &&
t.MarketOutcomeId != null)
.ToListAsync(ct); .ToListAsync(ct);
// Filter strictly to trades that are already applied
tradesToCompact = tradesToCompact
.Where(t => positions.TryGetValue(t.MarketOutcomeId!.Value, out var pos) && t.Id <= pos.LastAppliedTradeId)
.ToList();
if (tradesToCompact.Count == 0) continue; if (tradesToCompact.Count == 0) continue;
// Group trades by outcome, date, and side to aggregate // Group trades by outcome, date, and side to aggregate
@@ -160,6 +205,19 @@ public class TradeRetentionWorker : BackgroundService
// Add the compacted trade // Add the compacted trade
db.Trades.Add(compactedTrade); db.Trades.Add(compactedTrade);
// Save immediately so compactedTrade gets an ID
await db.SaveChangesAsync(ct);
// Bump the position checkpoint so it doesn't get double counted
if (positions.TryGetValue(outcomeId, out var pos))
{
pos.LastAppliedTradeId = Math.Max(pos.LastAppliedTradeId, compactedTrade.Id);
// Mark as pruned so we don't accidentally reset and replay (which would lose the exact intraday timestamps)
pos.IsHistoryPruned = true;
db.TraderPositions.Update(pos);
}
compactedTradeCount += list.Count - 1; compactedTradeCount += list.Count - 1;
} }
@@ -63,9 +63,11 @@ public class TraderAnalyticsWorker : BackgroundService
{ {
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>(); var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
// Find traders who have never been analyzed, or whose last analysis was before their latest trade. // Find traders who have never been analyzed, or whose last analysis was before their latest trade.
// Prioritize never-analyzed traders. // Bug 5 Fix: Add 30-minute cooldown to prevent CPU looping.
var cooldown = DateTime.UtcNow.AddMinutes(-30);
traderIds = await db.Traders traderIds = await db.Traders
.Where(t => t.LastAnalyzedAt == null || t.Trades.Any(tr => tr.ExecutedAt > t.LastAnalyzedAt)) .Where(t => t.LastAnalyzedAt == null ||
(t.LastAnalyzedAt < cooldown && t.Trades.Any(tr => tr.ExecutedAt > t.LastAnalyzedAt)))
.OrderBy(t => t.LastAnalyzedAt == null ? 0 : 1) .OrderBy(t => t.LastAnalyzedAt == null ? 0 : 1)
.ThenBy(t => t.LastAnalyzedAt) .ThenBy(t => t.LastAnalyzedAt)
.Select(t => t.Id) .Select(t => t.Id)
@@ -89,24 +91,31 @@ public class TraderAnalyticsWorker : BackgroundService
// Run CopytradingEstimator // Run CopytradingEstimator
var traderRepo = traderScope.ServiceProvider.GetRequiredService<ITraderRepository>(); var traderRepo = traderScope.ServiceProvider.GetRequiredService<ITraderRepository>();
var tradeRepo = traderScope.ServiceProvider.GetRequiredService<ITradeRepository>(); var db = traderScope.ServiceProvider.GetRequiredService<AppDbContext>();
var estimator = traderScope.ServiceProvider.GetRequiredService<ICopytradingEstimator>(); var estimator = traderScope.ServiceProvider.GetRequiredService<ICopytradingEstimator>();
var trader = await traderRepo.GetByIdAsync(id, ct); var trader = await traderRepo.GetByIdAsync(id, ct);
if (trader != null) if (trader != null)
{ {
var trades = await tradeRepo.GetByTraderIdAsync(id, 0, 1000, ct); var trades = await db.Trades
.Include(t => t.MarketOutcome)
.Include(t => t.Context)
.Where(t => t.TraderId == id && t.DbMarketId != null)
.OrderByDescending(t => t.ExecutedAt)
.Take(1000)
.ToListAsync(ct);
if (trades.Count > 0) if (trades.Count > 0)
{ {
var estScores = await estimator.CalculateScoresAsync(trader, trades, ct); var estScores = await estimator.CalculateScoresAsync(trader, trades, ct);
var scoreObj = trader.CurrentScore ?? new Predictalytics.Domain.Entities.TraderScore { TraderId = trader.Id }; var analyticsObj = trader.Analytics ?? new Predictalytics.Domain.Entities.TraderAnalytics { TraderId = trader.Id };
// Persist advanced copyability and quality scores derived from tape replay // Persist advanced copyability and quality scores derived from tape replay
scoreObj.CopytradingScore = estScores.CopyabilityScore; analyticsObj.CopytradingScore = estScores.CopyabilityScore;
scoreObj.QualityScore = estScores.QualityScore; analyticsObj.CopytradingQualityScore = estScores.QualityScore;
scoreObj.CalculatedAt = DateTime.UtcNow; analyticsObj.CopytradingCopyabilityScore = estScores.CopyabilityScore;
trader.CurrentScore = scoreObj; trader.Analytics = analyticsObj;
} }
trader.LastAnalyzedAt = DateTime.UtcNow; trader.LastAnalyzedAt = DateTime.UtcNow;
await traderRepo.UpdateAsync(trader, ct); await traderRepo.UpdateAsync(trader, ct);