Enhance UI, add AI integration, improve logging and database stats

src/Predictalytics.Worker/DependencyInjection.cs

@@ -19,6 +19,7 @@ public static class DependencyInjection
         services.AddHostedService<TraderAnalyticsWorker>();
         services.AddHostedService<ScoringAndAlertsWorker>();
         services.AddHostedService<TradeRetentionWorker>();
+        services.AddHostedService<TradeContextEnrichmentWorker>();
         return services;
     }
 }

src/Predictalytics.Worker/Predictalytics.Worker.csproj

@@ -7,6 +7,10 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.11">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.1" />
   </ItemGroup>
 

src/Predictalytics.Worker/Services/MarketHistoryWorker.cs

@@ -18,7 +18,7 @@ public class MarketHistoryWorker : BackgroundService
     private readonly ILogger<MarketHistoryWorker> _logger;
 
     private const int CooldownHours = 6;
-    private const int MarketsPerCycle = 10;
+    private const int MarketsPerCycle = 50;
     private const int TradesPerFetch = 100;
 
     public MarketHistoryWorker(IServiceProvider services, ILogger<MarketHistoryWorker> logger)
@@ -64,7 +64,7 @@ public class MarketHistoryWorker : BackgroundService
                         using var platformCtx = PlatformLogContext.Push(provider.PlatformName);
                         await rateLimiter.WaitAsync(market.Platform, stoppingToken);
 
-                        var trades = await provider.GetMarketTradesAsync(market.PlatformMarketId, TradesPerFetch, stoppingToken);
+                        var trades = await provider.GetMarketTradesAsync(market.ConditionId, TradesPerFetch, stoppingToken);
 
                         // Extract unique wallet addresses from TransientWallet [NotMapped].
                         // Providers set this during in-memory mapping; it is NOT stored in the DB.

src/Predictalytics.Worker/Services/MarketSyncWorker.cs

@@ -66,15 +66,15 @@ public class MarketSyncWorker : BackgroundService
                                     .First(x => x.Platform == p.Platform);
 
                                 await rateLimiter.WaitAsync(provider.Platform, stoppingToken);
-                                var markets = await provider.GetMarketsAsync(batchSize, offset.ToString(), includeClosed, stoppingToken);
+                                var events = await provider.GetEventsAsync(batchSize, offset.ToString(), includeClosed, stoppingToken);
 
-                                if (markets.Count == 0) break;
+                                if (events.Count == 0) break;
 
-                                await marketRepo.AddOrUpdateRangeAsync(markets, stoppingToken);
+                                await marketRepo.AddOrUpdateEventsAsync(events, stoppingToken);
 
-                                passSynced += markets.Count;
-                                cycleTotalSynced += markets.Count;
-                                _statsService.TrackMarketSync(provider.Platform, markets.Count);
+                                passSynced += events.Count;
+                                cycleTotalSynced += events.Count;
+                                _statsService.TrackMarketSync(provider.Platform, events.Count);
                                 offset += batchSize;
 
                                 if (passSynced % 500 == 0)

src/Predictalytics.Worker/Services/PollingWorker.cs

@@ -36,7 +36,7 @@ public class PollingWorker : BackgroundService
                 using (var scope = _services.CreateScope())
                 {
                     var repo = scope.ServiceProvider.GetRequiredService<ITraderRepository>();
-                    tradersToProcess = await repo.GetAllAsync(take: 100, ct: stoppingToken);
+                    tradersToProcess = await repo.GetTradersForPollingAsync(take: 100, ct: stoppingToken);
                 }
                 
                 _logger.LogWarning("📊 Polling {Count} traders...", tradersToProcess.Count);
@@ -121,7 +121,7 @@ public class PollingWorker : BackgroundService
                         {
                             // Final deduplication of the batch itself
                             var uniqueNewTrades = newTrades
-                                .GroupBy(tr => tr.PlatformTradeId)
+                                .GroupBy(tr => tr.PlatformTradeId, StringComparer.OrdinalIgnoreCase)
                                 .Select(g => g.First())
                                 .ToList();
 
@@ -135,7 +135,7 @@ public class PollingWorker : BackgroundService
                             }
                             catch (Exception ex) when (ex.ToString().Contains("Duplicate entry") || (ex.InnerException?.Message.Contains("Duplicate entry") ?? false))
                             {
-                                _logger.LogWarning("{Trader}: Skipping batch due to duplicate entries (likely already imported)", trader.DisplayName);
+                                _statsService.TrackDuplicateError(trader.Platform, 1);
                             }
                         }
                         else

src/Predictalytics.Worker/Services/ReportingWorker.cs

@@ -52,8 +52,9 @@ public class ReportingWorker : BackgroundService
 
         foreach (var (platform, stats) in statsMap)
         {
-            _logger.LogWarning("[{Platform}] Markets: {M} | New Traders: {T} | Activities: {A}",
-                platform, stats.MarketsSynced, stats.TradersDiscovered, stats.TradesProcessed);
+            var dupStr = stats.DuplicateErrors > 0 ? $" | Duplikatfehler: {stats.DuplicateErrors}" : "";
+            _logger.LogWarning("[{Platform}] Markets: {M} | New Traders: {T} | Activities: {A}{D}",
+                platform, stats.MarketsSynced, stats.TradersDiscovered, stats.TradesProcessed, dupStr);
         }
 
         _logger.LogWarning("------------------------------------------------");

src/Predictalytics.Worker/Services/TopHolderDiscoveryWorker.cs

@@ -37,7 +37,7 @@ public class TopHolderDiscoveryWorker : BackgroundService
                 var rateLimiter = scope.ServiceProvider.GetRequiredService<IRateLimiter>();
 
                 // Get top active markets by volume
-                var activeMarkets = await marketRepo.GetActiveAsync(20, stoppingToken);
+                var activeMarkets = await marketRepo.GetActiveAsync(100, stoppingToken);
                 _logger.LogInformation("👥 Scanning top holders across {Count} active markets", activeMarkets.Count);
 
                 int totalDiscovered = 0;
@@ -54,9 +54,9 @@ public class TopHolderDiscoveryWorker : BackgroundService
                         using var platformCtx = PlatformLogContext.Push(provider.PlatformName);
                         await rateLimiter.WaitAsync(market.Platform, stoppingToken);
 
-                        var holders = await provider.GetTopHoldersAsync(market.PlatformMarketId, 10, stoppingToken);
+                        var newHolders = await provider.GetTopHoldersAsync(market.ConditionId, 50, stoppingToken);
 
-                        foreach (var holder in holders)
+                        foreach (var holder in newHolders)
                         {
                             // Skip empty wallet addresses
                             if (string.IsNullOrWhiteSpace(holder.PlatformUserId)) continue;
@@ -95,9 +95,9 @@ public class TopHolderDiscoveryWorker : BackgroundService
             catch (OperationCanceledException) { break; }
             catch (Exception ex) { _logger.LogError(ex, "TopHolderDiscoveryWorker error"); }
 
-            // Run every 15 minutes
-            _logger.LogInformation("👥 Next top holder scan in 15 minutes.");
-            await Task.Delay(TimeSpan.FromMinutes(15), stoppingToken);
+            // Run every 5 minutes
+            _logger.LogInformation("🚀 Next top holder scan in 5 minutes.");
+            await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
         }
 
         _logger.LogInformation("👥 TopHolderDiscoveryWorker stopped");

src/Predictalytics.Worker/Services/TradeContextEnrichmentWorker.cs

@@ -0,0 +1,124 @@
+using Predictalytics.Domain.Entities;
+using Predictalytics.Domain.Interfaces;
+using Predictalytics.Infrastructure.Providers.Polymarket;
+using Predictalytics.Application.Interfaces;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
+using Microsoft.Extensions.Logging;
+
+namespace Predictalytics.Worker.Services;
+
+/// <summary>
+/// Retroactively enriches trades of top/watchlisted traders with high-resolution 
+/// 1-minute price contexts immediately before and after execution.
+/// Avoids burdening the live PollingWorker.
+/// </summary>
+public class TradeContextEnrichmentWorker : BackgroundService
+{
+    private readonly IServiceProvider _services;
+    private readonly ILogger<TradeContextEnrichmentWorker> _logger;
+
+    public TradeContextEnrichmentWorker(IServiceProvider services, ILogger<TradeContextEnrichmentWorker> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        _logger.LogInformation("🧠 TradeContextEnrichmentWorker started");
+        await Task.Delay(10000, stoppingToken); // Wait for app startup
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                using var scope = _services.CreateScope();
+                var tradeRepo = scope.ServiceProvider.GetRequiredService<ITradeRepository>();
+                var polymarketClient = scope.ServiceProvider.GetRequiredService<PolymarketApiClient>();
+                var rateLimiter = scope.ServiceProvider.GetRequiredService<IRateLimiter>();
+
+                // Fetch a batch of unenriched trades
+                var unenrichedTrades = await tradeRepo.GetTradesForContextEnrichmentAsync(500, stoppingToken);
+                
+                if (unenrichedTrades.Count == 0)
+                {
+                    // No work to do, sleep longer
+                    await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken);
+                    continue;
+                }
+
+                // Group by AssetId to minimize API calls (1 call per asset fetches the whole 1m history)
+                var tradesByAsset = unenrichedTrades.GroupBy(t => t.AssetId).ToList();
+                _logger.LogInformation("Enriching {TradeCount} trades across {AssetCount} assets...", unenrichedTrades.Count, tradesByAsset.Count);
+
+                int updatedCount = 0;
+
+                foreach (var group in tradesByAsset)
+                {
+                    if (stoppingToken.IsCancellationRequested) break;
+                    var assetId = group.Key;
+
+                    try
+                    {
+                        // Wait for rate limiter to respect global limits
+                        await rateLimiter.WaitAsync(Predictalytics.Domain.Enums.PlatformType.Polymarket, stoppingToken);
+
+                        // Fetch the 10-minute history for the entire market (using "max" since "1m" is invalid for full history)
+                        var history = await polymarketClient.GetPricesHistoryAsync(assetId, "max", stoppingToken);
+                        if (history == null || history.Count == 0)
+                        {
+                            // If history is not available, mark as enriched to prevent infinite loops, 
+                            // but prices remain null.
+                            foreach (var trade in group)
+                            {
+                                trade.IsContextEnriched = true;
+                                await tradeRepo.UpdateAsync(trade, stoppingToken);
+                            }
+                            continue;
+                        }
+
+                        // Order history chronologically for safe binary search / LINQ
+                        var orderedHistory = history.OrderBy(h => h.Timestamp).ToList();
+
+                        foreach (var trade in group)
+                        {
+                            var tradeTimeUnix = ((DateTimeOffset)trade.ExecutedAt).ToUnixTimeSeconds();
+
+                            // Find the closest point BEFORE the trade (approx 1 min before)
+                            var prePoint = orderedHistory
+                                .LastOrDefault(h => h.Timestamp < tradeTimeUnix);
+                            
+                            // Find the closest point AFTER the trade (approx 1 min after)
+                            var postPoint = orderedHistory
+                                .FirstOrDefault(h => h.Timestamp > tradeTimeUnix);
+
+                            trade.PreTradePrice1m = prePoint != null ? (decimal)prePoint.Price : null;
+                            trade.PostTradePrice1m = postPoint != null ? (decimal)postPoint.Price : null;
+                            trade.IsContextEnriched = true;
+
+                            await tradeRepo.UpdateAsync(trade, stoppingToken);
+                            updatedCount++;
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.LogError(ex, "Failed to enrich asset {AssetId}", assetId);
+                        // Do NOT mark as enriched on failure, try again later
+                    }
+                }
+
+                _logger.LogInformation("✅ Enriched {UpdatedCount} trades in this cycle.", updatedCount);
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Error in TradeContextEnrichmentWorker loop");
+            }
+
+            // Sleep briefly before next batch
+            await Task.Delay(TimeSpan.FromSeconds(30), stoppingToken);
+        }
+
+        _logger.LogInformation("🧠 TradeContextEnrichmentWorker stopped");
+    }
+}

src/Predictalytics.Worker/Services/TradeHistoryWorker.cs

@@ -151,14 +151,14 @@ public class TradeHistoryWorker : BackgroundService
 
                         if (newTrades.Count > 0)
                         {
-                            var uniqueNewTrades = newTrades.GroupBy(tr => tr.PlatformTradeId).Select(g => g.First()).ToList();
+                            var uniqueNewTrades = newTrades.GroupBy(tr => tr.PlatformTradeId, StringComparer.OrdinalIgnoreCase).Select(g => g.First()).ToList();
                             try {
                                 await tradeRepo.AddRangeAsync(uniqueNewTrades, ct);
                                 _statsService.TrackTradeActivity(trader.Platform, uniqueNewTrades.Count);
                                 trader.TotalTrades += uniqueNewTrades.Count;
                                 _logger.LogInformation("{Trader}: {New} new trades imported", trader.DisplayName, uniqueNewTrades.Count);
-                            } catch (Exception ex) when (ex.ToString().Contains("Duplicate entry")) {
-                                _logger.LogWarning("{Trader}: Skipping batch due to duplicates", trader.DisplayName);
+                            } catch (Exception ex) when (ex.ToString().Contains("Duplicate entry") || (ex.InnerException?.Message.Contains("Duplicate entry") ?? false)) {
+                                _statsService.TrackDuplicateError(trader.Platform, 1);
                             }
                         }
 

src/Predictalytics.Worker/Services/TraderAnalyticsWorker.cs

@@ -33,25 +33,26 @@ public class TraderAnalyticsWorker : BackgroundService
                 _logger.LogError(ex, "Error in TraderAnalyticsWorker");
             }
 
-            _logger.LogInformation("TraderAnalyticsWorker sleeping for 12 hours...");
-            await Task.Delay(TimeSpan.FromHours(12), ct);
+            _logger.LogInformation("TraderAnalyticsWorker sleeping for 2 hours...");
+            await Task.Delay(TimeSpan.FromHours(2), ct);
         }
     }
 
     private async Task RunAnalyticsAsync(CancellationToken ct)
     {
-        using var scope = _services.CreateScope();
-        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
-        var pnlEngine = scope.ServiceProvider.GetRequiredService<IPositionPnLEngine>();
-
         var cutoff30d = DateTime.UtcNow.AddDays(-30);
+        List<int> traderIds;
         
-        // Find traders active in the last 30 days
-        var traderIds = await db.Trades
-            .Where(t => t.ExecutedAt >= cutoff30d)
-            .Select(t => t.TraderId)
-            .Distinct()
-            .ToListAsync(ct);
+        using (var scope = _services.CreateScope())
+        {
+            var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+            // Find traders active in the last 30 days
+            traderIds = await db.Trades
+                .Where(t => t.ExecutedAt >= cutoff30d)
+                .Select(t => t.TraderId)
+                .Distinct()
+                .ToListAsync(ct);
+        }
 
         _logger.LogInformation("Found {Count} active traders to analyze", traderIds.Count);
 
@@ -59,6 +60,8 @@ public class TraderAnalyticsWorker : BackgroundService
         {
             try
             {
+                using var traderScope = _services.CreateScope();
+                var pnlEngine = traderScope.ServiceProvider.GetRequiredService<IPositionPnLEngine>();
                 await pnlEngine.RecalculateTraderPositionsAsync(id, ct);
             }
             catch (Exception ex)