Predictalytics/src/Predictalytics.Worker/Services/PollingWorker.cs

using Predictalytics.Application.Interfaces;
using Predictalytics.Domain.Enums;
using Predictalytics.Domain.Interfaces;
using Predictalytics.Infrastructure.Logging;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;

namespace Predictalytics.Worker.Services;

/// <summary>
/// Background service that periodically polls tracked traders for new trades.
/// Properly deduplicates trades using the Platform+PlatformTradeId unique index.
/// Resolves trades to MarketOutcomes via AssetId → TokenId mapping.
/// </summary>
public class PollingWorker : BackgroundService
{
    private static DateTime _lastDbError = DateTime.MinValue;
    private readonly IServiceProvider _services;
    private readonly IPlatformStatisticsService _statsService;
    private readonly ILogger<PollingWorker> _logger;

    public PollingWorker(IServiceProvider services, ILogger<PollingWorker> logger, IPlatformStatisticsService statsService)
    { _services = services; _logger = logger; _statsService = statsService; }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        _logger.LogWarning("📡 PollingWorker started");
        await Task.Delay(3000, stoppingToken);

        while (!stoppingToken.IsCancellationRequested)
        {
            try
            {
                IReadOnlyList<Domain.Entities.Trader> tradersToProcess;
                using (var scope = _services.CreateScope())
                {
                    var repo = scope.ServiceProvider.GetRequiredService<ITraderRepository>();
                    tradersToProcess = await repo.GetTradersForPollingAsync(take: 100, ct: stoppingToken);
                }
                
                _logger.LogWarning("📊 Polling {Count} traders...", tradersToProcess.Count);

                foreach (var t in tradersToProcess)
                {
                    if (stoppingToken.IsCancellationRequested) break;

                    using var scope = _services.CreateScope();
                    var traderRepo = scope.ServiceProvider.GetRequiredService<ITraderRepository>();
                    var tradeRepo = scope.ServiceProvider.GetRequiredService<ITradeRepository>();
                    var marketRepo = scope.ServiceProvider.GetRequiredService<IMarketRepository>();
                    var providers = scope.ServiceProvider.GetRequiredService<IEnumerable<IPlatformProvider>>();
                    var rateLimiter = scope.ServiceProvider.GetRequiredService<IRateLimiter>();
                    
                    var trader = await traderRepo.GetByIdAsync(t.Id, stoppingToken);
                    if (trader == null || trader.IngestMode == IngestMode.SnapshotOnly) continue;

                    var provider = providers.FirstOrDefault(p => p.Platform == trader.Platform && p.IsImplemented);
                    if (provider == null) continue;

                    try
                    {
                        using var platformCtx = PlatformLogContext.Push(provider.PlatformName);
                        await rateLimiter.WaitAsync(trader.Platform, stoppingToken);
                        var trades = await provider.GetTraderTradesAsync(trader.PlatformUserId, 100, stoppingToken);

                        var validTrades = trades
                            .Where(t => !string.IsNullOrWhiteSpace(t.PlatformTradeId))
                            .ToList();
                        
                        var updatedName = validTrades.FirstOrDefault(t => !string.IsNullOrEmpty(t.TransientDisplayName))?.TransientDisplayName;
                        if (!string.IsNullOrEmpty(updatedName) && !string.Equals(trader.DisplayName, updatedName, StringComparison.OrdinalIgnoreCase))
                        {
                            trader.DisplayName = updatedName;
                            await traderRepo.UpdateAsync(trader, stoppingToken);
                        }

                        if (validTrades.Count < trades.Count)
                        {
                            _logger.LogWarning("{Trader}: Skipped {Count} trades with empty PlatformTradeId",
                                trader.DisplayName, trades.Count - validTrades.Count);
                        }

                        // ── Deduplicate: check each trade against DB ──
                        var newTrades = new List<Domain.Entities.Trade>();
                        var fetchedTradeIds = validTrades.Select(tr => tr.PlatformTradeId).ToList();
                        var knownTradeIds = await tradeRepo.GetKnownPlatformTradeIdsAsync(trader.Platform, trader.Id, fetchedTradeIds, stoppingToken);

                        foreach (var trade in validTrades)
                        {
                            if (!knownTradeIds.Contains(trade.PlatformTradeId))
                            {
                                trade.TraderId = trader.Id;

                                // Resolve MarketOutcome via AssetId → TokenId
                                if (!string.IsNullOrEmpty(trade.AssetId))
                                {
                                    var outcome = await marketRepo.GetOutcomeByTokenIdAsync(trade.AssetId, stoppingToken);

                                    // If not found locally, fetch the market from provider
                                    if (outcome == null && !string.IsNullOrEmpty(trade.MarketId))
                                    {
                                        var newMarket = await provider.GetMarketAsync(trade.MarketId, stoppingToken);
                                        if (newMarket != null)
                                        {
                                            await marketRepo.AddOrUpdateAsync(newMarket, stoppingToken);
                                            outcome = await marketRepo.GetOutcomeByTokenIdAsync(trade.AssetId, stoppingToken);
                                        }
                                    }

                                    if (outcome != null)
                                    {
                                        trade.MarketOutcomeId = outcome.Id;
                                        trade.Outcome = outcome.Label;
                                        // Attempt to resolve DbMarketId via outcome's parent market
                                        if (outcome.Market != null)
                                            trade.DbMarketId = outcome.Market.Id;
                                    }
                                }

                                newTrades.Add(trade);
                            }
                        }

                        if (trader.IngestMode == IngestMode.Aggregated && newTrades.Count > 0)
                        {
                            // Only completed hours are aggregated+persisted; the current (growing)
                            // hour is deferred so aggregate rows stay immutable for the checkpoint
                            // engine. See TradeAggregation.AggregateCompletedHours.
                            newTrades = Predictalytics.Application.Services.TradeAggregation
                                .AggregateCompletedHours(newTrades, trader.Id, DateTime.UtcNow);
                        }

                        // ── Persist new trades ──
                        if (newTrades.Count > 0)
                        {
                            // Final deduplication of the batch itself
                            var uniqueNewTrades = newTrades
                                .GroupBy(tr => tr.PlatformTradeId, StringComparer.OrdinalIgnoreCase)
                                .Select(g => g.First())
                                .ToList();

                            try
                            {
                                await tradeRepo.AddRangeAsync(uniqueNewTrades, stoppingToken);
                                _statsService.TrackTradeActivity(trader.Platform, uniqueNewTrades.Count);
                                trader.TotalTrades += uniqueNewTrades.Count;
                                _logger.LogInformation("{Trader}: {New} new trades (of {Total} fetched)",
                                    trader.DisplayName, uniqueNewTrades.Count, validTrades.Count);
                            }
                            catch (Exception ex) when (ex.ToString().Contains("Duplicate entry") || (ex.InnerException?.Message.Contains("Duplicate entry") ?? false))
                            {
                                _statsService.TrackDuplicateError(trader.Platform, 1);
                            }
                        }
                        else
                        {
                            _logger.LogDebug("{Trader}: No new trades (all {Count} already known)",
                                trader.DisplayName, validTrades.Count);
                        }

                        trader.LastPolledAt = DateTime.UtcNow;
                        await traderRepo.UpdateAsync(trader, stoppingToken);
                    }
                    catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
                    {
                        _logger.LogInformation("Cancellation requested during polling, stopping batch.");
                        break;
                    }
                    catch (Exception ex)
                    {
                        _logger.LogError(ex, "Error polling trader {Trader} on {Platform}",
                            trader.DisplayName, trader.Platform);
                    }
                }



                _logger.LogWarning("✅ Polling cycle complete. Next in 60s.");
            }
            catch (OperationCanceledException) { break; }
            catch (Exception ex) when (ex.ToString().Contains("MySqlException") || ex.ToString().Contains("Connection"))
            {
                if (DateTime.UtcNow - _lastDbError > TimeSpan.FromMinutes(10))
                {
                    _logger.LogWarning("⚠️ Database connection lost in PollingWorker. Retrying in 60s. (Error: {Message})", ex.Message);
                    _lastDbError = DateTime.UtcNow;
                }
            }
            catch (Exception ex) { _logger.LogError(ex, "PollingWorker error"); }

            await Task.Delay(TimeSpan.FromSeconds(60), stoppingToken);
        }

        _logger.LogInformation("📡 PollingWorker stopped");
    }
}