diff --git a/src/Predictalytics.Worker/Services/TraderAnalyticsWorker.cs b/src/Predictalytics.Worker/Services/TraderAnalyticsWorker.cs index ef7ace4..aef90b4 100644 --- a/src/Predictalytics.Worker/Services/TraderAnalyticsWorker.cs +++ b/src/Predictalytics.Worker/Services/TraderAnalyticsWorker.cs @@ -86,9 +86,33 @@ public class TraderAnalyticsWorker : BackgroundService _logger.LogInformation("Found {Count} active or unanalyzed traders to update", traderIds.Count); } + async Task MarkJobFailedAsync(int traderId, Exception ex) + { + if (activeJob != null && activeJob.TraderId == traderId) + { + try + { + using var jobScope = _services.CreateScope(); + var updateJobRepo = jobScope.ServiceProvider.GetRequiredService(); + activeJob.Status = Predictalytics.Domain.Enums.JobStatus.Failed; + activeJob.CompletedAt = DateTime.UtcNow; + activeJob.ErrorMessage = ex.Message; + await updateJobRepo.UpdateAsync(activeJob, CancellationToken.None); + } + catch { /* Ignore secondary errors */ } + } + } + foreach (var id in traderIds) { if (ct.IsCancellationRequested) break; + + // Concurrent writers (reconciliation checkpoint resets, pollers) can touch the + // same TraderPositions/Traders rows mid-save ("Record has changed since last + // read"). A retry with a fresh context re-reads their changes and succeeds. + const int maxAttempts = 2; + for (int attempt = 1; attempt <= maxAttempts; attempt++) + { try { using var traderScope = _services.CreateScope(); @@ -149,27 +173,38 @@ public class TraderAnalyticsWorker : BackgroundService activeJob.CompletedAt = DateTime.UtcNow; await updateJobRepo.UpdateAsync(activeJob, ct); } + + break; // trader done } catch (OperationCanceledException) when (ct.IsCancellationRequested) { _logger.LogInformation("Cancellation requested during analysis, stopping batch."); break; } + catch (DbUpdateException ex) when (attempt < maxAttempts) + { + _logger.LogWarning("Concurrent modification while analyzing trader {TraderId} — retrying with a fresh context. ({Message})", + id, ex.InnerException?.Message ?? ex.Message); + await Task.Delay(250, CancellationToken.None); + } + catch (DbUpdateException ex) + { + // Still colliding after the retry. LastAnalyzedAt stays unset, so this + // trader is picked up again next cycle — a warning is enough. + _logger.LogWarning("Trader {TraderId} skipped after repeated concurrent modifications — will retry next cycle. ({Message})", + id, ex.InnerException?.Message ?? ex.Message); + await MarkJobFailedAsync(id, ex); + break; + } catch (Exception ex) { _logger.LogError(ex, "Error recalculating positions/PnL for trader {TraderId}", id); - if (activeJob != null && activeJob.TraderId == id) - { - try { - using var jobScope = _services.CreateScope(); - var updateJobRepo = jobScope.ServiceProvider.GetRequiredService(); - activeJob.Status = Predictalytics.Domain.Enums.JobStatus.Failed; - activeJob.CompletedAt = DateTime.UtcNow; - activeJob.ErrorMessage = ex.Message; - await updateJobRepo.UpdateAsync(activeJob, CancellationToken.None); - } catch { /* Ignore secondary errors */ } - } + await MarkJobFailedAsync(id, ex); + break; } + } + + if (ct.IsCancellationRequested) break; } if (traderIds.Count > 0)