diff --git a/Jiten.Api/Controllers/AdminController.cs b/Jiten.Api/Controllers/AdminController.cs index e5c31711..91a66fb7 100644 --- a/Jiten.Api/Controllers/AdminController.cs +++ b/Jiten.Api/Controllers/AdminController.cs @@ -906,6 +906,37 @@ public async Task ReaggregateParentDifficulty(int deckId) return Ok(new { Message = "Queued difficulty reaggregation", DeckId = deckId }); } + /// + /// Backstop sweep: find leaf decks that have raw text but no computed difficulty (e.g. children left at 0 by a + /// transient RunPod failure) and re-enqueue them, then schedule re-aggregation for any affected parents. + /// + [HttpPost("recompute-missing-difficulties")] + public async Task RecomputeMissingDifficulties() + { + var missing = await dbContext.Decks + .Where(d => d.RawText != null && d.DeckDifficulty == null && !d.Children.Any()) + .Select(d => new { d.DeckId, d.ParentDeckId }) + .ToListAsync(); + + foreach (var m in missing) + backgroundJobs.Enqueue( + job => job.ComputeDeckDifficulty(m.DeckId, false)); + + var parentIds = missing + .Where(m => m.ParentDeckId != null) + .Select(m => m.ParentDeckId!.Value) + .Distinct() + .ToList(); + + foreach (var parentId in parentIds) + backgroundJobs.Schedule( + job => job.ReaggregateParentDifficulty(parentId), TimeSpan.FromMinutes(45)); + + logger.LogInformation("Admin queued {Count} missing-difficulty recomputations across {ParentCount} parents", + missing.Count, parentIds.Count); + return Ok(new { Message = $"Queued {missing.Count} missing-difficulty recomputations", Count = missing.Count, AffectedParents = parentIds.Count }); + } + [HttpGet("issues")] public async Task GetIssues() { diff --git a/Jiten.Api/Jobs/DifficultyComputationJob.cs b/Jiten.Api/Jobs/DifficultyComputationJob.cs index a5ee0131..71074208 100644 --- a/Jiten.Api/Jobs/DifficultyComputationJob.cs +++ b/Jiten.Api/Jobs/DifficultyComputationJob.cs @@ -10,6 +10,7 @@ public class DifficultyComputationJob( IDbContextFactory contextFactory, IHttpClientFactory httpClientFactory, IConfiguration configuration, + IBackgroundJobClient backgroundJobs, ILogger logger) { private record DifficultyResponse( @@ -128,6 +129,7 @@ private async Task ComputeParentWithChildren(JitenDbContext context, Deck parent parent.DeckId, children.Count, forceRecompute); var computedCount = 0; + var failedChildren = new List(); foreach (var child in children) { if (!forceRecompute && child.DeckDifficulty != null) @@ -151,7 +153,10 @@ private async Task ComputeParentWithChildren(JitenDbContext context, Deck parent } catch (Exception ex) { - logger.LogError(ex, "Failed to compute difficulty for child deck {DeckId}, skipping", child.DeckId); + // Usually a transient RunPod worker failure (e.g. CUDA "no kernel image"). Re-enqueue the child + // below as its own job so it inherits the per-deck delayed-retry policy and lands on a fresh worker. + logger.LogError(ex, "Failed to compute difficulty for child deck {DeckId}, will re-enqueue", child.DeckId); + failedChildren.Add(child.DeckId); } } @@ -162,8 +167,22 @@ private async Task ComputeParentWithChildren(JitenDbContext context, Deck parent .OrderBy(d => d.DeckOrder) .ToListAsync(); - // Aggregate parent difficulty from children + // Aggregate parent difficulty from whatever succeeded so the parent is never blocked by a few failures await AggregateParentDifficulty(context, parent, childrenWithDifficulty); + + // Self-heal stragglers: re-enqueue each failed child (leaf → 3× delayed retry) and schedule a single + // delayed re-aggregation so the parent picks up the recovered children once their retries complete. + if (failedChildren.Count > 0) + { + logger.LogWarning("Re-enqueueing {Count} failed children for parent {ParentId}: {ChildIds}", + failedChildren.Count, parent.DeckId, string.Join(",", failedChildren)); + + foreach (var childId in failedChildren) + backgroundJobs.Enqueue(j => j.ComputeDeckDifficulty(childId, false)); + + backgroundJobs.Schedule( + j => j.ReaggregateParentDifficulty(parent.DeckId), TimeSpan.FromMinutes(45)); + } } private async Task ComputeSingleDeckDifficulty(JitenDbContext context, Deck deck) diff --git a/Jiten.Core/JitenHelper.cs b/Jiten.Core/JitenHelper.cs index 602de370..9e9f7d15 100644 --- a/Jiten.Core/JitenHelper.cs +++ b/Jiten.Core/JitenHelper.cs @@ -13,9 +13,38 @@ public static class JitenHelper { // Cap concurrent PostgreSQL COPY operations to reduce server pressure/timeouts private static readonly SemaphoreSlim CopySemaphore = new SemaphoreSlim(6); + + private static string? StripNul(string? value) => + value != null && value.Contains('\0') ? value.Replace("\0", "") : value; + + private static void SanitizeDeckStrings(Deck deck) + { + deck.OriginalTitle = StripNul(deck.OriginalTitle) ?? deck.OriginalTitle; + deck.RomajiTitle = StripNul(deck.RomajiTitle); + deck.EnglishTitle = StripNul(deck.EnglishTitle); + deck.Description = StripNul(deck.Description); + + foreach (var title in deck.Titles) + title.Title = StripNul(title.Title) ?? title.Title; + + foreach (var link in deck.Links) + link.Url = StripNul(link.Url) ?? link.Url; + + if (deck.RawText != null) + deck.RawText.RawText = StripNul(deck.RawText.RawText) ?? deck.RawText.RawText; + + if (deck.ExampleSentences != null) + foreach (var sentence in deck.ExampleSentences) + sentence.Text = StripNul(sentence.Text) ?? sentence.Text; + + foreach (var child in deck.Children) + SanitizeDeckStrings(child); + } + public static async Task InsertDeck(IDbContextFactory contextFactory, Deck deck, byte[] cover, bool update = false) { var totalTimer = Stopwatch.StartNew(); + SanitizeDeckStrings(deck); Console.WriteLine($"[{DateTime.UtcNow:O}] Inserting deck {deck.OriginalTitle}..."); byte[]? optimizedCoverBytes = null; @@ -204,7 +233,10 @@ public static async Task InsertDeck(IDbContextFactory contextFac /* ignore */ } - Console.WriteLine($"[{DateTime.UtcNow:O}] Error inserting deck: {ex.Message}"); + var detail = ex.Message; + for (var inner = ex.InnerException; inner != null; inner = inner.InnerException) + detail += $"\n -> {inner.GetType().Name}: {inner.Message}"; + Console.WriteLine($"[{DateTime.UtcNow:O}] Error inserting deck: {detail}"); return; } }