Skip to content
Merged

Fixes #421

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions Jiten.Api/Controllers/AdminController.cs
Original file line number Diff line number Diff line change
Expand Up @@ -906,6 +906,37 @@ public async Task<IActionResult> ReaggregateParentDifficulty(int deckId)
return Ok(new { Message = "Queued difficulty reaggregation", DeckId = deckId });
}

/// <summary>
/// Backstop sweep: find leaf decks that have raw text but no computed difficulty (e.g. children left at 0 by a
/// transient RunPod failure) and re-enqueue them, then schedule re-aggregation for any affected parents.
/// </summary>
[HttpPost("recompute-missing-difficulties")]
public async Task<IActionResult> RecomputeMissingDifficulties()
{
var missing = await dbContext.Decks
.Where(d => d.RawText != null && d.DeckDifficulty == null && !d.Children.Any())
.Select(d => new { d.DeckId, d.ParentDeckId })
.ToListAsync();

foreach (var m in missing)
backgroundJobs.Enqueue<DifficultyComputationJob>(
job => job.ComputeDeckDifficulty(m.DeckId, false));

var parentIds = missing
.Where(m => m.ParentDeckId != null)
.Select(m => m.ParentDeckId!.Value)
.Distinct()
.ToList();

foreach (var parentId in parentIds)
backgroundJobs.Schedule<DifficultyComputationJob>(
job => job.ReaggregateParentDifficulty(parentId), TimeSpan.FromMinutes(45));

logger.LogInformation("Admin queued {Count} missing-difficulty recomputations across {ParentCount} parents",
missing.Count, parentIds.Count);
return Ok(new { Message = $"Queued {missing.Count} missing-difficulty recomputations", Count = missing.Count, AffectedParents = parentIds.Count });
}

[HttpGet("issues")]
public async Task<IActionResult> GetIssues()
{
Expand Down
23 changes: 21 additions & 2 deletions Jiten.Api/Jobs/DifficultyComputationJob.cs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ public class DifficultyComputationJob(
IDbContextFactory<JitenDbContext> contextFactory,
IHttpClientFactory httpClientFactory,
IConfiguration configuration,
IBackgroundJobClient backgroundJobs,
ILogger<DifficultyComputationJob> logger)
{
private record DifficultyResponse(
Expand Down Expand Up @@ -128,6 +129,7 @@ private async Task ComputeParentWithChildren(JitenDbContext context, Deck parent
parent.DeckId, children.Count, forceRecompute);

var computedCount = 0;
var failedChildren = new List<int>();
foreach (var child in children)
{
if (!forceRecompute && child.DeckDifficulty != null)
Expand All @@ -151,7 +153,10 @@ private async Task ComputeParentWithChildren(JitenDbContext context, Deck parent
}
catch (Exception ex)
{
logger.LogError(ex, "Failed to compute difficulty for child deck {DeckId}, skipping", child.DeckId);
// Usually a transient RunPod worker failure (e.g. CUDA "no kernel image"). Re-enqueue the child
// below as its own job so it inherits the per-deck delayed-retry policy and lands on a fresh worker.
logger.LogError(ex, "Failed to compute difficulty for child deck {DeckId}, will re-enqueue", child.DeckId);
failedChildren.Add(child.DeckId);
}
}

Expand All @@ -162,8 +167,22 @@ private async Task ComputeParentWithChildren(JitenDbContext context, Deck parent
.OrderBy(d => d.DeckOrder)
.ToListAsync();

// Aggregate parent difficulty from children
// Aggregate parent difficulty from whatever succeeded so the parent is never blocked by a few failures
await AggregateParentDifficulty(context, parent, childrenWithDifficulty);

// Self-heal stragglers: re-enqueue each failed child (leaf → 3× delayed retry) and schedule a single
// delayed re-aggregation so the parent picks up the recovered children once their retries complete.
if (failedChildren.Count > 0)
{
logger.LogWarning("Re-enqueueing {Count} failed children for parent {ParentId}: {ChildIds}",
failedChildren.Count, parent.DeckId, string.Join(",", failedChildren));

foreach (var childId in failedChildren)
backgroundJobs.Enqueue<DifficultyComputationJob>(j => j.ComputeDeckDifficulty(childId, false));

backgroundJobs.Schedule<DifficultyComputationJob>(
j => j.ReaggregateParentDifficulty(parent.DeckId), TimeSpan.FromMinutes(45));
}
}

private async Task ComputeSingleDeckDifficulty(JitenDbContext context, Deck deck)
Expand Down
34 changes: 33 additions & 1 deletion Jiten.Core/JitenHelper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,38 @@ public static class JitenHelper
{
// Cap concurrent PostgreSQL COPY operations to reduce server pressure/timeouts
private static readonly SemaphoreSlim CopySemaphore = new SemaphoreSlim(6);

private static string? StripNul(string? value) =>
value != null && value.Contains('\0') ? value.Replace("\0", "") : value;

private static void SanitizeDeckStrings(Deck deck)
{
deck.OriginalTitle = StripNul(deck.OriginalTitle) ?? deck.OriginalTitle;
deck.RomajiTitle = StripNul(deck.RomajiTitle);
deck.EnglishTitle = StripNul(deck.EnglishTitle);
deck.Description = StripNul(deck.Description);

foreach (var title in deck.Titles)
title.Title = StripNul(title.Title) ?? title.Title;

foreach (var link in deck.Links)
link.Url = StripNul(link.Url) ?? link.Url;

if (deck.RawText != null)
deck.RawText.RawText = StripNul(deck.RawText.RawText) ?? deck.RawText.RawText;

if (deck.ExampleSentences != null)
foreach (var sentence in deck.ExampleSentences)
sentence.Text = StripNul(sentence.Text) ?? sentence.Text;

foreach (var child in deck.Children)
SanitizeDeckStrings(child);
}

public static async Task InsertDeck(IDbContextFactory<JitenDbContext> contextFactory, Deck deck, byte[] cover, bool update = false)
{
var totalTimer = Stopwatch.StartNew();
SanitizeDeckStrings(deck);
Console.WriteLine($"[{DateTime.UtcNow:O}] Inserting deck {deck.OriginalTitle}...");

byte[]? optimizedCoverBytes = null;
Expand Down Expand Up @@ -204,7 +233,10 @@ public static async Task InsertDeck(IDbContextFactory<JitenDbContext> contextFac
/* ignore */
}

Console.WriteLine($"[{DateTime.UtcNow:O}] Error inserting deck: {ex.Message}");
var detail = ex.Message;
for (var inner = ex.InnerException; inner != null; inner = inner.InnerException)
detail += $"\n -> {inner.GetType().Name}: {inner.Message}";
Console.WriteLine($"[{DateTime.UtcNow:O}] Error inserting deck: {detail}");
return;
}
}
Expand Down