Fixes + composition filters

Jiten.Api/Controllers/SrsController.cs

@@ -1166,13 +1166,21 @@ public async Task<IResult> CompositionInferencePreview(CompositionInferenceReque
         var inferred = await ComputeInferredPairs(userId, request.Direction);
         if (inferred == null) return Results.BadRequest($"Invalid direction: {request.Direction}");
 
-        var total = inferred.Count;
+        var (items, counts) = await CategorizeAndFilter(
+            userId, inferred, request.ShowNew, request.ShowLearning, request.ShowMature);
+
+        var total = items.Count;
         var limit = Math.Clamp(request.Limit, 1, 100);
         var offset = Math.Max(0, request.Offset);
-        var page = inferred.Skip(offset).Take(limit).ToList();
+        var page = items.Skip(offset).Take(limit).ToList();
+
+        var dtos = await HydrateInferredPairs(userId, page.Select(p => (p.WordId, p.ReadingIndex)).ToList());
+        var catByPair = page.ToDictionary(p => (p.WordId, p.ReadingIndex), p => p.Category);
+        foreach (var d in dtos)
+            d.Category = catByPair.GetValueOrDefault((d.WordId, d.ReadingIndex));
 
-        var dtos = await HydrateInferredPairs(userId, page);
-        return Results.Ok(new PaginatedResponse<List<MassActionCardDto>>(dtos, total, limit, offset));
+        return Results.Ok(new CompositionInferencePreviewResponse(
+            dtos, total, limit, offset, counts.New, counts.Learning, counts.Mature));
     }
 
     [HttpPost("composition-inference/execute")]
@@ -1203,7 +1211,9 @@ public async Task<IResult> CompositionInferenceExecute(CompositionInferenceExecu
         {
             var inferred = await ComputeInferredPairs(userId, request.Direction);
             if (inferred == null) return Results.BadRequest($"Invalid direction: {request.Direction}");
-            pairs = inferred;
+            var (items, _) = await CategorizeAndFilter(
+                userId, inferred, request.ShowNew, request.ShowLearning, request.ShowMature);
+            pairs = items.Select(x => (x.WordId, x.ReadingIndex)).ToList();
         }
 
         if (pairs.Count == 0) return Results.Json(new { success = true, affectedCount = 0 });
@@ -1315,6 +1325,69 @@ public async Task<IResult> CompositionInferenceExecute(CompositionInferenceExecu
         return null;
     }
 
+    private async Task<(List<(int WordId, byte ReadingIndex, string Category)> Items, (int New, int Learning, int Mature) Counts)>
+        CategorizeAndFilter(string userId, List<(int WordId, byte ReadingIndex)> pairs,
+                            bool showNew, bool showLearning, bool showMature)
+    {
+        if (pairs.Count == 0)
+            return (new List<(int, byte, string)>(), (0, 0, 0));
+
+        var wordIds = pairs.Select(p => p.WordId).Distinct().ToList();
+        var cards = await userContext.FsrsCards
+            .Where(c => c.UserId == userId && wordIds.Contains(c.WordId))
+            .Select(c => new { c.WordId, c.ReadingIndex, c.State, c.Due, c.LastReview })
+            .ToListAsync();
+        var cardDict = cards.ToDictionary(c => (c.WordId, c.ReadingIndex));
+
+        var threshold = TimeSpan.FromDays(21);
+
+        string Categorize((int WordId, byte ReadingIndex) p)
+        {
+            if (!cardDict.TryGetValue(p, out var c) || c.State == FsrsState.New)
+                return "new";
+            if (c.State == FsrsState.Review && c.LastReview != null && c.Due - c.LastReview.Value >= threshold)
+                return "mature";
+            return "learning";
+        }
+
+        var categorized = pairs
+            .Select(p => (p.WordId, p.ReadingIndex, Category: Categorize(p)))
+            .ToList();
+
+        var counts = (
+            New: categorized.Count(x => x.Category == "new"),
+            Learning: categorized.Count(x => x.Category == "learning"),
+            Mature: categorized.Count(x => x.Category == "mature")
+        );
+
+        var allowed = new HashSet<string>();
+        if (showNew) allowed.Add("new");
+        if (showLearning) allowed.Add("learning");
+        if (showMature) allowed.Add("mature");
+
+        var filtered = categorized.Where(x => allowed.Contains(x.Category)).ToList();
+        if (filtered.Count == 0)
+            return (new List<(int, byte, string)>(), counts);
+
+        var filteredWordIds = filtered.Select(x => x.WordId).Distinct().ToList();
+        var freqDict = await WordFormHelper.LoadWordFormFrequencies(context, filteredWordIds);
+
+        var items = filtered
+            .Select(x =>
+            {
+                var rank = freqDict.GetValueOrDefault((x.WordId, (short)x.ReadingIndex))?.FrequencyRank ?? 0;
+                return (x.WordId, x.ReadingIndex, x.Category, Rank: rank);
+            })
+            .OrderBy(x => x.Rank <= 0)
+            .ThenBy(x => x.Rank)
+            .ThenBy(x => x.WordId)
+            .ThenBy(x => x.ReadingIndex)
+            .Select(x => (x.WordId, x.ReadingIndex, x.Category))
+            .ToList();
+
+        return (items, counts);
+    }
+
     private async Task<(HashSet<(int WordId, byte ReadingIndex)> Known, HashSet<(int WordId, byte ReadingIndex)> Blocked)>
         LoadKnownAndBlockedSets(string userId)
     {

Jiten.Api/Dtos/CompositionInferencePreviewResponse.cs

@@ -0,0 +1,16 @@
+namespace Jiten.Api.Dtos;
+
+public class CompositionInferencePreviewResponse(
+    List<MassActionCardDto> data, int totalItems, int pageSize, int currentOffset,
+    int newCount, int learningCount, int matureCount)
+    : PaginatedResponse<List<MassActionCardDto>>(data, totalItems, pageSize, currentOffset)
+{
+    /// <summary>Total never-studied words available for this direction (ignores the active filter).</summary>
+    public int NewCount { get; set; } = newCount;
+
+    /// <summary>Total in-progress words (Learning/Relearning/young Review/Suspended) available for this direction.</summary>
+    public int LearningCount { get; set; } = learningCount;
+
+    /// <summary>Total mature words (Review with interval ≥ 21 days) available for this direction.</summary>
+    public int MatureCount { get; set; } = matureCount;
+}

Jiten.Api/Dtos/MassActionCardDto.cs

@@ -12,4 +12,9 @@ public class MassActionCardDto
     public FsrsState State { get; set; }
     public DateTime Due { get; set; }
     public DateTime CreatedAt { get; set; }
+
+    /// <summary>
+    /// Composition-inference maturity bucket: "new", "learning" or "mature". Null for other endpoints.
+    /// </summary>
+    public string? Category { get; set; }
 }

Jiten.Api/Dtos/Requests/CompositionInferenceExecuteRequest.cs

@@ -6,6 +6,10 @@ public class CompositionInferenceExecuteRequest
     public required string TargetState { get; set; }
     public List<WordKey>? WordKeys { get; set; }
 
+    public bool ShowNew { get; set; } = true;
+    public bool ShowLearning { get; set; } = false;
+    public bool ShowMature { get; set; } = false;
+
     public class WordKey
     {
         public int WordId { get; set; }

Jiten.Api/Dtos/Requests/CompositionInferenceRequest.cs

@@ -6,4 +6,8 @@ public class CompositionInferenceRequest
 
     public int Offset { get; set; }
     public int Limit { get; set; } = 50;
+
+    public bool ShowNew { get; set; } = true;
+    public bool ShowLearning { get; set; } = false;
+    public bool ShowMature { get; set; } = false;
 }

Jiten.Api/Helpers/ConfusableReadingsHelper.cs

@@ -38,7 +38,7 @@ public static class ConfusableReadingsHelper
 
         var sourceReadings = sourceKanaForms
             .GroupBy(kf => kf.WordId)
-            .ToDictionary(g => g.Key, g => g.Select(x => x.Text).ToHashSet());
+            .ToDictionary(g => g.Key, g => g.Select(x => JapaneseTextHelper.KatakanaToHiragana(x.Text)).ToHashSet());
 
         var distinctTexts = kanjiTexts.Select(kt => kt.Text).Distinct().ToList();
         var textToSourcePairs = kanjiTexts
@@ -119,7 +119,7 @@ public static class ConfusableReadingsHelper
             var readings = confIds
                 .Where(validReadings.ContainsKey)
                 .Select(id => (reading: validReadings[id], rank: freqByWord.GetValueOrDefault(id, int.MaxValue)))
-                .Where(x => ownReadings == null || !ownReadings.Contains(x.reading))
+                .Where(x => ownReadings == null || !ownReadings.Contains(JapaneseTextHelper.KatakanaToHiragana(x.reading)))
                 .OrderBy(x => x.rank)
                 .Select(x => x.reading)
                 .ToList();

Jiten.Api/Services/TtsService.cs

@@ -74,8 +74,9 @@ public async Task<byte[]> GetWordAudioAsync(int wordId, int readingIndex, string
 
 
         string? text = !string.IsNullOrEmpty(rubyText) ? RubyToTtsKana(rubyText) : null;
+        var usedRuby = !string.IsNullOrWhiteSpace(text) && !ContainsKanji(text);
 
-        if (string.IsNullOrWhiteSpace(text) || ContainsKanji(text))
+        if (!usedRuby)
         {
             text = wordForms
                 .Where(f => f.FormType == JmDictFormType.KanaForm)
@@ -93,6 +94,19 @@ public async Task<byte[]> GetWordAudioAsync(int wordId, int readingIndex, string
         // (詳しくは→"kuwashiku wa", ではない→"de wa nai").
         text = FixLeadingParticleKana(text);
 
+        var readingKana = WanaKana.ToHiragana(text ?? "");
+        var matchingKanjiRubies = wordForms
+            .Where(f => f.FormType == JmDictFormType.KanjiForm && !string.IsNullOrEmpty(f.RubyText) && f.RubyText.Contains('['))
+            .Where(f => WanaKana.ToHiragana(RubyPattern.Replace(f.RubyText, m => m.Groups[1].Value)) == readingKana)
+            .Select(f => f.RubyText!)
+            .ToList();
+        var hasLiteralHa = matchingKanjiRubies.Any(r => r.Contains('は') && !RubyPattern.Replace(r, "").Contains('は'));
+        var hasParticleHa = matchingKanjiRubies.Any(r => RubyPattern.Replace(r, "").Contains('は'));
+
+        var fixInteriorHa = !string.IsNullOrEmpty(text)
+                            && text.Contains('は') && !text.Contains('わ')
+                            && hasLiteralHa && !hasParticleHa;
+
 
         int? pitchPosition = null;
         var distinctReadings = wordForms
@@ -110,8 +124,9 @@ public async Task<byte[]> GetWordAudioAsync(int wordId, int readingIndex, string
         }
 
         var storageText = pitchPosition.HasValue ? $"{text}|p{pitchPosition.Value}" : text;
+        if (fixInteriorHa) storageText += "|h"; // distinct cache key: audio differs though the text string does not
         var key = $"{voice}:w:{storageText}";
-        return await _inflight.GetOrAdd(key, _ => GenerateAsync(key, text, storageText, pitchPosition, TtsType.Word, voice, rateLimitKey, ct, bypassGenerationLimit));
+        return await _inflight.GetOrAdd(key, _ => GenerateAsync(key, text, storageText, pitchPosition, TtsType.Word, voice, rateLimitKey, ct, bypassGenerationLimit, fixInteriorHa));
     }
 
     public async Task<byte[]> GetSentenceAudioAsync(int sentenceId, string voice, string rateLimitKey, CancellationToken ct)
@@ -209,14 +224,14 @@ private async Task<byte[]> GenerateSentenceAsync(string key, string rawText, str
         }
     }
 
-    private async Task<byte[]> GenerateAsync(string key, string ttsText, string storageText, int? pitchPosition, TtsType type, string voice, string rateLimitKey, CancellationToken ct, bool bypassGenerationLimit = false)
+    private async Task<byte[]> GenerateAsync(string key, string ttsText, string storageText, int? pitchPosition, TtsType type, string voice, string rateLimitKey, CancellationToken ct, bool bypassGenerationLimit = false, bool fixInteriorHa = false)
     {
         try
         {
             var cached = await TryGetFromCdn(storageText, type, voice, ct);
             if (cached != null) return cached;
 
-            return await SynthesizeAndUpload(key, ttsText, storageText, pitchPosition, type, voice, rateLimitKey, ct, bypassGenerationLimit);
+            return await SynthesizeAndUpload(key, ttsText, storageText, pitchPosition, type, voice, rateLimitKey, ct, bypassGenerationLimit, fixInteriorHa);
         }
         finally
         {
@@ -242,7 +257,7 @@ private async Task<byte[]> GenerateAsync(string key, string ttsText, string stor
         return null;
     }
 
-    private async Task<byte[]> SynthesizeAndUpload(string key, string ttsText, string storageText, int? pitchPosition, TtsType type, string voice, string rateLimitKey, CancellationToken ct, bool bypassGenerationLimit = false)
+    private async Task<byte[]> SynthesizeAndUpload(string key, string ttsText, string storageText, int? pitchPosition, TtsType type, string voice, string rateLimitKey, CancellationToken ct, bool bypassGenerationLimit = false, bool fixInteriorHa = false)
     {
         if (!bypassGenerationLimit)
         {
@@ -270,6 +285,8 @@ private async Task<byte[]> SynthesizeAndUpload(string key, string ttsText, strin
             queryDict["speedScale"] = JsonSerializer.SerializeToElement(config.SpeedScale);
         if (type == TtsType.Sentence)
             queryDict["intonationScale"] = JsonSerializer.SerializeToElement(1.5);
+        if (fixInteriorHa)
+            FixInteriorHaMoras(queryDict);
         if (pitchPosition.HasValue)
         {
             try { await ApplyPitchAccent(vvClient, queryDict, pitchPosition.Value, speakerId, ct); }
@@ -417,6 +434,36 @@ private static void BoostVoicedConsonants(Dictionary<string, JsonElement> queryD
         queryDict["accent_phrases"] = JsonSerializer.SerializeToElement(phrases);
     }
 
+    // Rewrites every "wa" mora (ワ, consonant "w") to "ha" (ハ, consonant "h") in the audio query.
+    // Caller guarantees this query came from a reading whose は is a literal kanji reading and that
+    // contains no genuine わ, so any ワ mora can only be an interior は that OpenJTalk wrongly voiced
+    // as the topic particle. Editing the mora in place keeps the
+    // accent/length/pitch VOICEVOX assigned to the natural hiragana, so the prosody is unchanged.
+    private static void FixInteriorHaMoras(Dictionary<string, JsonElement> queryDict)
+    {
+        if (!queryDict.TryGetValue("accent_phrases", out var phrasesEl)) return;
+        var phrases = JsonSerializer.Deserialize<List<Dictionary<string, JsonElement>>>(phrasesEl.GetRawText());
+        if (phrases == null) return;
+
+        foreach (var phrase in phrases)
+        {
+            if (!phrase.TryGetValue("moras", out var morasEl)) continue;
+            var moras = JsonSerializer.Deserialize<List<Dictionary<string, JsonElement>>>(morasEl.GetRawText());
+            if (moras == null) continue;
+
+            foreach (var mora in moras)
+            {
+                if (!mora.TryGetValue("text", out var tEl) || tEl.ValueKind != JsonValueKind.String || tEl.GetString() != "ワ") continue;
+                mora["text"] = JsonSerializer.SerializeToElement("ハ");
+                mora["consonant"] = JsonSerializer.SerializeToElement("h");
+            }
+
+            phrase["moras"] = JsonSerializer.SerializeToElement(moras);
+        }
+
+        queryDict["accent_phrases"] = JsonSerializer.SerializeToElement(phrases);
+    }
+
     private static async Task ApplyPitchAccent(HttpClient client, Dictionary<string, JsonElement> queryDict, int position, int speakerId, CancellationToken ct)
     {
         if (!queryDict.TryGetValue("accent_phrases", out var phrasesEl) || phrasesEl.ValueKind != JsonValueKind.Array) return;

Jiten.Cli/CliOptions.cs

@@ -112,6 +112,12 @@ public class CliOptions
     [Option(longName: "split-type", Required = false, Default = "CB", HelpText = "SudachiDict split type to ingest (C, B, CB). Default CB.")]
     public string SplitType { get; set; } = "CB";
 
+    [Option(longName: "cleanup-compositions", Required = false, HelpText = "Delete name-parent, particle-component and all-single-char WordCompositions (idempotent/rerunnable). Combine with --dry-run to preview counts.")]
+    public bool CleanupCompositions { get; set; }
+
+    [Option(longName: "backfill-compositions", Required = false, HelpText = "Fill WordCompositions for compounds with no rows via Sudachi Mode A + sense resolver (idempotent/rerunnable). Combine with --dry-run for a sampled coverage report.")]
+    public bool BackfillCompositions { get; set; }
+
     [Option(longName: "sync-jmnedict", Required = false, HelpText = "Sync missing JMNedict entries and update partial entries with missing readings/definitions.")]
     public string? SyncJMNedict { get; set; }
 

Jiten.Cli/Commands/ImportCommands.cs

@@ -54,6 +54,16 @@ await WordCompositionImporter.Import(
             options.DryRun);
     }
 
+    public async Task CleanupCompositions(CliOptions options)
+    {
+        await CompositionCleaner.Cleanup(context.ContextFactory, options.DryRun);
+    }
+
+    public async Task BackfillCompositions(CliOptions options)
+    {
+        await CompositionBackfill.Backfill(context.ContextFactory, options.DryRun, dryRunSample: 12000);
+    }
+
     public async Task ImportPitchAccents(CliOptions options)
     {
         Console.WriteLine("Importing pitch accents...");