diff --git a/CLAUDE.md b/CLAUDE.md index 097ddc73..b25ec394 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -112,7 +112,7 @@ dotnet run --project Jiten.Cli -- --search-lookup "そうする" # Lookups ta **Key files for parser fixes:** - `Jiten.Parser/MorphologicalAnalyser.cs`: `SpecialCases2`/`SpecialCases3` (hardcoded token combinations), `MisparsesRemove` (tokens to filter), `Combine*` methods (merging logic), `PreprocessText()` (forced splits), `RepairNTokenisation()` (ん-form fixes) -- `Jiten.Parser/Stages/MorphologicalAnalyser.RewriteRules.cs`: declarative token-rewrite table + engine. **A new single-surface lexical fix (pin a WordId, or split/merge a specific surface into fixed tokens) should be a `RewriteRule` row here, not a hand-rolled `if`-block.** The engine owns offsets, readings-from-templates, cloning, and pin/conjugation recovery. Phases: `Cleanup` (runs before `FilterMisparse`), `Late` (mora-theft position), `Early` (ProcessSpecialCases position). Add a new pipeline **stage** only for a pattern-general mechanism, never a single lexical pattern. Context-heavy logic (window scans, dynamic readings, POS-section edits, order-dependent-on-another-block) legitimately stays as code. +- `Jiten.Parser/Stages/MorphologicalAnalyser.RewriteRules.cs`: declarative token-rewrite table + engine. **A new single-surface lexical fix (pin a WordId, or split/merge a specific surface into fixed tokens) should be a `RewriteRule` row here, not a hand-rolled `if`-block.** The engine owns offsets, readings-from-templates, cloning, and pin/conjugation recovery. Phases: `Cleanup` (runs before `FilterMisparse`), `Late` (mora-theft position), `Early` (ProcessSpecialCases position). A rule pin is soft by default (lookup-time compound matching may absorb the token into a longer attested span); set `HardPin` when the pin is a final word decision that compound matching must not swallow. Add a new pipeline **stage** only for a pattern-general mechanism, never a single lexical pattern. Context-heavy logic (wider/dynamic window scans, dynamic readings, POS-section edits, multi-token mutation, order-dependent-on-another-block) legitimately stays as code. - `Shared/resources/deconjugator.json`: Rule-based deconjugation (~1500 rules). Types: `stdrule`, `rewriterule`, `onlyfinalrule`, `neverfinalrule`, `contextrule`. Fields: `dec_end`/`con_end` (endings), `dec_tag`/`con_tag` (grammar tags). Search for specific endings or `"detail": "past"` etc. - `Shared/resources/user_dic.xml`: Custom Sudachi dictionary entries. Format: `surface,leftId,rightId,cost,display,pos1,pos2,pos3,pos4,conjType,conjForm,reading,normalised,dictFormId,splitType,splitA,splitB,unused`. Regenerate after editing: `sudachi ubuild "Y:\CODE\Jiten\Shared\resources\user_dic.xml" -s "S:\Jiten\sudachi.rs\resources\system_full.dic" -o "Y:\CODE\Jiten\Shared\resources\user_dic.dic"` diff --git a/Jiten.Core/Data/PosMapper.cs b/Jiten.Core/Data/PosMapper.cs index fb8b6aaf..bcaa8823 100644 --- a/Jiten.Core/Data/PosMapper.cs +++ b/Jiten.Core/Data/PosMapper.cs @@ -350,6 +350,10 @@ public static bool IsJmDictCompatibleWithSudachi( // Sudachi 接尾辞 (Suffix) should match JMDict n-suf (NounSuffix) and suf (Suffix). // E.g. だらけ is n-suf in JMDict but 接尾辞 in Sudachi; 達 (たち) is suf in JMDict. + // Counter (ctr) entries are deliberately NOT compatible here: counter eligibility is + // contextual (needs a preceding numeral), which this context-free check cannot see — + // a suffix-tagged kanji inside a plain compound (結晶|石) must not prefer a counter + // homograph. Numeral contexts select counters in adjacent rescoring instead. if (sudachiPos == PartOfSpeech.Suffix && (convertedPosList.Contains(PartOfSpeech.NounSuffix) || convertedPosList.Contains(PartOfSpeech.Suffix))) return true; diff --git a/Jiten.Core/JapaneseTextHelper.cs b/Jiten.Core/JapaneseTextHelper.cs index 80ce7d38..f0c32dfd 100644 --- a/Jiten.Core/JapaneseTextHelper.cs +++ b/Jiten.Core/JapaneseTextHelper.cs @@ -35,6 +35,11 @@ public static bool IsAllKatakana(ReadOnlySpan s) return true; } + /// Katakana letter (ァ–ヺ) or the long-vowel mark ー — the characters a katakana + /// word is spelled with, excluding middle dots and iteration marks. + public static bool IsKatakanaWordChar(char c) => + c is (>= 'ァ' and <= 'ヺ') or 'ー'; + /// Fullwidth (Ａ-Ｚ/ａ-ｚ) or halfwidth (A-Z/a-z) Latin letter. public static bool IsLatinLetter(char c) => c is (>= '\uFF21' and <= '\uFF3A') or (>= '\uFF41' and <= '\uFF5A') // fullwidth Ａ-Ｚ / ａ-ｚ @@ -86,4 +91,14 @@ public static bool IsKanji(Rune r) (>= 0xF900 and <= 0xFAFF) or // Compatibility Ideographs (>= 0x2F800 and <= 0x2FA1F); // Compatibility Supplement } + + /// + /// A numeral character: an ASCII or full-width digit, a kanji digit (一〜九), a kanji place + /// marker (十百千万億兆), or a kanji zero (〇零). Covers the counting/quantity vocabulary shared by + /// the numeral-context guards; contexts that need a narrower kanji-only set test IsKanji separately. + /// + public static bool IsNumeralChar(char c) => + c is (>= '0' and <= '9') or (>= '０' and <= '９') + or '一' or '二' or '三' or '四' or '五' or '六' or '七' or '八' or '九' + or '十' or '百' or '千' or '万' or '億' or '兆' or '〇' or '零'; } diff --git a/Jiten.Parser/Data/WordInfo.cs b/Jiten.Parser/Data/WordInfo.cs index f321aff0..1067e7a5 100644 --- a/Jiten.Parser/Data/WordInfo.cs +++ b/Jiten.Parser/Data/WordInfo.cs @@ -38,6 +38,23 @@ public class WordInfo /// ProcessWord escalation chain performs. public bool IsKanaExclamation { get; set; } + /// Set when Sudachi originally tagged this all-katakana token as a noun — a name/loanword + /// shape. Deconjugation of its hiragana conversion must not land on kanji-primary words + /// (ハガナ → はが+な → 剥ぐ fabricates vocabulary out of a name). Survives the POS rewrites + /// that the ProcessWord escalation chain performs. + public bool IsKatakanaNounSurface { get; set; } + + /// Set when a rewrite-rule template pinned this token — a deliberate lexical decision the + /// misparse gates must not overrule (え|っつった). PreMatchedWordId alone can't carry this: + /// parser-level machinery (compound matches, fallbacks) reuses it for ordinary tokens. + public bool PinnedByRewriteRule { get; set; } + + /// Set when a gate's pin is a final word decision that compound formation must not absorb + /// or override (the ぶん of a fraction frame, the ordinal 目). Soft pins — reading defaults + /// like the する-family — stay absorbable: an attested expression spanning them (そうした, + /// 臆病風に吹かれる) is the better parse. + public bool HardPinned { get; set; } + /// Sudachi lattice segmentation margin: extra cost of the cheapest competing lattice path /// crossing one of this token's boundaries (clamped to 99999 = no competitor). /// Null when margin output was not requested. Low values = uncertain segmentation. @@ -70,6 +87,9 @@ public WordInfo(WordInfo other) ResolvedWordId = other.ResolvedWordId; SudachiBoundaryMargin = other.SudachiBoundaryMargin; IsKanaExclamation = other.IsKanaExclamation; + IsKatakanaNounSurface = other.IsKatakanaNounSurface; + PinnedByRewriteRule = other.PinnedByRewriteRule; + HardPinned = other.HardPinned; } public WordInfo(string sudachiLine) diff --git a/Jiten.Parser/Grammar/TransitionRuleEngine.cs b/Jiten.Parser/Grammar/TransitionRuleEngine.cs index a47f412b..af0a5e4e 100644 --- a/Jiten.Parser/Grammar/TransitionRuleEngine.cs +++ b/Jiten.Parser/Grammar/TransitionRuleEngine.cs @@ -156,15 +156,20 @@ or PartOfSpeech.PrenounAdjectival or PartOfSpeech.Counter MatchCondition.NextIsNotQuotative => w.Next == null || !w.Next.Text.StartsWith("という"), - // Valid hosts for a sentence-final particle: auxiliaries/particles (だな, はね) plus — - // for な only — plain-form verbs/i-adjectives (prohibitive えぐるな, exclamatory - // 欲しいな), which stay valid mid-sentence because run-on speech has no punctuation. - // Other SFPs keep the merge: ない+か must still form ないか. + // Valid hosts for a sentence-final particle: auxiliaries/particles (だな, はね), plain + // predicates before な (prohibitive えぐるな, exclamatory 欲しいな), and plain + // predicates before よ/ね only when the next token looks vocative + // (やべえよ母ちゃん). The vocative gate keeps ordinary run-on text such as + // 食べるよ明日 subject to the existing merge/remove repair. MatchCondition.PrevIsSfpValidHost => w.Prev?.PartOfSpeech is PartOfSpeech.Auxiliary or PartOfSpeech.Particle || (w.Current.Text == "な" && w.Prev is { PartOfSpeech: PartOfSpeech.Verb or PartOfSpeech.IAdjective } - && w.Prev.Text == w.Prev.DictionaryForm), + && w.Prev.Text == w.Prev.DictionaryForm) + || (w.Current.Text is "よ" or "ね" + && w.Prev is { PartOfSpeech: PartOfSpeech.Verb or PartOfSpeech.IAdjective } + && w.Prev.Text == w.Prev.DictionaryForm + && IsVocativeFollower(w.Next)), _ => false }; @@ -173,6 +178,17 @@ or PartOfSpeech.PrenounAdjectival or PartOfSpeech.Counter return true; } + private static bool IsVocativeFollower(WordInfo? next) + { + if (next?.PartOfSpeech is PartOfSpeech.Name or PartOfSpeech.Pronoun) + return true; + if (next?.PartOfSpeech is not (PartOfSpeech.Noun or PartOfSpeech.CommonNoun)) + return false; + + return TransitionRuleSets.HonorificSuffixes.Any( + suffix => next.Text.EndsWith(suffix, StringComparison.Ordinal)); + } + private static void ApplyViolation( TransitionRule rule, List<(WordInfo word, int pos, int len)> words, diff --git a/Jiten.Parser/Grammar/TransitionRuleSets.cs b/Jiten.Parser/Grammar/TransitionRuleSets.cs index a5bf48a7..45172973 100644 --- a/Jiten.Parser/Grammar/TransitionRuleSets.cs +++ b/Jiten.Parser/Grammar/TransitionRuleSets.cs @@ -160,10 +160,12 @@ internal static class TransitionRuleSets [ScoringCondition.NextIsExplanatoryN], 25), + // High enough that a numeral+counter pair outbids a top-priority noun homograph together + // with its ruby prior (第二話: the counter わ must beat はなし). new("numeral-counter-cohesion", [ScoringCondition.CandidateIsCounter], [ScoringCondition.PrevIsNumeral], - 40), + 60), new("orphan-counter-penalty", [ScoringCondition.CandidateIsCounter, ScoringCondition.CandidateIsNotNounLike], diff --git a/Jiten.Parser/Helpers/MorphologicalAnalyser.Diagnostics.cs b/Jiten.Parser/Helpers/MorphologicalAnalyser.Diagnostics.cs index 672f9819..58b0ce1d 100644 --- a/Jiten.Parser/Helpers/MorphologicalAnalyser.Diagnostics.cs +++ b/Jiten.Parser/Helpers/MorphologicalAnalyser.Diagnostics.cs @@ -30,12 +30,13 @@ private static List ParseSudachiOutputToDiagnosticTokens(string ra return tokens; } - private static List TrackStage(TokenStage stage, List input, ParserDiagnostics? diagnostics) + private static List TrackStage(TokenStage stage, List input, ParserDiagnostics? diagnostics, + TokenFeatureScan? scan = null) { var inputSnapshot = diagnostics != null ? input.Select(TokenSnapshot.From).ToList() : null; var sw = diagnostics != null ? Stopwatch.StartNew() : null; - var result = stage.Apply(input); + var result = stage.Apply(input, scan); if (diagnostics == null) return result; diff --git a/Jiten.Parser/Helpers/MorphologicalAnalyser.RuleData.cs b/Jiten.Parser/Helpers/MorphologicalAnalyser.RuleData.cs index ab9279cb..f2a8feae 100644 --- a/Jiten.Parser/Helpers/MorphologicalAnalyser.RuleData.cs +++ b/Jiten.Parser/Helpers/MorphologicalAnalyser.RuleData.cs @@ -111,6 +111,7 @@ public partial class MorphologicalAnalyser ("すぐ", "に", PartOfSpeech.Adverb), ("たしか", "に", PartOfSpeech.Adverb), ("確か", "に", PartOfSpeech.Adverb), // 確かに 1205770 — kanji variant of the たしか row above + ("直", "に", PartOfSpeech.Adverb), // 直に 1430690 (じかに, "directly/in person") — the bare 直 otherwise reads ちょく ("ゆえ", "に", PartOfSpeech.Conjunction), // 故に 1267130 ("therefore") — literary connective Sudachi splits ("故", "に", PartOfSpeech.Conjunction), // kanji variant of the ゆえ row above ("とう", "に", PartOfSpeech.Adverb), // 疾うに 1633370 ("long ago") — the bare とう otherwise strands on 塔 diff --git a/Jiten.Parser/Misparse/MisparseGates.cs b/Jiten.Parser/Misparse/MisparseGates.cs index 15518a6a..80c4f74c 100644 --- a/Jiten.Parser/Misparse/MisparseGates.cs +++ b/Jiten.Parser/Misparse/MisparseGates.cs @@ -37,7 +37,9 @@ internal static class MisparseGates /// digits, or a long mixed-kana surface — are excluded before the caller assembles flags, /// neighbour frames and blob context. The character test accepts a superset of the kana the /// gates accept (full/half-width kana blocks plus stretch marks), so it can only skip tokens - /// no gate would touch. + /// no gate would touch. The ≤4 cap excludes 6-character reduplications such as ギュウギュウ; + /// those overlap attested on-mim adverbs such as ごちゃごちゃとして and require separate + /// mimetic preference handling. public static bool MayBeKanaFragment(string s) { if (s.Length == 0) return false; @@ -95,7 +97,14 @@ private static bool IsSfxMimeticFragment(in MisparseGateContext ctx) ? ctx.SelectedWord.OriginalText : ctx.Token.Text; - if (surface.Length == 0 || surface.Length > 4 || !WanaKana.IsKana(surface)) return false; + // Pure reduplication (コクコク) is mimetic orthography — it extends the local length + // window and counts as a positive marker below. Six-character reduplications + // (ギュウギュウ) only arrive when a shorter resolved token carries the longer + // OriginalText — MayBeKanaFragment's ≤4 cap excludes them as whole tokens upstream. + bool reduplicated = surface.Length is 4 or 6 + && surface[..(surface.Length / 2)] == surface[(surface.Length / 2)..]; + if (surface.Length == 0 || (surface.Length > 4 && !reduplicated) || surface.Length > 6 + || !WanaKana.IsKana(surface)) return false; // A neighbour already discarded as a misparse shred is part of the same burst; so is an // unresolved kana scrap. Both make the current token frame-transparent on that side. @@ -117,17 +126,24 @@ private static bool IsSfxMimeticFragment(in MisparseGateContext ctx) // unsegmentable blob. A case particle or content word on either side means real syntax, // which the gate must not touch. When a sokuon is clipped straight onto the token (ず|がんっ), // a short kana neighbour is part of the same burst even if it happened to resolve. + // A reduplicated surface heading the mimetic-adverb frame 〜と+verb (コクコクと振る) is + // phonetic wherever it sits — the construction itself is the isolation. + bool reduplicatedToFrame = reduplicated + && ctx.Next is { Text: "と", PartOfSpeech: PartOfSpeech.Particle }; + bool frameBefore = ctx.Prev == null || ctx.SymbolsBefore.Length > 0 || ctx.IsSentenceInitial || ctx.Prev.PartOfSpeech == PartOfSpeech.Interjection || (ctx.Prev.PartOfSpeech == PartOfSpeech.Particle && ctx.Prev.Text is "な" or "よ" or "ね" or "ぞ" or "ぜ" or "わ" or "さ") || (exclamatoryClip && ctx.Prev.Text.Length <= 2 && WanaKana.IsKana(ctx.Prev.Text)) - || prevShard; + || prevShard + || reduplicatedToFrame; if (!frameBefore) return false; bool nextIsQuotativeTo = ctx.Next is { Text: "と", PartOfSpeech: PartOfSpeech.Particle } && ctx.SymbolsAfter.Length > 0; - bool frameAfter = ctx.Next == null || ctx.SymbolsAfter.Length > 0 || nextShard; + bool frameAfter = ctx.Next == null || ctx.SymbolsAfter.Length > 0 || nextShard + || reduplicatedToFrame; if (!frameAfter) return false; // Require a positive mimetic signal; isolation alone describes any one-word answer (「梨」). @@ -138,6 +154,7 @@ private static bool IsSfxMimeticFragment(in MisparseGateContext ctx) || ctx.SymbolsAfter.IndexOfAny(GapMimeticChars) >= 0 || nextIsQuotativeTo || prevShard || nextShard + || reduplicated // An in-surface sokuon is ordinary orthography when the entry spells the // surface — literally (おっさん) or through the expressive-deformation check // in GetWordFlags (バカッ/ほんっと) — but phonetic evidence when it does not @@ -174,9 +191,15 @@ private static bool IsSfxMimeticFragment(in MisparseGateContext ctx) // うぜえ、と, ごくり、と, ハルさんっ): attested as written, usually-kana, or reached // through conjugation. A non-noun invented by text mutation (うぅ→うん) stays in scope // as noise. + // An entry with an adverbial sense used in the adverbial と frame is that sense, whatever + // POS happens to be listed first (ぽつぽつ [n, adv, adv-to] + と + 灯す) — first-POS alone + // must not pull an attested mimetic back into the plain-noun scope. + bool adverbialInToFrame = ctx.Next is { Text: "と", PartOfSpeech: PartOfSpeech.Particle } + && ctx.SelectedWord.PartsOfSpeech.Any(p => p is PartOfSpeech.Adverb or PartOfSpeech.AdverbTo); if ((ctx.SelectedWord.PartsOfSpeech.Count == 0 || ctx.SelectedWord.PartsOfSpeech[0] is not (PartOfSpeech.Noun - or PartOfSpeech.CommonNoun or PartOfSpeech.Numeral or PartOfSpeech.NaAdjective)) + or PartOfSpeech.CommonNoun or PartOfSpeech.Numeral or PartOfSpeech.NaAdjective) + || adverbialInToFrame) && (ctx.SurfaceAttestsListedForm || ctx.IsUsuallyKana || (ctx.SelectedWord.Conjugations.Count > 0 && ctx.SelectedWord.Conjugations[0] is not ("(stem)" or "(infinitive)" @@ -367,6 +390,23 @@ public static (bool isUsuallyKana, bool hasKanjiSpelling, bool readingIsIchi, bo bool surfaceAttestsLiterally = surfaceAttestsForm; + // Mimetic adverbs and interjections are written in either kana script interchangeably and + // JMdict lists most of them hiragana-only — a katakana spelling (スタスタと) is ordinary + // orthography for the same word, not a deformation. Restricted to the adverbial/ + // interjection class so a katakana SFX cannot claim an unrelated hiragana word's identity + // through the script fold. + if (!surfaceAttestsForm && surface is { Length: > 0 } + && word.PartsOfSpeech.Any(p => p is "adv" or "adv-to" or "int" or "on-mim") + && JapaneseTextHelper.IsAllKatakana(surface)) + { + var folded = WanaKana.ToHiragana(surface); + if (folded != surface && word.Forms.Any(f => f.Text == folded)) + { + surfaceAttestsForm = true; + surfaceAttestsLiterally = true; + } + } + // Expressive spelling deforms a word without changing it: emphatic gemination writes a // sokuon in (ほんっと, バカッ, マジッす), a chōonpu stretches a mora (おーっと), and a // trailing stretch elongates the final one (そっかー, だってぇ, なんだとぉ). Such a diff --git a/Jiten.Parser/MorphologicalAnalyser.cs b/Jiten.Parser/MorphologicalAnalyser.cs index beb7d6c5..70646546 100644 --- a/Jiten.Parser/MorphologicalAnalyser.cs +++ b/Jiten.Parser/MorphologicalAnalyser.cs @@ -16,6 +16,7 @@ public partial class MorphologicalAnalyser public Func? GetNonNameCompoundWordId { get; set; } public Func? GetNonNameCompoundFrequencyRank { get; set; } public Func? HasVerbOrAdjectiveLookup { get; set; } + public Func? HasCounterSenseLookup { get; set; } // Captured per Parse call for RetokeniseOovBlobs; an instance must not serve two parses with // different Sudachi configs concurrently (all production paths construct one instance per parse). diff --git a/Jiten.Parser/Parser.cs b/Jiten.Parser/Parser.cs index 4bb76f09..4fe2cb7f 100644 --- a/Jiten.Parser/Parser.cs +++ b/Jiten.Parser/Parser.cs @@ -71,6 +71,11 @@ public static bool RubyPriorsEnabled "ウー", "うー", "ううう", "うう", "ウウウウ", "ウウ", "ううっ", "かー", "ぐわー", "違", "タ", "ッ", "ニヒヒ" ]; + // Kanji digits that form one positional number. Narrower than JapaneseTextHelper.IsNumeralChar + // by design: the myriad markers 万/億/兆 are standalone words, so a run containing them must + // still shatter for those to survive as words. + private const string NumeralKanji = "一二三四五六七八九十百千〇零"; + // Excluded (WordId, ReadingIndex) pairs to filter from final parsing results private static readonly HashSet<(int WordId, byte ReadingIndex)> ExcludedMisparses = [ @@ -704,6 +709,20 @@ await ResolveAsNameEntry(surface, batch[j].occurrences, batchWordCache), } } + // A surface made only of kanji numerals denotes a number. JMnedict covers some + // of those surfaces as personal names (二千, 四万) — for several the only lookup — + // and a number must never resolve through a name-only entry. Myriad markers count + // here (unlike in NumeralKanji's shatter guard): 四万 is still a number. + if (result.Word != null && surface.Length >= 2 + && surface.All(c => NumeralKanji.Contains(c) || c is '万' or '億' or '兆') + && WordMeta.TryGetValue(result.Word.WordId, out var numeralNameMeta) + && numeralNameMeta.Pos.Length > 0 + && numeralNameMeta.Pos.All(p => p is PartOfSpeech.Name or PartOfSpeech.Unknown)) + { + allProcessedWords.Add((null, null, null)); + continue; + } + allProcessedWords.Add((result.Word, result.Margin, result.FirstPassCandidates)); } } @@ -741,7 +760,7 @@ public static async Task> ParseText(IDbContextFactory> ParseTextsToDeck(IDbContextFactory(); var rawCharCounts = new List(); var batchedSentences = await parser.ParseBatch(cleanTexts, diagnostics: diagnostics, timings: timings, userDictCsv: userDictCsv, cleanedOriginals: cleanedOriginals, rawContentCharCounts: rawCharCounts); @@ -1057,7 +1076,7 @@ private static async Task ProcessSentencesToDeck( { await EnsureInitializedAsync(contextFactory); - var parser = new MorphologicalAnalyser { HasCompoundLookup = HasLookupForCompound, HasNonNameCompoundLookup = HasNonNameLookup, HasPrioritizedNonNameCompoundLookup = HasPrioritizedNonNameLookup, HasKanaAppropriateCompoundLookup = HasKanaAppropriateLookup, HasSuruVerbCompoundLookup = HasSuruVerbLookup, GetNonNameCompoundWordId = GetNonNameCompoundId, GetNonNameCompoundFrequencyRank = GetBestNonNameFrequencyRank, HasVerbOrAdjectiveLookup = HasVerbOrAdjectiveLookup }; + var parser = new MorphologicalAnalyser { HasCompoundLookup = HasLookupForCompound, HasNonNameCompoundLookup = HasNonNameLookup, HasPrioritizedNonNameCompoundLookup = HasPrioritizedNonNameLookup, HasKanaAppropriateCompoundLookup = HasKanaAppropriateLookup, HasSuruVerbCompoundLookup = HasSuruVerbLookup, GetNonNameCompoundWordId = GetNonNameCompoundId, GetNonNameCompoundFrequencyRank = GetBestNonNameFrequencyRank, HasVerbOrAdjectiveLookup = HasVerbOrAdjectiveLookup, HasCounterSenseLookup = HasCounterSenseAvailable }; var sentences = await parser.Parse(text, morphemesOnly: true, diagnostics: diagnostics); var wordInfos = sentences.SelectMany(s => s.Words).Select(w => w.word).ToList(); @@ -1253,6 +1272,9 @@ private static async Task ProcessWord((WordInfo wordInfo, int wordData.wordInfo.IsKanaExclamation = wordData.wordInfo.PartOfSpeech is PartOfSpeech.Interjection or PartOfSpeech.Filler && WanaKana.IsKana(wordData.wordInfo.Text); + wordData.wordInfo.IsKatakanaNounSurface = + wordData.wordInfo.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun + && JapaneseTextHelper.IsAllKatakana(wordData.wordInfo.Text); bool isProcessed = false; int attemptCount = 0; const int maxAttempts = 3; // Limit how many attempts we make to prevent infinite loops @@ -1621,6 +1643,23 @@ static int NounVerbScore(JmDictWord word, byte readingIndex) firstPassCandidates = null; } + // A hiragana surface never denotes a loanword written only in katakana: + // mutating kana away until a gairaigo reading key bites (てりゃあ→てりあ→テリア, + // たぁっぷり→たぷり→タプル) is not a resolution either. + if (processedWord != null + && wordData.wordInfo.Text != baseWord + && IsHiraganaSurface(baseWord) + && WordMeta.TryGetValue(processedWord.WordId, out var mutatedMeta) + && mutatedMeta.Origin == WordOrigin.Gairaigo + && !(_lookups.TryGetValue(baseWord, out var gairaigoDirectIds) + && gairaigoDirectIds.Contains(processedWord.WordId))) + { + processedWord = null; + resolvedMargin = null; + firstPassCandidates = null; + } + + if (processedWord != null) { if (wordData.wordInfo.Text != baseWord) @@ -1634,10 +1673,13 @@ static int NounVerbScore(JmDictWord word, byte readingIndex) } // We haven't found a match, let's try to remove the last character if it's a っ, a ー, - // an expressive small vowel (なんちゃってぇ) or a duplicate + // an expressive small vowel (なんちゃってぇ) or a duplicate. On a pure-katakana + // surface a trailing small vowel is part of the word's identity, not stretching + // (an OOV name ソフィ must not become ソフ) — only ー/っ/duplicates strip there. if (wordData.wordInfo.Text.Length > 2 && - (wordData.wordInfo.Text[^1] is 'っ' or 'ー' or 'ぁ' or 'ぃ' or 'ぅ' or 'ぇ' or 'ぉ' - or 'ァ' or 'ィ' or 'ゥ' or 'ェ' or 'ォ' || + (wordData.wordInfo.Text[^1] is 'っ' or 'ー' or 'ぁ' or 'ぃ' or 'ぅ' or 'ぇ' or 'ぉ' || + (wordData.wordInfo.Text[^1] is 'ァ' or 'ィ' or 'ゥ' or 'ェ' or 'ォ' + && !JapaneseTextHelper.IsAllKatakana(wordData.wordInfo.Text)) || wordData.wordInfo.Text[^2] == wordData.wordInfo.Text[^1])) { wordData.wordInfo.Text = wordData.wordInfo.Text[..^1]; @@ -1647,6 +1689,17 @@ static int NounVerbScore(JmDictWord word, byte readingIndex) { wordData.wordInfo.Text = wordData.wordInfo.Text[1..]; } + // An emphatic small vowel that echoes the preceding kana's vowel row is pure + // stretching (たぁっぷり→たっぷり, ですぅ→です) — delete just those before the + // blanket strips below, which would also eat geminates and land on unrelated + // words. Foreign-mora smalls (ふぁ, うぃ) don't echo and are left intact. + // Not on pure-katakana surfaces: there the small vowel is part of the word's + // identity (ソフィ is not a stretched ソフ), not stretching. + else if (!JapaneseTextHelper.IsAllKatakana(wordData.wordInfo.Text) + && TryRemoveEchoedSmallVowels(wordData.wordInfo.Text, out var echoStripped)) + { + wordData.wordInfo.Text = echoStripped; + } // Let's try without any long vowel mark else if (wordData.wordInfo.Text.Contains('ー')) { @@ -2158,11 +2211,29 @@ private static bool NormalizedFormIntroducesKanji(string text, string normalized var deconjugated = deconjugator.Deconjugate(normalizedText) .OrderByDescending(d => d.Text.Length).ToList(); + // A katakana noun is a name/loanword shape: unwinding a conjugation from its hiragana + // conversion and landing on a kanji-primary word fabricates vocabulary out of a name + // (ハガナ → はが+な "casual request" → 剥ぐ). Kana-natural words stay reachable + // (ヤバイ → やばい), as do unconjugated spellings (identity forms carry no process). + // The flag survives the escalation chain's POS rewrites. + bool katakanaNounSurface = wordData.wordInfo.IsKatakanaNounSurface; + List<(DeconjugationForm form, List ids)> candidates = new(); foreach (var form in deconjugated) { if (_lookups.TryGetValue(form.Text, out List? lookup)) { + // A suru-noun stem (tagged n, アクシュシヨウ → あくしゅ+しよう) is the noun + // itself in styled spelling, not a fabricated conjugation — it stays. + if (katakanaNounSurface && form.Process.Count > 0 && !form.Tags.Contains("n")) + { + var kanaIds = lookup.Where(IsKanaAppropriateId).ToList(); + if (kanaIds.Count == 0) + continue; + candidates.Add((form, kanaIds)); + continue; + } + candidates.Add((form, lookup)); } } @@ -2703,6 +2774,18 @@ private static void CombineCompounds(List sentences) { bool nounTrigger = word.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun or PartOfSpeech.Name; var match = TryMatchCompounds(wordInfos, i, tokenHashes, forceExpressionOnly: nounTrigger); + + // A hard-pinned token is a gate's final word decision; a compound must not + // swallow it (分 pinned to ぶん in ４分の１ would re-fuse into ４分 "four + // minutes"). Soft pins stay absorbable — attested expressions spanning them + // (臆病風に吹かれる over the フウ pin) are the better parse. + if (match.HasValue) + { + for (int j = match.Value.startIndex; j <= i && match.HasValue; j++) + if (wordInfos[j] is { HardPinned: true, PreMatchedWordId: not null }) + match = null; + } + if (match.HasValue) { var (startIndex, dictForm, wordId) = match.Value; @@ -2784,6 +2867,33 @@ private static void CombineLexicalAdverbs(List sentences) { var (word, _, _) = words[i]; + // と+したら/すれば/すると after a non-volitional predicate is the suppositional + // conjunction (ためだとしたら). After a volitional it is とする "try to" + // (押そうとしたら) and must stay split. + if (i + 1 < words.Count + && word is { Text: "と", PartOfSpeech: PartOfSpeech.Particle } + && words[i + 1].word.Text is "したら" or "すれば" or "すると" + && words[i + 1].word.DictionaryForm is "する" or "為る" + && !(i > 0 && IsVolitionalSurface(words[i - 1].word.Text)) + && HasConjunctionEntry(word.Text + words[i + 1].word.Text)) + { + var next = words[i + 1]; + var combinedText = word.Text + next.word.Text; + var merged = new WordInfo(word) + { + Text = combinedText, + DictionaryForm = combinedText, + NormalizedForm = combinedText, + Reading = word.Reading + next.word.Reading, + PartOfSpeech = PartOfSpeech.Conjunction, + EndOffset = next.word.EndOffset, + }; + result.Add((merged, words[i].position, words[i].length + next.length)); + changed = true; + i++; + continue; + } + if (i + 1 < words.Count && word.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun or PartOfSpeech.NaAdjective or PartOfSpeech.NominalAdjective) @@ -2796,8 +2906,17 @@ private static void CombineLexicalAdverbs(List sentences) && LexicalizedAdverbPairs.Contains(word.Text + next.word.Text); bool isHodoPair = next.word.Text == "ほど" && next.word.PartOfSpeech == PartOfSpeech.Particle; - - if ((isNiPair || isCuratedPair || isHodoPair) && HasLexicalAdverbEntry(word.Text + next.word.Text)) + // Noun + merged particle-cluster attested whole as an adverbial expression. + // 先に-tails are unambiguous (一足先に); には-tails only clause-initially — + // mid-clause X+には is the genuine case/topic sequence (その時には). + bool isSakiNiPair = next.word.Text == "先に"; + bool isNiwaPair = next.word.Text == "には" + && next.word.PartOfSpeech == PartOfSpeech.Particle + && (i == 0 || words[i - 1].word.PartOfSpeech + is PartOfSpeech.Symbol or PartOfSpeech.SupplementarySymbol or PartOfSpeech.BlankSpace); + + if ((isNiPair || isCuratedPair || isHodoPair || isSakiNiPair || isNiwaPair) + && HasLexicalAdverbEntry(word.Text + next.word.Text)) { var combinedText = word.Text + next.word.Text; var merged = new WordInfo(word) @@ -2853,6 +2972,26 @@ static bool HasAdverbIds(string key) } } + private static bool HasConjunctionEntry(string text) + { + if (!_lookups.TryGetValue(text, out var ids)) return false; + foreach (var id in ids) + { + if (_nameOnlyWordIds.Contains(id)) continue; + if (WordMeta.TryGetValue(id, out var meta) && meta.Pos.Contains(PartOfSpeech.Conjunction)) + return true; + } + + return false; + } + + /// A volitional predicate surface (押そう, 行こう, 食べよう): final う preceded by an o-row kana. + private static bool IsVolitionalSurface(string text) + { + if (text.Length < 2 || text[^1] != 'う') return false; + return "おこそとのほもよろごぞどぼぽょ".IndexOf(text[^2]) >= 0; + } + private static void RepairLongVowels(List sentences) { foreach (var sentence in sentences) @@ -3194,12 +3333,14 @@ private static void FilterOrphanedMisparses(List sentences, Parser bool nextIsLongVowel = result.Count > 0 && result[^1].word.Text == "ー"; + // A pinned token is a repair stage's explicit word decision, never stutter noise + // (親ソ splits to 親 + ソ with ソ pinned to the Soviet-Union abbreviation; + // す pinned to contracted する must not be join-rescued into すんだ=済んだ). bool shouldFilter = MisparsesRemove.Contains(word.Text) && + word.PreMatchedWordId == null && !(nextIsLongVowel && word.Text.Length == 1 && WanaKana.IsKana(word.Text)) && !word.Text.EndsWith("ー"); - // A pinned token is a repair stage's explicit word decision, never stutter noise - // (親ソ splits to 親 + ソ with ソ pinned to the Soviet-Union abbreviation). bool isSingleKanaStutter = !nextIsLongVowel && word.Text.Length == 1 && WanaKana.IsKana(word.Text) && word.PreMatchedWordId == null; if (shouldFilter || @@ -3358,9 +3499,15 @@ private static void SplitUnknownNounTokens(List sentences, HashSet } } + // A compositional digit-run (四十七) has no dictionary entry but is still a + // well-formed token — shattering it lets the recombiner bleed its tail digit + // into the following counter (四十 + 七分). Myriad groups (五万, 一兆八千億) + // still shatter: their units 万/億/兆 are words, and keeping the run whole + // would drop them (numerals skip resegmentation). if (word.Text.Length >= 2 && PosMapper.IsNounForCompounding(word.PartOfSpeech) && word.Text.All(JapaneseTextHelper.IsKanji) && + !word.Text.All(c => NumeralKanji.Contains(c)) && !(_lookups.TryGetValue(word.Text, out var ids) && ids.Count > 0)) { diagnostics?.LogParserEvent( @@ -3598,6 +3745,89 @@ private static bool HasVerbOrAdjectiveLookup(string text) return false; } + /// Strips emphatic small vowels that echo the preceding kana's vowel row (たぁ→た, + /// きゃぁ→きゃ, ですぅ→です) while leaving foreign-mora smalls (ふぁ, うぃ) intact. + /// Returns true when anything was stripped. + private static bool TryRemoveEchoedSmallVowels(string text, out string result) + { + System.Text.StringBuilder? sb = null; + for (int i = 0; i < text.Length; i++) + { + char c = text[i]; + char vowel = SmallVowelKanaVowel(c); + if (i > 0 && vowel != '\0' && KanaEndsWithVowel(text[i - 1], vowel)) + { + sb ??= new System.Text.StringBuilder(text[..i]); + continue; + } + + sb?.Append(c); + } + + result = sb?.ToString() ?? text; + return sb != null; + } + + private static char SmallVowelKanaVowel(char c) => c switch + { + 'ぁ' or 'ァ' => 'a', + 'ぃ' or 'ィ' => 'i', + 'ぅ' or 'ゥ' => 'u', + 'ぇ' or 'ェ' => 'e', + 'ぉ' or 'ォ' => 'o', + _ => '\0', + }; + + private static bool KanaEndsWithVowel(char kana, char vowel) + { + var romaji = WanaKana.ToRomaji(kana.ToString()); + return romaji.Length > 0 && char.ToLowerInvariant(romaji[^1]) == vowel; + } + + /// True when the surface is hiragana material (contains hiragana, no katakana) — the + /// script a katakana-only loanword is never written in. + private static bool IsHiraganaSurface(string text) + { + bool hasHiragana = false; + foreach (var c in text) + { + if (c is >= 'ぁ' and <= 'ゖ') hasHiragana = true; + else if (c is >= 'ァ' and <= 'ヺ') return false; + } + + return hasHiragana; + } + + /// True when the surface has a JMDict entry whose PRIMARY POS is a counter (話, 羽, 体, 回 …). + /// Primary only — a minor counter sense on a pronoun/noun entry (何, 差し) must not make the + /// token counter-rescorable after numerals. + private static bool HasCounterLookup(string text) + { + if (!_lookups.TryGetValue(text, out var ids)) return false; + foreach (var id in ids) + if (WordMeta.TryGetValue(id, out var meta) && meta.GetPrimaryPos() == PartOfSpeech.Counter) + return true; + return false; + } + + /// True when the surface has a JMDict entry with a counter sense anywhere in its POS list + /// (度, 発 — counter behind the plain noun; 回, 話 — counter-primary). Looser than + /// HasCounterLookup: re-cutting a stolen mora after a numeral only needs the sense to + /// exist, while counter rescoring needs it to be the entry's identity. + private static bool HasCounterSenseAvailable(string text) + { + if (!_lookups.TryGetValue(text, out var ids)) return false; + foreach (var id in ids) + { + if (!WordMeta.TryGetValue(id, out var meta)) continue; + foreach (var p in meta.Pos) + if (p == PartOfSpeech.Counter) + return true; + } + + return false; + } + /// A word id is "kana-appropriate" when matching it from a pure-kana surface is plausible: /// the word is usually written in kana (uk), or it has no kanji written form at all. /// Kanji-backed non-uk words (配送, 童男, 遺影) reject — their kana lookup keys are readings, @@ -3726,13 +3956,17 @@ PartOfSpeech.Suffix or PartOfSpeech.NounSuffix or PartOfSpeech.Counter or continue; } - // Strategy 1a: Trailing particle strip (with lookup check on remainder) + // Strategy 1a: Trailing particle strip (with lookup check on remainder). + // A remainder whose only matches are name entries doesn't count as known: + // stripping a "particle" off an OOV noun must reveal vocabulary, not fabricate + // a name plus a stray mora (ブリタニカ is not ブリタニ+か). char lastChar = textHira[^1]; bool hasTrailingParticle = textHira.Length >= 3 && TrailingParticles.Contains(lastChar); - bool remainderKnown = hasTrailingParticle && (HasLookup(text[..^1]) || - (textHira.Length >= 5 && - Deconjugator.Instance.Deconjugate(textHira[..^1]) - .Any(f => HasLookup(f.Text)))); + bool remainderKnown = hasTrailingParticle && + (HasNonNameLookup(text[..^1]) || + (textHira.Length >= 5 && + Deconjugator.Instance.Deconjugate(textHira[..^1]) + .Any(f => HasNonNameLookup(f.Text)))); if (remainderKnown) { var remainder = text[..^1]; @@ -3795,7 +4029,8 @@ private static void CombineNounCompounds(List sentences) word.Text.Any(JapaneseTextHelper.IsKanji); if ((word.PartOfSpeech is PartOfSpeech.IAdjective or PartOfSpeech.Verb || kanjiAdverbLead || misparsedKanjiLead) && i + 1 < sentence.Words.Count && - PosMapper.IsNounForCompounding(sentence.Words[i + 1].word.PartOfSpeech)) + PosMapper.IsNounForCompounding(sentence.Words[i + 1].word.PartOfSpeech) && + !word.HardPinned && !sentence.Words[i + 1].word.HardPinned) { var combinedText = word.Text + sentence.Words[i + 1].word.Text; if (_lookups.TryGetValue(combinedText, out var adjSuffixIds) && adjSuffixIds.Count > 0 && @@ -3843,10 +4078,10 @@ private static void CombineNounCompounds(List sentences) break; } - // A 目 pinned as the ordinal suffix (三つ目/四つ目 → number + 目-th) must not be - // re-absorbed into the noun 三つ目 "three-eyed being". (一つ目/二つ目 keep their - // own ordinal entries and are never pinned, so they don't reach here.) - if (j > 0 && w.PreMatchedWordId == 1604890) + // A hard-pinned token is a gate's final word decision (三つ目's + // ordinal 目, the ぶん of a fraction frame) and must not be re-absorbed + // into a compound; soft pins stay absorbable. + if (w is { HardPinned: true, PreMatchedWordId: not null }) { allValid = false; break; @@ -4651,6 +4886,13 @@ MisparseDecision EvaluateAt(int i, DeckWord deckWord) { var token = flatTokens[i]; + // A rewrite-rule pin is a deliberate lexical decision, never burst/SFX material — + // a reassembled contraction (え|っつった) must survive its own っ-marks. Scoped to + // the explicit flag: PreMatchedWordId alone is reused by parser-level machinery + // for ordinary tokens, and exempting it wholesale resurrects gated junk (ぐすっ→具す). + if (token.PinnedByRewriteRule && token.PreMatchedWordId != null) + return default; + // Skip the flag/neighbour/blob assembly outright for tokens no gate can judge. if (!MisparseGates.MayBeKanaFragment(token.Text) && (deckWord.OriginalText.Length == 0 @@ -5134,10 +5376,24 @@ private static (int startIndex, string dictionaryForm, int wordId)? TryMatchComp if (_lookupsAlt.TryGetValue(candidateSpan, out var wordIds) && wordIds.Count > 0) { - bool isKana = true; + // A span made only of kanji numerals is a number; several such surfaces exist + // in JMnedict as personal names only (四万, 二十三) — never join number tokens + // through a name-only entry. + bool numeralSpan = true; foreach (var c in candidateSpan) - if (c is not ((>= 'ぁ' and <= 'ゟ') or (>= '゠' and <= 'ヿ'))) { isKana = false; break; } - (exprWordId, compWordId) = CompoundWordSelector.FindCompoundWordIds(wordIds, WordMeta, isKana); + if (!NumeralKanji.Contains(c) && c is not ('万' or '億' or '兆')) { numeralSpan = false; break; } + if (numeralSpan) + wordIds = wordIds.Where(id => !(WordMeta.TryGetValue(id, out var m) + && m.Pos.Length > 0 + && m.Pos.All(p => p is PartOfSpeech.Name or PartOfSpeech.Unknown))) + .ToList(); + if (wordIds.Count > 0) + { + bool isKana = true; + foreach (var c in candidateSpan) + if (c is not ((>= 'ぁ' and <= 'ゟ') or (>= '゠' and <= 'ヿ'))) { isKana = false; break; } + (exprWordId, compWordId) = CompoundWordSelector.FindCompoundWordIds(wordIds, WordMeta, isKana); + } } if (exprWordId == null && compWordId == null) @@ -5353,6 +5609,16 @@ private static async Task> ApplyAdjacentScoringCore( bool nextIsCopula = nextInfo != null && TransitionRuleSets.CopulaForms.Contains(nextInfo.Text); bool nextIsForwardAnchor = nextInfo != null && TransitionRuleSets.ForwardAnchorSurfaces.Contains(nextInfo.Text); + // A token with a counter homograph right after numeric material (第二|話) must stay + // rescorable even on a confident first pass — only pass 2's numeral-counter + // cohesion can see the numeral context (same shape as the forward anchors). + // A pick that is itself a numeral (ガンつく1|ワン) is already the cohesive reading + // and must not be reshuffled by the general synergies. + bool prevNumericCounter = i > 0 + && AdjacentWordScorer.IsNumericSurface(sentenceWords[i - 1].word.Text) + && HasCounterLookup(currentInfo.Text) + && !currentResult.PartsOfSpeech.Contains(PartOfSpeech.Numeral); + var cachedHint = FindMatchingHint(relocatedHints, currentInfo); bool hasHint = cachedHint != null; bool isPreMatched = currentInfo.PreMatchedWordId.HasValue && currentInfo.PreMatchedCandidateWordIds == null; @@ -5377,10 +5643,10 @@ private static async Task> ApplyAdjacentScoringCore( bool sentenceFinalVerb = i == sentenceWords.Count - 1 && currentResult.PartsOfSpeech.Any(p => p is PartOfSpeech.Verb); - if (!isArchaicPass1 && !nextIsCopula && !nextIsForwardAnchor && !hasHint && !infinitiveAfterNominal && !sentenceFinalVerb + if (!isArchaicPass1 && !nextIsCopula && !nextIsForwardAnchor && !prevNumericCounter && !hasHint && !infinitiveAfterNominal && !sentenceFinalVerb && ScoringPolicy.IsHighConfidence(currentMargin)) continue; - bool forceRederive = hasHint || nextIsForwardAnchor || ScoringPolicy.IsLowConfidence(currentMargin); + bool forceRederive = hasHint || nextIsForwardAnchor || prevNumericCounter || ScoringPolicy.IsLowConfidence(currentMargin); if (!forceRederive) for (int k = Math.Max(0, i - 3); k < i; k++) @@ -5472,7 +5738,14 @@ private static async Task> ApplyAdjacentScoringCore( } else if (rederiveStates.TryGetValue((si, i), out var state)) { - candidates = RederivationHelper.BuildCandidatesFromWords(state, wordCache, skipPosFilter: tokenHasHint); + // Kanji surfaces only: a kana counter homograph after numeric-like material + // is usually something else entirely (何|か = question particle, not 箇/課). + bool admitCounters = i > 0 + && AdjacentWordScorer.IsNumericSurface(sentenceWords[i - 1].word.Text) + && currentInfo.Text.Any(JapaneseTextHelper.IsKanji) + && HasCounterLookup(currentInfo.Text); + candidates = RederivationHelper.BuildCandidatesFromWords(state, wordCache, skipPosFilter: tokenHasHint, + allowCounterCandidates: admitCounters); } // A kana surface can't resolve to a kanji-only homograph (kana なくなる ≠ 亡くなる). diff --git a/Jiten.Parser/Resegmentation/ResegmentationEngine.cs b/Jiten.Parser/Resegmentation/ResegmentationEngine.cs index d9f9e74b..6ba1ed45 100644 --- a/Jiten.Parser/Resegmentation/ResegmentationEngine.cs +++ b/Jiten.Parser/Resegmentation/ResegmentationEngine.cs @@ -34,7 +34,7 @@ public static void TryImproveUncertainSpans( SpanPath? path; if (isCompoundNumeral) { - path = TrySplitCompoundNumeral(span.Text, lookups); + path = TrySplitCompoundNumeral(span.Text, lookups, wordMeta); if (path == null) continue; } @@ -55,6 +55,8 @@ _ when IsKatakanaConjugatedWordSpelling(span.Text, lookups, wordMeta) => "katakana conjugated-word spelling", _ when HasSuruConjugationTail(path, span.Text) => "suru-conjugation tail", _ when HasShortPureNameSegment(path, wordMeta) => "short pure-name segment", + _ when HasNameSegmentOutsidePersonContext(path, span, sentence, wordMeta) => + "name segment outside person context", _ when ResegmentationScorer.ScorePath(path, frequencyRanks, span.Text) < 0 => "negative path score", _ => null }; @@ -185,7 +187,12 @@ public static bool TryResegmentLowConfidenceTokens( // Splits compound kanji numerals at the last place marker (十/百/千/万/億/兆). // E.g. 五十七 → 五十+七, 三十八 → 三十+八, 六十一 → 六十+一. - private static SpanPath? TrySplitCompoundNumeral(string text, Dictionary> lookups) + // Pieces must be attested as non-name words: JMnedict covers some numeral surfaces as + // personal names (二十三, 五十六), and a number must never resolve through those. A piece + // attested only as a name splits recursively instead (二十三 → 二十+三), so the run still + // lands on real number words. + private static SpanPath? TrySplitCompoundNumeral(string text, Dictionary> lookups, + Dictionary wordMeta) { for (int i = text.Length - 1; i >= 1; i--) { @@ -195,20 +202,42 @@ public static bool TryResegmentLowConfidenceTokens( var left = text[..i]; var right = text[i..]; - if (!lookups.TryGetValue(left, out var leftIds) || leftIds.Count == 0) - continue; - if (!lookups.TryGetValue(right, out var rightIds) || rightIds.Count == 0) + var leftIds = NonNameLookupIds(left, lookups, wordMeta); + if (leftIds == null) continue; - return new SpanPath([ - new SpanTokenCandidate(0, left.Length, leftIds), - new SpanTokenCandidate(i, right.Length, rightIds) - ]); + var rightIds = NonNameLookupIds(right, lookups, wordMeta); + if (rightIds != null) + return new SpanPath([ + new SpanTokenCandidate(0, left.Length, leftIds), + new SpanTokenCandidate(i, right.Length, rightIds) + ]); + + var sub = TrySplitCompoundNumeral(right, lookups, wordMeta); + if (sub != null) + return new SpanPath([ + new SpanTokenCandidate(0, left.Length, leftIds), + .. sub.Segments.Select(s => new SpanTokenCandidate(i + s.StartChar, s.Length, s.WordIds)), + ]); } return null; } + // Name-band means every POS is Name/Unknown — this deliberately includes JMnedict "unclass" + // entries (五十五 "Isoi"), which IsTrueName excludes: on a numeral surface an unclass entry is + // never legitimate vocabulary either. + private static List? NonNameLookupIds(string text, Dictionary> lookups, + Dictionary wordMeta) + { + if (!lookups.TryGetValue(text, out var ids) || ids.Count == 0) + return null; + var filtered = ids.Where(id => !wordMeta.TryGetValue(id, out var meta) + || meta.Pos.Length == 0 + || !meta.Pos.All(p => p is PartOfSpeech.Name or PartOfSpeech.Unknown)).ToList(); + return filtered.Count > 0 ? filtered : null; + } + // A short segment whose every candidate is a pure JMnedict name entry is not evidence of a // word boundary — it shreds OOV names into coincidental fragments (ファルマ → ファ+ルマ "Ruma"). // Long pure-name segments stay allowed: fused name sequences legitimately resegment through @@ -239,6 +268,62 @@ private static bool HasShortPureNameSegment(SpanPath path, Dictionary NameContextSuffixes = + [ + .. Grammar.TransitionRuleSets.HonorificSuffixes, + "さま", "君", "どの", "先生", "先輩", "嬢", "卿", + "市", "町", "村", "区", "郡", "県", "府", "都", "駅", "山", "川", "島", "湖", "港", "橋", + "城", "寺", "神社", "線", "地方", "出身", "産", "語", + ]; + + private static bool HasNameSegmentOutsidePersonContext( + SpanPath path, UncertainSpan span, SentenceInfo sentence, Dictionary wordMeta) + { + // A name-only span being re-split into constituent names is the intended outcome + // of the name respray, whatever its context. + if (span.NameOnly) return false; + + var word = sentence.Words[span.WordIndex].word; + if (word.IsPersonNameContext) return false; + + // A katakana-styled particle segment marks a styled sentence (ボクノナマエハ) — those + // spans legitimately mix vocabulary with name-looking fragments. + foreach (var s in path.Segments) + if (s.Length == 1 && ResegmentationScorer.IsKatakanaParticleChar(span.Text[s.StartChar])) + return false; + + for (int j = 0; j < path.Segments.Count; j++) + { + var s = path.Segments[j]; + bool anyName = false, allName = true; + foreach (var id in s.WordIds) + { + if (!wordMeta.TryGetValue(id, out var meta)) continue; + if (meta.IsTrueName) anyName = true; + else { allName = false; break; } + } + + if (!anyName || !allName) continue; + + // A name segment is licensed by the surface that follows it — the next segment + // in the path, or for a span-final segment the next token in the sentence. + string? following = j + 1 < path.Segments.Count + ? span.Text.Substring(path.Segments[j + 1].StartChar, path.Segments[j + 1].Length) + : span.WordIndex + 1 < sentence.Words.Count + ? sentence.Words[span.WordIndex + 1].word.Text + : null; + if (following == null || !NameContextSuffixes.Contains(following)) + return true; + } + + return false; + } + // Single-kana segments are noise matches (high-frequency entries like 部/リ/ン win on // frequency score and shred OOV katakana names, e.g. ゴブリンスレイヤー → ゴ+ブ+リ+ン+スレイヤー). // Exceptions: honorific お/ご at the start, and katakana-styled particles anywhere diff --git a/Jiten.Parser/Resegmentation/ResegmentationScorer.cs b/Jiten.Parser/Resegmentation/ResegmentationScorer.cs index 91480072..aa729562 100644 --- a/Jiten.Parser/Resegmentation/ResegmentationScorer.cs +++ b/Jiten.Parser/Resegmentation/ResegmentationScorer.cs @@ -16,6 +16,8 @@ internal static class ResegmentationScorer internal static bool IsKatakanaParticleChar(char c) => c is 'ガ' or 'ヲ' or 'ニ' or 'ハ' or 'ヘ' or 'デ' or 'ト' or 'モ' or 'カ' or 'ノ'; + private static bool IsKatakanaRunChar(char c) => JapaneseTextHelper.IsKatakanaWordChar(c); + public static List BuildEdges( string spanText, int startPos, @@ -128,8 +130,17 @@ or PartOfSpeech.Counter or PartOfSpeech.Numeral or PartOfSpeech.IAdjective if (filterKatakanaFragments) edges.RemoveAll(e => e.Length <= 2 && e.Length < spanText.Length && !(e.Length == 1 && IsKatakanaParticleChar(spanText[e.StartChar])) + && !(e.Length == 1 && pos == 0 && spanText[0] == 'ド') && !IsPlausibleSegment(e, wordMeta!, pos == 0, pos + e.Length == spanText.Length)); + // In a mixed span, a cut between two katakana characters is never a real word + // boundary: ポンコツ車 splits ポンコツ|車, not ポン|コツ|車; クオンツが splits + // クオンツ|が. Pure-katakana spans keep the plausibility filter above instead + // (テラバイト|ディスク is a legitimate intra-katakana split). + if (!JapaneseTextHelper.IsAllKatakana(spanText)) + edges.RemoveAll(e => pos + e.Length < spanText.Length + && IsKatakanaRunChar(spanText[pos + e.Length - 1]) + && IsKatakanaRunChar(spanText[pos + e.Length])); if (debug) Console.WriteLine($"[reseg] '{spanText}' pos={pos} states={states.Count} edges: " + string.Join(", ", edges.Select(e => $"{spanText.Substring(e.StartChar, e.Length)}({e.WordIds.Count})"))); @@ -214,6 +225,20 @@ or PartOfSpeech.Counter or PartOfSpeech.Numeral or PartOfSpeech.IAdjective if (validStates.Count == 0) validStates = completeStates; + // A long pure-katakana span shredding into three-plus short fragments (コーポ|レイ|テッド) is + // an OOV loanword, not a compound of real words — no path is better than a junk path. Two-piece + // splits stay (a two-name sequence, or a real katakana compound テラバイト|ディスク), as do + // katakana-styled sentences carrying particle segments. + if (filterKatakanaFragments && spanText.Length >= 5) + { + validStates.RemoveAll(s => s.segs.Count >= 3 + && s.segs.All(seg => seg.Length < 4) + && !s.segs.Any(seg => seg.Length == 1 + && IsKatakanaParticleChar(spanText[seg.StartChar]))); + if (validStates.Count == 0) + return null; + } + if (frequencyRanks != null) { // Use full ScorePath for final selection — adds structural bonuses not tracked in partial score. diff --git a/Jiten.Parser/Resolution/RederivationHelper.cs b/Jiten.Parser/Resolution/RederivationHelper.cs index 1545882f..6906b45d 100644 --- a/Jiten.Parser/Resolution/RederivationHelper.cs +++ b/Jiten.Parser/Resolution/RederivationHelper.cs @@ -88,7 +88,8 @@ internal sealed class RederiveState public static List BuildCandidatesFromWords( RederiveState state, Dictionary wordCache, - bool skipPosFilter = false) + bool skipPosFilter = false, + bool allowCounterCandidates = false) { var allCandidates = new List(); @@ -126,8 +127,12 @@ public static List BuildCandidatesFromWords( if (!skipPosFilter) { bool isNameWord = word.CachedPOS.All(p => p is PartOfSpeech.Name or PartOfSpeech.Unknown); + // Counter (ctr) entries are never POS-compatible with Sudachi Suffix (the check is + // context-free); the caller sets allowCounterCandidates after numeric material so + // numeral-counter cohesion can select them (第二|話 → わ). if (!PosMapper.IsJmDictCompatibleWithSudachi(word.CachedPOS, state.WordInfo.PartOfSpeech) - && !(state.WordInfo.IsPersonNameContext && isNameWord)) + && !(state.WordInfo.IsPersonNameContext && isNameWord) + && !(allowCounterCandidates && word.CachedPOS.Contains(PartOfSpeech.Counter))) continue; } diff --git a/Jiten.Parser/Scoring/AdjacentWordScorer.cs b/Jiten.Parser/Scoring/AdjacentWordScorer.cs index 5447cb32..cd9871cf 100644 --- a/Jiten.Parser/Scoring/AdjacentWordScorer.cs +++ b/Jiten.Parser/Scoring/AdjacentWordScorer.cs @@ -23,6 +23,12 @@ public static AdjacentContext Create( uint nextMask = nextPOS != null ? PosMask.FromList(nextPOS) : 0; bool hasNext = nextPOS != null; + // JMDict tags ordinals and number compounds as plain nouns (第二 [n], 百八 [n]), so the + // resolved-POS mask loses their numeral-ness — but a counter reading after them behaves + // exactly as after a bare numeral (第二話 = だいにわ). Restore the bit from the surface. + if (hasPrev && IsNumericSurface(prevText)) + prevMask |= PosMask.Numeral; + // Soft-rule ContextMatch depends only on context, so compute the applicable-rule set once // per token here rather than re-evaluating it for every candidate in EvaluateSoftRules. ulong applicable = TransitionRuleEngine.ComputeContextApplicableMask( @@ -32,6 +38,21 @@ public static AdjacentContext Create( } } + // Numeric surface material: kanji/ASCII/full-width digits, optionally opened by the ordinal + // prefix 第 or the quantity interrogative/approximator 何・数 (何話, 数分). + internal static bool IsNumericSurface(string? text) + { + if (string.IsNullOrEmpty(text)) return false; + int i = text[0] is '第' or '何' or '数' ? 1 : 0; + if (i == text.Length) return text[0] is '何' or '数'; + for (; i < text.Length; i++) + { + if (!Jiten.Core.JapaneseTextHelper.IsNumeralChar(text[i])) return false; + } + + return true; + } + internal static int CalculateContextBonus( FormCandidate candidate, AdjacentContext context, diff --git a/Jiten.Parser/Stages/MorphologicalAnalyser.CombineStages.cs b/Jiten.Parser/Stages/MorphologicalAnalyser.CombineStages.cs index 5a9eea91..9032de23 100644 --- a/Jiten.Parser/Stages/MorphologicalAnalyser.CombineStages.cs +++ b/Jiten.Parser/Stages/MorphologicalAnalyser.CombineStages.cs @@ -423,7 +423,13 @@ private List CombinePrefixes(List wordInfos) { var currentWord = new WordInfo(wordInfos[i]); - if (currentWord.PartOfSpeech == PartOfSpeech.Prefix && i + 1 < wordInfos.Count) + // The emphatic prefix ど is tagged Adverb (truncated どう) by Sudachi; before an + // i-adjective it is the intensifier (ど偉い, どでかい) — the attested-compound guards + // below decide whether a real compound exists. + bool isEmphaticDo = currentWord.Text == "ど" && currentWord.PartOfSpeech == PartOfSpeech.Adverb + && i + 1 < wordInfos.Count && wordInfos[i + 1].PartOfSpeech == PartOfSpeech.IAdjective; + + if ((currentWord.PartOfSpeech == PartOfSpeech.Prefix || isEmphaticDo) && i + 1 < wordInfos.Count) { var nextWord = wordInfos[i + 1]; bool isKanjiPrefix = IsKanjiPrefix(currentWord.Text); @@ -432,7 +438,7 @@ private List CombinePrefixes(List wordInfos) // Kana prefixes (お, ご) should only combine with nouns/NaAdjectives bool isContentWord = nextWord.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.NaAdjective or PartOfSpeech.Adverb or PartOfSpeech.NominalAdjective or PartOfSpeech.CommonNoun - || (isKanjiPrefix && nextWord.PartOfSpeech is PartOfSpeech.Verb or PartOfSpeech.IAdjective); + || ((isKanjiPrefix || isEmphaticDo) && nextWord.PartOfSpeech is PartOfSpeech.Verb or PartOfSpeech.IAdjective); if (isContentWord) { @@ -504,6 +510,9 @@ or PartOfSpeech.Adverb or PartOfSpeech.NominalAdjective or PartOfSpeech.CommonNo // Try partial combination: prefix + beginning of next token // Only when the next token itself is NOT a valid word (Sudachi drew wrong boundaries) // e.g. 相+当腹 → 相当+腹 (当腹 is not a valid word, so Sudachi mis-segmented) + // The remainder must be a word too: a re-cut that strands a multi-char unattested + // blob is not a boundary repair (おバ[小母] + junk). A single stray kana is fine — + // the stutter filter cleans it (お+にぃ → おに + ぃ). if (nextWord.Text.Length >= 2 && !PrefixCombineExclusions.Contains(combinedText) && !HasCompoundLookup(nextWord.Text)) @@ -513,7 +522,9 @@ or PartOfSpeech.Adverb or PartOfSpeech.NominalAdjective or PartOfSpeech.CommonNo { var partialText = currentWord.Text + nextWord.Text[..len]; if (!PrefixCombineExclusions.Contains(partialText) && - HasCompoundLookup(partialText)) + HasCompoundLookup(partialText) && + (HasCompoundLookup(nextWord.Text[len..]) + || (nextWord.Text.Length - len == 1 && JapaneseTextHelper.IsKana(nextWord.Text[len])))) { var combinedWord = new WordInfo(nextWord); combinedWord.Text = partialText; @@ -777,6 +788,11 @@ currentWord.Text is "て" or "で" or "ちゃ" or "ば" && return newList; } + // The quote-taking verbs that mark a preceding って as quotative — the same set the + // ShouldStopMerging re-cut uses (kanji and kana lemmas both: Sudachi tags 言う as いう freely). + private static bool IsQuoteTakingVerb(WordInfo w) => + w.DictionaryForm is "思う" or "おもう" or "言う" or "いう" or "聞く" or "きく" or "考える" or "感じる"; + private List CombineAuxiliary(List wordInfos) { if (wordInfos.Count < 2) @@ -849,6 +865,13 @@ or PartOfSpeech.Auxiliary && currentWord.Text != "やしない" && currentWord.Text != "し" && !(currentWord.Text == "って" && previousWord.IsImperative) + // A って-final copula before a quote-taking verb carries the quotative, not an + // inflection: 大袈裟|だって|言いたい is 大袈裟だ + って + 言いたい, never the + // 大袈裟だった-style fold. Only the copula だ — a volitional ようって must fold + // (来ようって|いう) and gets re-cut by CombineQuotativeToIu afterwards. + && !(currentWord.DictionaryForm == "だ" + && currentWord.Text.EndsWith("って", StringComparison.Ordinal) + && i + 1 < wordInfos.Count && IsQuoteTakingVerb(wordInfos[i + 1])) && currentWord.Text != "なのだ" && !currentWord.Text.StartsWith("なん") && currentWord.Text != "だろ" @@ -1084,11 +1107,14 @@ private List CombineParticles(List wordInfos) { WordInfo currentWord = wordInfos[i]; - // Combine かもしれ* (kamoshirenai, kamoshiremasen, etc.) into single expression + // Combine かもしれ* (kamoshirenai, kamoshiremasen, etc.) into single expression. + // The ん-contracted しんない/しんねえ is the same expression — the deconjugator's + // しんない ending-rule recovers かもしれない from the fused tail. if (i + 2 < wordInfos.Count && currentWord.Text == "か" && wordInfos[i + 1].Text == "も" && - wordInfos[i + 2].Text.StartsWith("しれ")) + (wordInfos[i + 2].Text.StartsWith("しれ") || + wordInfos[i + 2].Text.StartsWith("しんな") || wordInfos[i + 2].Text.StartsWith("しんね"))) { WordInfo combinedWord = new WordInfo(currentWord); combinedWord.Text = currentWord.Text + wordInfos[i + 1].Text + wordInfos[i + 2].Text; @@ -1101,6 +1127,26 @@ private List CombineParticles(List wordInfos) continue; } + // Same expression when か+も has already been fused into かも upstream (the polite + // かもしれません and the plain かもしれない both reach here as [かも][しれ*] once the + // ん-repair has glued the しれ* tail): join the remaining しれ* token. + if (i + 1 < wordInfos.Count && + currentWord.Text == "かも" && + (wordInfos[i + 1].Text.StartsWith("しれ", StringComparison.Ordinal) || + wordInfos[i + 1].Text.StartsWith("しんな", StringComparison.Ordinal) || + wordInfos[i + 1].Text.StartsWith("しんね", StringComparison.Ordinal))) + { + WordInfo combinedWord = new WordInfo(currentWord); + combinedWord.Text = currentWord.Text + wordInfos[i + 1].Text; + combinedWord.EndOffset = wordInfos[i + 1].EndOffset; + combinedWord.Reading = currentWord.Reading + wordInfos[i + 1].Reading; + combinedWord.PartOfSpeech = PartOfSpeech.Expression; + newList.Add(combinedWord); + i += 2; + changed = true; + continue; + } + // A fused じゃない / ではない token directly followed by か → じゃないか expression. Sudachi usually // splits では|ない|か (joined below), but when ない is already glued into a single じゃない token // the か is left stranded; rejoin it so じゃないか stays one unit. Only the copula-negative diff --git a/Jiten.Parser/Stages/MorphologicalAnalyser.Disambiguation.cs b/Jiten.Parser/Stages/MorphologicalAnalyser.Disambiguation.cs index 5889f45b..15a17f95 100644 --- a/Jiten.Parser/Stages/MorphologicalAnalyser.Disambiguation.cs +++ b/Jiten.Parser/Stages/MorphologicalAnalyser.Disambiguation.cs @@ -6,6 +6,54 @@ namespace Jiten.Parser; public partial class MorphologicalAnalyser { + // Hours of the clock: surface → (entry, clock reading). The clock forces native readings for + // two digits (四時=よじ, 九時=くじ), which Sudachi's per-token onyomi can't produce. + private static readonly Dictionary HourPins = new() + { + // 一時 is deliberately absent: いちじ/いっとき/ひととき share the surface and the numeral + // context can't separate them — baseline scoring already resolves the o'clock uses. + ["二時"] = (2612170, "ニジ"), + ["三時"] = (1300520, "サンジ"), + ["四時"] = (1307230, "ヨジ"), + ["五時"] = (2845367, "ゴジ"), + ["六時"] = (2583620, "ロクジ"), + ["七時"] = (2845363, "シチジ"), + ["八時"] = (2845368, "ハチジ"), + ["九時"] = (2845349, "クジ"), + ["十時"] = (2845369, "ジュウジ"), + ["十一時"] = (2845370, "ジュウイチジ"), + ["十二時"] = (1334960, "ジュウニジ"), + ["零時"] = (1557690, "レイジ"), + }; + + // Surface → duration-reading pin for fused numeral+分/日/月 homographs; applied only before + // a temporal anchor (see the anchored gate in FixReadingAmbiguity). + private static readonly Dictionary TemporalDurationPins = new() + { + ["一分"] = (1166290, 1), + ["１分"] = (1166290, 0), + ["十分"] = (1335070, 1), + ["１０分"] = (1335070, 0), + ["五分"] = (2039350, 1), + ["５分"] = (2039350, 0), + ["二分"] = (2219180, 1), + ["２分"] = (2219180, 0), + ["三分"] = (1814040, 1), + ["３分"] = (1814040, 0), + ["四分"] = (2863218, 1), + ["４分"] = (2863218, 0), + ["六分"] = (2056150, 1), + ["６分"] = (2056150, 0), + ["七分"] = (2864067, 1), + ["７分"] = (2864067, 0), + ["何分"] = (1189320, 0), + ["三十日"] = (1300670, 1), + ["一月"] = (1162130, 0), + }; + + private static readonly HashSet TemporalAnchorTexts = + ["前", "後", "間", "ぐらい", "くらい", "ほど", "経つ", "経った", "経って", "待っ"]; + private List FilterMisparse(List wordInfos) { for (int i = wordInfos.Count - 1; i >= 0; i--) @@ -256,14 +304,18 @@ private List FilterMisparse(List wordInfos) word.PreMatchedWordId = 1448820; } - // ばっか directly after a te-form is the ばかり contraction (してばっか "nothing but - // doing"), never a noun — a predicative 馬鹿 needs its own clause slot. Pinned because - // the surface is also a listed kana form of unrelated kanji words (麦価/幕下), whose - // exact-form matches otherwise compete with the particle depending on neighbour scoring. + // ばっか directly after a te-form or a bare noun is the ばかり contraction (してばっか + // "nothing but doing", 仕事ばっか "nothing but work"), never a noun — a predicative 馬鹿 + // needs its own clause slot, which puts a topic particle in between (あんたはばっか). + // Pinned because the surface is also a listed kana form of unrelated kanji words + // (麦価/幕下), whose exact-form matches otherwise compete with the particle depending + // on neighbour scoring. if (word is { Text: "ばっか", PreMatchedWordId: null } && i > 0 && ((wordInfos[i - 1] is { PartOfSpeech: PartOfSpeech.Verb } vprev && (vprev.Text.EndsWith("て") || vprev.Text.EndsWith("で"))) - || wordInfos[i - 1] is { PartOfSpeech: PartOfSpeech.Particle, Text: "て" or "で" })) + || wordInfos[i - 1] is { PartOfSpeech: PartOfSpeech.Particle, Text: "て" or "で" } + || wordInfos[i - 1].PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun + or PartOfSpeech.Pronoun or PartOfSpeech.Name)) { word.PreMatchedWordId = 2857403; word.DictionaryForm = "ばっか"; @@ -311,9 +363,14 @@ private List FilterMisparse(List wordInfos) word.EndOffset = shoEnd >= 0 ? shoEnd - 1 : -1; wordInfos.Insert(i + 1, new WordInfo { - Text = "に", DictionaryForm = "に", NormalizedForm = "に", Reading = "ニ", - PartOfSpeech = PartOfSpeech.Particle, PartOfSpeechSection1 = PartOfSpeechSection.CaseMarkingParticle, - StartOffset = shoEnd >= 0 ? shoEnd - 1 : -1, EndOffset = shoEnd + Text = "に", + DictionaryForm = "に", + NormalizedForm = "に", + Reading = "ニ", + PartOfSpeech = PartOfSpeech.Particle, + PartOfSpeechSection1 = PartOfSpeechSection.CaseMarkingParticle, + StartOffset = shoEnd >= 0 ? shoEnd - 1 : -1, + EndOffset = shoEnd }); } @@ -434,9 +491,14 @@ private List FilterMisparse(List wordInfos) int mid = word.EndOffset >= 0 ? word.EndOffset - 2 : -1; var tte = new WordInfo(word) { - Text = "って", DictionaryForm = "って", NormalizedForm = "って", Reading = "ッテ", - PartOfSpeech = PartOfSpeech.Particle, PreMatchedWordId = 2086960, - StartOffset = mid, EndOffset = word.EndOffset + Text = "って", + DictionaryForm = "って", + NormalizedForm = "って", + Reading = "ッテ", + PartOfSpeech = PartOfSpeech.Particle, + PreMatchedWordId = 2086960, + StartOffset = mid, + EndOffset = word.EndOffset }; word.Text = word.Text[..^2]; word.EndOffset = mid; @@ -480,12 +542,24 @@ private List FilterMisparse(List wordInfos) word.HasPartOfSpeechSection(PartOfSpeechSection.Counter)) word.PartOfSpeech = PartOfSpeech.Counter; - // 人 after a numeral should be the counter にん, not the suffix じん + // 人 after a numeral should be the counter にん, not the suffix じん. Still needed even + // with the Suffix↔ctr-primary compatibility in place: for ６５億人-shapes both じん (suf) + // and にん (ctr) are POS-compatible with the Suffix tag, and scoring alone picks じん — + // only the POS reclassify restricts the field to the counter. if (word is { Text: "人", PartOfSpeech: PartOfSpeech.Suffix } && i > 0 && (wordInfos[i - 1].PartOfSpeech == PartOfSpeech.Numeral || wordInfos[i - 1].HasPartOfSpeechSection(PartOfSpeechSection.Numeral))) word.PartOfSpeech = PartOfSpeech.Counter; + // 度 after a numeral is the degree/occurrence counter ど (1445160), never the + // standalone times-entry たび (１０度 = "10 degrees", not "the 10th time"). + if (word is { Text: "度", PreMatchedWordId: null } && + i > 0 && AdjacentWordScorer.IsNumericSurface(wordInfos[i - 1].Text)) + { + word.PreMatchedWordId = 1445160; + word.PreMatchedReadingIndex = 0; + } + // 家 followed by a case particle should be the noun いえ, not the suffix け if (word is { Text: "家", PartOfSpeech: PartOfSpeech.Suffix } && i + 1 < wordInfos.Count && @@ -564,6 +638,7 @@ static bool IsNumeral(WordInfo w) => word.NormalizedForm = "目"; word.PreMatchedWordId = 1604890; word.PreMatchedReadingIndex = 0; + word.HardPinned = true; } } @@ -612,6 +687,7 @@ static bool IsNumeral(WordInfo w) => word.NormalizedForm = "目"; word.PreMatchedWordId = 1604890; word.PreMatchedReadingIndex = 0; + word.HardPinned = true; } } @@ -740,19 +816,22 @@ private List FixReadingAmbiguity(List wordInfos) word.Reading = "アト"; } - // The passive 弾かれ* is はじく "to be repelled/deflected" (bullets, blades, backboards, - // and the startle idiom 弾かれたように) — the ひく "to be played" passive occurs almost - // exclusively around instruments/music, so an instrument in the window keeps the ヒカレ - // reading (コンサートでピアノが弾かれた). Passive only: negatives and causatives - // (弾かない, 弾かせてやる) are overwhelmingly the play-verb and stay untouched. - if (word.DictionaryForm == "弾く" && word.Reading.StartsWith("ヒカレ", StringComparison.Ordinal)) + // 弾く read ヒ* is はじく "to flick/repel/deflect" (bullets, glasses, the startle idiom + // 弾かれたように) unless an instrument/music word holds the window — ひく "to play" only + // lives around music (ピアノが弾かれた, バンドで弾かせてやる, ピアノはもう弾かない). The + // scope is semantic, not conjugational: plain/te/past flick uses (グラスを弾いた) default + // to ひく just as wrongly as passives did. ヒケ* stays: the 弾く potential is handled with + // 弾ける below. + if (word.DictionaryForm == "弾く" + && word.Reading.StartsWith("ヒ", StringComparison.Ordinal) + && !word.Reading.StartsWith("ヒケ", StringComparison.Ordinal)) { bool musicContext = false; for (int k = Math.Max(0, i - 6); k < Math.Min(wordInfos.Count, i + 4) && !musicContext; k++) musicContext = wordInfos[k].Text is "ピアノ" or "ギター" or "エレキギター" or "バイオリン" or "ヴァイオリン" or "曲" or "演奏" or "コンサート" or "音楽" or "弦" or "楽器" or "バンド"; if (!musicContext) - word.Reading = "ハジカレ" + word.Reading[3..]; + word.Reading = "ハジ" + word.Reading[1..]; } // 生 after の with a life-cycle determiner (次の生を迎える) is せい "life/incarnation" @@ -802,6 +881,137 @@ private List FixReadingAmbiguity(List wordInfos) word.PreMatchedWordId = 1409150; } + // A numeral+分/日/月 run before a temporal anchor is the duration reading, never the + // fraction/idiom homograph (十分後 = じっぷん, not じゅうぶん "enough"; 一月経つ = ひとつき, + // not January). Pinned at token level, where punctuation still separates — 十分、間を置いて + // keeps じゅうぶん. The anchors never follow the fraction frame (四分の一). The lattice + // sometimes leaves the pair split (十|分後) — merge it under the same pin. + if (word.PreMatchedWordId == null) + { + int anchorIdx = i + 1; + while (anchorIdx < wordInfos.Count && wordInfos[anchorIdx].PartOfSpeech == PartOfSpeech.BlankSpace) + anchorIdx++; + bool anchored = anchorIdx < wordInfos.Count + && (TemporalAnchorTexts.Contains(wordInfos[anchorIdx].Text) + || wordInfos[anchorIdx].Text.StartsWith("待っ", StringComparison.Ordinal) + || wordInfos[anchorIdx].Text.StartsWith("経っ", StringComparison.Ordinal)); + if (anchored) + { + if (TemporalDurationPins.TryGetValue(word.Text, out var durationPin)) + { + word.PreMatchedWordId = durationPin.WordId; + word.PreMatchedReadingIndex = durationPin.ReadingIndex; + } + else if (word.Text is "分" or "日" or "月" && i > 0 + && TemporalDurationPins.TryGetValue(wordInfos[i - 1].Text + word.Text, out var pairPin)) + { + var numeral = wordInfos[i - 1]; + word.Text = numeral.Text + word.Text; + word.Reading = numeral.Reading + word.Reading; + word.StartOffset = numeral.StartOffset; + word.PartOfSpeech = PartOfSpeech.Noun; + word.DictionaryForm = word.Text; + word.NormalizedForm = word.Text; + word.PreMatchedWordId = pairPin.WordId; + word.PreMatchedReadingIndex = pairPin.ReadingIndex; + wordInfos.RemoveAt(i - 1); + i--; + } + } + } + + // A numeral + 時 read ジ is an hour of the clock. Left split, the pairwise recombiner + // steals digits across the boundary (十二|時 → 十 + 二時) and merged readings + // concatenate Sudachi's per-token onyomi (四+時 → シジ, matching the four-seasons + // homograph). Merge under the hour entry with the clock reading (四時=ヨジ, 九時=クジ). + if (word is { Text: "時", Reading: "ジ", PreMatchedWordId: null } && i > 0 + && HourPins.TryGetValue(wordInfos[i - 1].Text + "時", out var hourPin)) + { + var numeral = wordInfos[i - 1]; + word.Text = numeral.Text + word.Text; + word.Reading = hourPin.Reading; + word.StartOffset = numeral.StartOffset; + word.PartOfSpeech = PartOfSpeech.Noun; + word.DictionaryForm = word.Text; + word.NormalizedForm = word.Text; + word.PreMatchedWordId = hourPin.WordId; + wordInfos.RemoveAt(i - 1); + i--; + } + + // [N分]の[numeral] is the fraction frame: 分 is the part-noun ぶん (四分の一 = "one of + // four parts"), never the minutes counter. Split the fused token back so the numeral + // stays a numeral — unless the whole frame is itself an entry (三分の一), which the + // compound matcher should keep. + if (word.PreMatchedWordId == null && word.Text.Length >= 2 && word.Text.EndsWith('分') + && word.Text[..^1].All(c => JapaneseTextHelper.IsNumeralChar(c)) + && i + 2 < wordInfos.Count && wordInfos[i + 1].Text == "の" + && wordInfos[i + 2].Text.Length > 0 && JapaneseTextHelper.IsNumeralChar(wordInfos[i + 2].Text[0]) + && HasNonNameCompoundLookup?.Invoke(word.Text + "の" + wordInfos[i + 2].Text) != true) + { + var numeralText = word.Text[..^1]; + var reading = word.Reading ?? ""; + var numeralReading = reading.EndsWith("ブン", StringComparison.Ordinal) ? reading[..^2] + : reading.EndsWith("プン", StringComparison.Ordinal) || reading.EndsWith("フン", StringComparison.Ordinal) + ? reading[..^2] + : ""; + var numeral = new WordInfo(word) + { + Text = numeralText, + DictionaryForm = numeralText, + NormalizedForm = numeralText, + Reading = numeralReading, + EndOffset = word.StartOffset >= 0 ? word.StartOffset + numeralText.Length : -1, + PartOfSpeech = PartOfSpeech.Noun, + }; + word.Text = "分"; + word.DictionaryForm = "分"; + word.NormalizedForm = "分"; + word.Reading = "ブン"; + word.StartOffset = numeral.EndOffset; + word.PreMatchedWordId = 1502860; + word.HardPinned = true; + wordInfos.Insert(i, numeral); + i++; + } + + // The same frame arriving already split just needs the ぶん pin — Sudachi's reading + // for the bare 分 varies (フン in 七|分|の|一), so gate on the frame, not the reading. + if (word is { Text: "分", PreMatchedWordId: null } && i > 0 && i + 2 < wordInfos.Count + && wordInfos[i - 1].Text.Length > 0 + && wordInfos[i - 1].Text.All(c => JapaneseTextHelper.IsNumeralChar(c)) + && wordInfos[i + 1].Text == "の" + && wordInfos[i + 2].Text.Length > 0 && JapaneseTextHelper.IsNumeralChar(wordInfos[i + 2].Text[0]) + && HasNonNameCompoundLookup?.Invoke(wordInfos[i - 1].Text + "分の" + wordInfos[i + 2].Text) != true) + { + word.Reading = "ブン"; + word.PreMatchedWordId = 1502860; + word.HardPinned = true; + } + + // Sudachi sometimes lemmatises a っ-bearing verb to its っ-less homograph + // (かっこつけ→かこつける); when the surface itself deconjugates to an attested + // っ-bearing base, that base is the real lemma. Genuine っ-onbin conjugations + // (言って→言う) are untouched — their bases carry no っ. + if (word.PartOfSpeech == PartOfSpeech.Verb && word.PreMatchedWordId == null + && word.Text.Contains('っ') + && word.DictionaryForm is { Length: > 0 } tsuDf && !tsuDf.Contains('っ')) + { + var tsuForms = Deconjugator.Instance.Deconjugate(word.Text); + // Only when the current lemma is NOT itself a valid deconjugation of the surface — + // 背負っていた→背負う and よって→よる are real onbin chains and stay. + var tsuBetter = tsuForms.Any(f => f.Text == word.DictionaryForm) + ? null + : tsuForms.FirstOrDefault(f => f.Text.Contains('っ') + && f.Tags.Any(t => t.StartsWith("v", StringComparison.Ordinal)) + && HasNonNameCompoundLookup?.Invoke(f.Text) == true); + if (tsuBetter != null) + { + word.DictionaryForm = tsuBetter.Text; + word.NormalizedForm = tsuBetter.Text; + } + } + // 縁 after の with a rimmed-object noun is ふち "rim/edge" (浴槽の縁), not えん "fate". if (word is { Text: "縁", Reading: "エン" } && i >= 2 && wordInfos[i - 1].Text == "の" @@ -853,7 +1063,8 @@ private List FixReadingAmbiguity(List wordInfos) if (word is { Text: "隙", Reading: "ヒマ" }) word.Reading = "スキ"; - // 弄* (イラ*) → イジ* — いらう is archaic; modern 弄る is always いじる + // 弄* (イラ*) → イジ*: Sudachi lemmatises 弄った/弄っていた to the archaic 弄う (いらう, 2849632); + // modern usage is 弄る (いじる, 1560700). if (word.DictionaryForm == "弄う") { word.DictionaryForm = "弄る"; @@ -1027,6 +1238,7 @@ or PartOfSpeech.NaAdjective or PartOfSpeech.Counter or PartOfSpeech.Numeral word.DictionaryForm = word.Text + "る"; } + // 露 directly before になる is あらわ "exposed" (服が露になった), not dew/Russia — // Sudachi's ロ lexeme reading otherwise drags scoring to the wrong homograph. if (word is { Text: "露", PartOfSpeech: PartOfSpeech.Noun } diff --git a/Jiten.Parser/Stages/MorphologicalAnalyser.Pipeline.cs b/Jiten.Parser/Stages/MorphologicalAnalyser.Pipeline.cs index 2c2575b6..cbe6487a 100644 --- a/Jiten.Parser/Stages/MorphologicalAnalyser.Pipeline.cs +++ b/Jiten.Parser/Stages/MorphologicalAnalyser.Pipeline.cs @@ -15,6 +15,12 @@ private static TokenStage Stage( TokenFeatures requires = TokenFeatures.None) => new(process.Method.Name, group, process, requires); + private static TokenStage CandidateStage( + TokenStageGroup group, + Func, IReadOnlyList, List> process, + TokenFeatures candidateFeature) => + new(process.Method.Name, group, static input => input, candidateFeature, process); + private IReadOnlyList BuildTokenStages() => [ Stage(TokenStageGroup.Split, SplitOovGarbageTokens, TokenFeatures.OovGarbage), @@ -37,7 +43,12 @@ private IReadOnlyList BuildTokenStages() => Stage(TokenStageGroup.Repair, RetokeniseOovBlobs, TokenFeatures.HiraganaOovBlob), Stage(TokenStageGroup.Repair, RepairColloquialNegativeNee, TokenFeatures.Interjection), Stage(TokenStageGroup.Repair, RepairColloquialRanNai, TokenFeatures.TextRan), + Stage(TokenStageGroup.Repair, RepairIntensifierKaeru, TokenFeatures.VerbKaeru), Stage(TokenStageGroup.Repair, RepairQuotativeTte, TokenFeatures.EndsWithTsu), + CandidateStage(TokenStageGroup.Repair, RepairGeminateSuffixTheft, TokenFeatures.GeminateSuffixShape), + CandidateStage(TokenStageGroup.Repair, RepairKatakanaShreds, TokenFeatures.KatakanaRun), + CandidateStage(TokenStageGroup.Repair, RepairCompoundBoundaryTheft, TokenFeatures.CompoundBoundaryShape), + CandidateStage(TokenStageGroup.Repair, RepairKanjiVerbShred, TokenFeatures.SingleKanjiNoun), Stage(TokenStageGroup.Repair, RecombineHiraganaTokens), Stage(TokenStageGroup.Repair, ApplyTokenRewriteRulesLate), @@ -68,6 +79,7 @@ private IReadOnlyList BuildTokenStages() => Stage(TokenStageGroup.Repair, RepairTteNani), Stage(TokenStageGroup.Cleanup, ApplyTokenRewriteRulesCleanup), + Stage(TokenStageGroup.Cleanup, RepairDanTobashi, TokenFeatures.TextTobashi), Stage(TokenStageGroup.Cleanup, FilterMisparse), Stage(TokenStageGroup.Disambiguation, FixReadingAmbiguity), ]; @@ -77,11 +89,20 @@ private List RunPipeline(List wordInfos, ParserDiagnostics? { _pipelineDeconjCache = new Dictionary>(StringComparer.Ordinal); _pipelineDeconjCacheAlt = _pipelineDeconjCache.GetAlternateLookup>(); + TokenFeatureScan? candidateScan = null; var features = TokenFeatureScanner.Scan(wordInfos); + bool candidateScanDirty = true; Stopwatch? sw = timings != null ? Stopwatch.StartNew() : null; foreach (var stage in GetTokenStages()) { + if (stage.UsesCandidatePositions && candidateScanDirty) + { + candidateScan = TokenFeatureScanner.ScanWithCandidates(wordInfos); + features = candidateScan.Features; + candidateScanDirty = false; + } + if (stage.RequiredFeatures != TokenFeatures.None && (features & stage.RequiredFeatures) == TokenFeatures.None) { @@ -91,13 +112,22 @@ private List RunPipeline(List wordInfos, ParserDiagnostics? sw?.Restart(); var prev = wordInfos; - wordInfos = TrackStage(stage, wordInfos, diagnostics); + wordInfos = TrackStage(stage, wordInfos, diagnostics, candidateScan); if (Environment.GetEnvironmentVariable("JITEN_STAGE_DEBUG") is { Length: > 0 }) Console.WriteLine($"[stage] {stage.Name}: {string.Join("|", wordInfos.Select(w => w.Text))}"); if (!ReferenceEquals(prev, wordInfos)) + { features = TokenFeatureScanner.Scan(wordInfos); + candidateScanDirty = true; + } + else if (!stage.UsesCandidatePositions) + { + // Earlier stages may edit tokens in place. Refresh once, immediately before the + // structural candidate block, instead of trusting positions collected before it. + candidateScanDirty = true; + } if (sw != null) { @@ -116,7 +146,7 @@ internal IReadOnlyList GetPipelineStageNamesForTesting() => internal List ApplyStageForTesting(string stageName, List input, ParserDiagnostics? diagnostics = null) { var stage = GetTokenStages().First(s => s.Name == stageName); - return TrackStage(stage, input, diagnostics); + return TrackStage(stage, input, diagnostics, TokenFeatureScanner.ScanWithCandidates(input)); } internal List RunPipelineForTesting(List input, ParserDiagnostics? diagnostics = null) => diff --git a/Jiten.Parser/Stages/MorphologicalAnalyser.RepairStages.cs b/Jiten.Parser/Stages/MorphologicalAnalyser.RepairStages.cs index 55396c59..f78fbbca 100644 --- a/Jiten.Parser/Stages/MorphologicalAnalyser.RepairStages.cs +++ b/Jiten.Parser/Stages/MorphologicalAnalyser.RepairStages.cs @@ -7,6 +7,612 @@ namespace Jiten.Parser; public partial class MorphologicalAnalyser { + // The geminate suffixes 〜っぱなし/っぷり/っぽい attach to a verb 連用形 or noun, and Sudachi + // routinely lets the っ (and sometimes more) bleed into the neighbours: 流|しっ|ぱなし + // (しっ→知る), 上|が|りっぱ|な|し (りっぱ→立派), 脳天|気っぷ|り (気っぷ→気風), 皮肉っ|ぽい + // (→皮肉る). Re-cut so っ heads its suffix, pin the suffix entry, and reassemble the stem + // leftward while the concatenation deconjugates to an attested verb (上+が+り→上がり). + private static readonly Dictionary GeminateSuffixes = new() + { + ["ぱなし"] = ("っぱなし", 1008020), + ["ぷり"] = ("っぷり", 2202980), + ["ぽい"] = ("っぽい", 2083720), + ["ぽく"] = ("っぽい", 2083720), + ["ぽさ"] = ("っぽい", 2083720), + }; + + private List RepairGeminateSuffixTheft(List wordInfos, IReadOnlyList candidates) + { + if (wordInfos.Count < 2) return wordInfos; + + var deconj = Deconjugator.Instance; + + // Extend the stem when the concatenation is a verb renyoukei AND either attests as a + // surface itself (上がり, こもり) or the previous token is an orphaned kanji fragment + // (失+い, 打+ち). A complete kana word before the stem (ワクワク+し) stays split so し + // resolves as する through the normal machinery. + bool StemExtends(WordInfo prev, string candidate) + { + if (candidate.Length == 0) return false; + var forms = deconj.Deconjugate(NormalizeToHiragana(candidate)); + bool isVerbStem = forms.Any(f => f.Tags.Any(t => t.StartsWith("v", StringComparison.Ordinal)) + && HasVerbOrAdjectiveLookup?.Invoke(f.Text) == true); + if (!isVerbStem) return false; + if (HasNonNameCompoundLookup?.Invoke(candidate) == true) return true; + return prev.Text.Any(JapaneseTextHelper.IsKanji); + } + + List? list = null; + int indexDelta = 0; + int skipOriginalThrough = -1; + foreach (int originalIndex in candidates) + { + if (originalIndex <= skipOriginalThrough) continue; + var toks = list ?? wordInfos; + int i = originalIndex + indexDelta; + if ((uint)i >= (uint)toks.Count) continue; + var t = toks[i]; + string? stemText = null, suffixText = null; + int consumed = 0; + + // The stem left of っ must be renyoukei-shaped (kanji, or i/e-row kana final) or a + // noun-ish kanji — a-row finals (やっ|ぱなし = やっぱ+なし) are never the suffix host. + static bool CanHostGeminateSuffix(string stem) => + stem.Length > 0 && (JapaneseTextHelper.IsKanji(stem[^1]) + || "いきしちにひみりぎじぢびぴえけせてねへめれげぜでべぺ".Contains(stem[^1])); + + if (t.Text.Length > 1 && t.Text.EndsWith('っ') && i + 1 < toks.Count + && GeminateSuffixes.ContainsKey(toks[i + 1].Text) + && CanHostGeminateSuffix(t.Text[..^1])) + { + stemText = t.Text[..^1]; + suffixText = "っ" + toks[i + 1].Text; + consumed = 2; + } + else if (t.Text.Length > 2 && t.Text.EndsWith("っぱ", StringComparison.Ordinal) + && i + 1 < toks.Count + && (toks[i + 1].Text == "なし" + || (i + 2 < toks.Count && toks[i + 1].Text == "な" && toks[i + 2].Text == "し")) + && CanHostGeminateSuffix(t.Text[..^2])) + { + stemText = t.Text[..^2]; + suffixText = "っぱなし"; + consumed = toks[i + 1].Text == "なし" ? 2 : 3; + } + else if (t.Text.Length > 2 && t.Text.EndsWith("っぷ", StringComparison.Ordinal) + && i + 1 < toks.Count && toks[i + 1].Text == "り" + && CanHostGeminateSuffix(t.Text[..^2])) + { + stemText = t.Text[..^2]; + suffixText = "っぷり"; + consumed = 2; + } + + if (stemText == null) continue; + + list ??= new List(wordInfos); + skipOriginalThrough = originalIndex + consumed - 1; + var (suffixDict, suffixPin) = GeminateSuffixes[suffixText!.TrimStart('っ')]; + var last = list[i + consumed - 1]; + + var stem = new WordInfo(t) + { + Text = stemText, + DictionaryForm = stemText, + NormalizedForm = stemText, + Reading = t.Reading != null && t.Reading.Length > t.Text.Length - stemText.Length + ? t.Reading[..^(t.Text.Length - stemText.Length)] + : "", + EndOffset = t.StartOffset >= 0 ? t.StartOffset + stemText.Length : -1, + PartOfSpeech = PartOfSpeech.Noun, + }; + var suffix = new WordInfo + { + Text = suffixText, + DictionaryForm = suffixDict, + NormalizedForm = suffixDict, + Reading = WanaKana.ToKatakana(suffixText), + PartOfSpeech = PartOfSpeech.Suffix, + StartOffset = stem.EndOffset, + EndOffset = last.EndOffset, + PreMatchedWordId = suffixPin, + HardPinned = true, + }; + + list.RemoveRange(i, consumed); + list.Insert(i, suffix); + list.Insert(i, stem); + + // Reassemble the stem leftward while it still deconjugates to an attested verb + // (上+が+り→上がり→上がる); stop before over-reaching (を+飛ばし fails the test). + int merges = 0; + int stemIdx = i; + while (stemIdx > 0 && merges < 3) + { + var prev = list[stemIdx - 1]; + var cand = prev.Text + list[stemIdx].Text; + if (!StemExtends(prev, cand)) break; + var merged = new WordInfo(list[stemIdx]) + { + Text = cand, + DictionaryForm = cand, + NormalizedForm = cand, + Reading = (prev.Reading ?? "") + (list[stemIdx].Reading ?? ""), + StartOffset = prev.StartOffset, + }; + list[stemIdx] = merged; + list.RemoveAt(stemIdx - 1); + stemIdx--; + merges++; + } + + indexDelta = list.Count - wordInfos.Count; + } + + return list ?? wordInfos; + } + + // True when the surface cuts into two attested words (ハロー|ワーク): such a blob belongs + // to the resegmentation lattice, which scores the cut properly. + private bool HasAttestedBipartition(string text) + { + for (int cutAt = 2; cutAt <= text.Length - 2; cutAt++) + { + if (HasNonNameCompoundLookup?.Invoke(text[..cutAt]) == true + && HasNonNameCompoundLookup?.Invoke(text[cutAt..]) == true) + return true; + } + + return false; + } + + // Sudachi shreds OOV katakana compounds into coincidentally-attested fragments (ポン|コツ, + // ホロ|グラフ "graph", ドシ|ロウト "funnel", コーポ|レイ|テッド). Re-merge a contiguous + // katakana run when the whole attests, or when any piece is fragment-shaped (≤2 chars or + // unattested) — the merged span then resolves or fails as one unit downstream instead of + // the fragments matching junk separately. Two genuine long words meeting (システム|エラー) + // stay separate. + private List RepairKatakanaShreds(List wordInfos, IReadOnlyList candidates) + { + if (wordInfos.Count == 0) return wordInfos; + + static bool IsKatakanaRunToken(WordInfo w) => + w.Text.Length > 0 && w.PreMatchedWordId == null + && !w.IsPersonNameContext + && !PosMapper.IsNameLikeSudachiNoun(w.PartOfSpeech, w.PartOfSpeechSection1, + w.PartOfSpeechSection2, w.PartOfSpeechSection3) + && w.Text.All(JapaneseTextHelper.IsKatakanaWordChar); + + List? list = null; + int indexDelta = 0; + int skipOriginalThrough = -1; + foreach (int originalIndex in candidates) + { + if (originalIndex <= skipOriginalThrough) continue; + var toks = list ?? wordInfos; + int i = originalIndex + indexDelta; + // A sentence-final token has no run to extend but can still take the tail-word split. + if (i < 0 || i >= toks.Count) continue; + + if (!IsKatakanaRunToken(toks[i])) continue; + + // Runs must be offset-contiguous in the source text: stripped markup leaves + // unrelated tokens adjacent ([name]アリサ[line]“リア充 → アリサ|リア充), and merging + // across that gap manufactures アリサリア. + static bool Contiguous(WordInfo a, WordInfo b) => + a.EndOffset >= 0 && b.StartOffset >= 0 && a.EndOffset == b.StartOffset; + + int end = i; + while (end + 1 < toks.Count && end - i < 4 && IsKatakanaRunToken(toks[end + 1]) + && Contiguous(toks[end], toks[end + 1])) + end++; + + // A katakana-headed mixed token continues the run (ブロー|アップされる, クオン|ツが): + // the head belongs to the compound, the tail is grammar to split back off. + string mixedTail = ""; + int mixedIdx = -1; + if (end + 1 < toks.Count && end - i < 4 && toks[end + 1].PreMatchedWordId == null + && Contiguous(toks[end], toks[end + 1])) + { + var cand = toks[end + 1].Text; + int kl = 0; + while (kl < cand.Length && JapaneseTextHelper.IsKatakanaWordChar(cand[kl])) kl++; + // The tail must be grammar (hiragana される/が) — a kanji tail means the token is + // its own compound (リア充), not a shredded head plus grammar. + if (kl >= 1 && kl < cand.Length && cand[kl..].All(c => c is >= 'ぁ' and <= 'ゖ')) + { + mixedIdx = end + 1; + mixedTail = cand[kl..]; + } + } + + // A single unattested blob can still hold a real loanword at its tail (an OOV name + // fused with レベル). Split the longest attested ≥3-char tail off when the remaining + // head is itself unattested — an attested head would mean cutting a plausible whole + // into two coincidental words, which drops whole instead. The head must be ≥4 chars + // to be confidently its own name unit; a shorter head defers to the resegmentation + // lattice, which keeps name candidates. + if (end == i && mixedIdx < 0) + { + var text = toks[i].Text; + if (text.Length >= 7 && HasNonNameCompoundLookup?.Invoke(text) != true + && !HasAttestedBipartition(text)) + { + for (int headLen = 4; headLen <= text.Length - 3; headLen++) + { + var head = text[..headLen]; + var tail = text[headLen..]; + if (HasNonNameCompoundLookup?.Invoke(tail) != true + || HasNonNameCompoundLookup?.Invoke(head) == true) + continue; + + list ??= new List(wordInfos); + var source = list[i]; + int cut = source.StartOffset >= 0 ? source.StartOffset + headLen : -1; + var headToken = new WordInfo(source) + { + Text = head, DictionaryForm = head, NormalizedForm = head, + Reading = source.Reading is { Length: > 0 } r && r.Length >= headLen + ? r[..headLen] : head, + EndOffset = cut, + }; + var tailToken = new WordInfo(source) + { + Text = tail, DictionaryForm = tail, NormalizedForm = tail, + Reading = source.Reading is { Length: > 0 } r2 && r2.Length >= headLen + ? r2[headLen..] : tail, + StartOffset = cut, + }; + list[i] = headToken; + list.Insert(i + 1, tailToken); + skipOriginalThrough = originalIndex; + indexDelta = list.Count - wordInfos.Count; + break; + } + } + + continue; + } + + string full = string.Concat(toks.Skip(i).Take(end - i + 1).Select(t => t.Text)); + if (mixedIdx >= 0) + full += toks[mixedIdx].Text[..^mixedTail.Length]; + bool wholeAttested = HasNonNameCompoundLookup?.Invoke(full) == true; + bool anyFragment = mixedIdx >= 0; + for (int j = i; j <= end && !anyFragment; j++) + anyFragment = toks[j].Text.Length <= 2 + || HasNonNameCompoundLookup?.Invoke(toks[j].Text) != true; + + if (!wholeAttested && !anyFragment) + { + skipOriginalThrough = originalIndex + end - i; + continue; + } + + // An attested long tail piece is its own word, not shred material: a shredded OOV name + // ending in レベル merges only the unattested head and keeps レベル. Only while at + // least two pieces remain to merge — a two-piece run of coincidentally-attested halves + // is name material that must still merge whole and drop, not leave its tail behind. + if (!wholeAttested && mixedIdx < 0) + { + while (end > i + 1 && toks[end].Text.Length >= 3 + && HasNonNameCompoundLookup?.Invoke(toks[end].Text) == true) + end--; + + full = string.Concat(toks.Skip(i).Take(end - i + 1).Select(t => t.Text)); + wholeAttested = HasNonNameCompoundLookup?.Invoke(full) == true; + } + + list ??= new List(wordInfos); + int last = mixedIdx >= 0 ? mixedIdx : end; + int mergedEnd = mixedIdx >= 0 + ? (list[mixedIdx].EndOffset >= 0 ? list[mixedIdx].EndOffset - mixedTail.Length : -1) + : list[end].EndOffset; + var merged = new WordInfo(list[i]) + { + Text = full, + DictionaryForm = full, + NormalizedForm = full, + Reading = string.Concat(list.Skip(i).Take(end - i + 1).Select(t => t.Reading ?? "")) + + (mixedIdx >= 0 ? full[^(list[mixedIdx].Text.Length - mixedTail.Length)..] : ""), + PartOfSpeech = PartOfSpeech.Noun, + EndOffset = mergedEnd, + }; + list.RemoveRange(i, last - i + 1); + list.Insert(i, merged); + if (mixedIdx >= 0) + list.InsertRange(i + 1, TokenizeGrammarRemainder(mixedTail, mergedEnd)); + skipOriginalThrough = originalIndex + last - i; + indexDelta = list.Count - wordInfos.Count; + } + + return list ?? wordInfos; + } + + // Sudachi's lattice steals compound boundaries in two recurring shapes: a location suffix + // pulls the compound's last kanji into itself (滑走|路上 — 上 belongs to 滑走路), and a + // demonstrative expression eats the next compound's first kanji (その時|系列 — 時 belongs + // to 時系列). Both re-cut on attestation of the corrected pieces. + // Location/relational suffixes plus 的: all attach to the FULL preceding compound + // (滑走路+上, 消去法+的), so a 2-char token headed by a stolen kanji re-cuts. + private static readonly HashSet LocationSuffixChars = + ['上', '中', '内', '外', '前', '後', '間', '下', '先', '際', '的']; + + // B must outrank the extended compound by this factor before its own-word reading beats the + // theft re-cut — the tuned cutoff separating 日本|国内 (stays) from 滑走|路上 (re-cuts). + private const int BoundaryTheftRankDominanceFactor = 4; + + private static readonly string[] DemonstrativeHeads = ["その", "この", "あの", "どの"]; + + // Sudachi splits a conjugated verb's kanji head from its kana tail, lemmatising both as + // unrelated words: 会|おう (会+王), 信|じろ (信+囲炉裏?), 歩|こう, 過|ごせ, 見|直し. When the + // concatenation deconjugates in one step to an attested verb that starts with the kanji, + // the split is spurious — merge back as that verb. + private List RepairKanjiVerbShred(List wordInfos, IReadOnlyList candidates) + { + if (wordInfos.Count < 2) return wordInfos; + + var deconj = Deconjugator.Instance; + bool changed = false; + + int indexDelta = 0; + foreach (int originalIndex in candidates) + { + int i = originalIndex + indexDelta; + if (i < 0 || i + 1 >= wordInfos.Count) continue; + var k = wordInfos[i]; + var tail = wordInfos[i + 1]; + if (k.PreMatchedWordId != null || tail.PreMatchedWordId != null) continue; + if (k.Text.Length != 1 || !JapaneseTextHelper.IsKanji(k.Text[0])) continue; + if (k.PartOfSpeech is not (PartOfSpeech.Noun or PartOfSpeech.CommonNoun)) continue; + // A clause-initial single kanji is speaker-name territory in script dumps + // ([name]至[line]“そうなん?” would merge into 至る). + if (i == 0 || wordInfos[i - 1].PartOfSpeech is PartOfSpeech.Symbol + or PartOfSpeech.SupplementarySymbol or PartOfSpeech.BlankSpace) continue; + if (tail.Text.Length is < 1 or > 4 || !tail.Text.All(c => c is >= 'ぁ' and <= 'ゖ')) continue; + if (tail.PartOfSpeech is PartOfSpeech.Particle or PartOfSpeech.Auxiliary + or PartOfSpeech.Symbol or PartOfSpeech.SupplementarySymbol) continue; + // A copula tail belongs to the sentence (目|だった must not become 目立った), and a + // dictionary-form verb standing complete is its own word (今|いる). + if (tail.DictionaryForm is "だ" or "です" + || tail.Text is "だ" or "だっ" or "だった" or "だったら" or "で" or "です" or "でし" + or "でした" or "だろ" or "だろう" or "じゃ" or "じゃない" or "じゃなく" + // する-negatives after a noun are noun+しない (話|しないで), never a shred. + or "しない" or "しないで" or "しなく" or "せず") continue; + // A bare し before ない is the same する-negative one token earlier (話|し|ないで). + if (tail.Text == "し" && i + 2 < wordInfos.Count + && wordInfos[i + 2].Text.StartsWith("な", StringComparison.Ordinal)) continue; + if (tail.PartOfSpeech == PartOfSpeech.Verb && tail.Text == tail.DictionaryForm) continue; + // 何 heads no verb — its kana continuations are always their own words (何|かって). + if (k.Text == "何") continue; + + var cand = k.Text + tail.Text; + var form = deconj.Deconjugate(cand).FirstOrDefault(f => + f.Text.Length > 1 && f.Text != cand + && f.Text.StartsWith(k.Text, StringComparison.Ordinal) + && f.Tags.Any(t => t.StartsWith("v", StringComparison.Ordinal)) + && f.Process.Count > 0 + && HasNonNameCompoundLookup?.Invoke(f.Text) == true); + if (form == null) continue; + + k.Text = cand; + k.DictionaryForm = form.Text; + k.NormalizedForm = form.Text; + k.Reading = ""; + k.PartOfSpeech = PartOfSpeech.Verb; + k.EndOffset = tail.EndOffset; + wordInfos.RemoveAt(i + 1); + changed = true; + indexDelta--; + } + + // A fresh list signals the pipeline to rescan token features (a Noun became a Verb). + return changed ? new List(wordInfos) : wordInfos; + } + + private List RepairCompoundBoundaryTheft(List wordInfos, IReadOnlyList candidates) + { + if (wordInfos.Count < 2) return wordInfos; + + bool changed = false; + foreach (int i in candidates) + { + if (i < 0 || i + 1 >= wordInfos.Count) continue; + var a = wordInfos[i]; + var b = wordInfos[i + 1]; + if (a.PreMatchedWordId != null || b.PreMatchedWordId != null) continue; + + // [滑走][路上] → [滑走路][上], [小][動物的] → [小動物][的]: B is a 2–3 char word whose + // last char is a compound-final suffix, and A's extension by B's head attests. + if (b.Text.Length is 2 or 3 && b.Text != "時間" && LocationSuffixChars.Contains(b.Text[^1]) + && b.Text[..^1].All(JapaneseTextHelper.IsKanji) + && a.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun or PartOfSpeech.Prefix + && b.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun or PartOfSpeech.NaAdjective + && a.Text.Length >= 1 && a.Text.All(JapaneseTextHelper.IsKanji) + && (a.Text.Length >= 2 || a.PartOfSpeech == PartOfSpeech.Prefix) + // A numeral head means B is a duration/counter word (二十|時間), never a theft. + && !a.Text.All(JapaneseTextHelper.IsNumeralChar) + && HasNonNameCompoundLookup?.Invoke(a.Text + b.Text[..^1]) == true + // B is a genuine constituent when it heads its own compound with what follows + // (人類|史上|初 — 史上初 attests; 滑走|路上|で does not). + && !(i + 2 < wordInfos.Count + && HasNonNameCompoundLookup?.Invoke(b.Text + wordInfos[i + 2].Text) == true) + // A much more frequent B is its own word, not a theft (日本|国内 stays: 国内 + // outranks 日本国 by far; 滑走|路上 re-cuts: 滑走路 holds its own against 路上). + && !(GetNonNameCompoundFrequencyRank?.Invoke(b.Text) is int bRank + && GetNonNameCompoundFrequencyRank?.Invoke(a.Text + b.Text[..^1]) is var extRank + && (extRank == null || bRank * BoundaryTheftRankDominanceFactor < extRank.Value))) + { + var extended = a.Text + b.Text[..^1]; + a.Text = extended; + a.DictionaryForm = extended; + a.NormalizedForm = extended; + a.Reading = ""; + a.PartOfSpeech = PartOfSpeech.Noun; + a.EndOffset = a.EndOffset >= 0 ? a.EndOffset + b.Text.Length - 1 : -1; + var suffix = b.Text[^1..]; + b.Text = suffix; + b.DictionaryForm = suffix; + b.NormalizedForm = suffix; + b.Reading = ""; + b.PartOfSpeech = PartOfSpeech.Suffix; + b.StartOffset = a.EndOffset; + changed = true; + continue; + } + + // [その時][系列] → [その][時系列]: a demonstrative expression whose trailing kanji + + // the next noun attests as a compound. + if (a.PartOfSpeech == PartOfSpeech.Expression && a.Text.Length == 3 + && DemonstrativeHeads.Contains(a.Text[..2]) + && JapaneseTextHelper.IsKanji(a.Text[2]) + && b.Text.Length >= 1 && b.Text.All(JapaneseTextHelper.IsKanji) + && b.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun + && HasNonNameCompoundLookup?.Invoke(a.Text[2] + b.Text) == true) + { + var stolen = a.Text[2]; + a.Text = a.Text[..2]; + a.DictionaryForm = a.Text; + a.NormalizedForm = a.Text; + // The demonstrative is always two kana, so its reading is the first two katakana — + // the stolen kanji's reading length varies (時=トキ, 日=ヒ) and can't be trimmed off. + a.Reading = a.Reading is { Length: >= 2 } ar ? ar[..2] : ""; + a.PartOfSpeech = PartOfSpeech.PrenounAdjectival; + a.EndOffset = a.EndOffset >= 0 ? a.EndOffset - 1 : -1; + b.Text = stolen + b.Text; + b.DictionaryForm = b.Text; + b.NormalizedForm = b.Text; + b.Reading = ""; + b.StartOffset = b.StartOffset >= 0 ? b.StartOffset - 1 : -1; + changed = true; + } + } + + // A fresh list signals the pipeline to rescan token features (recuts introduce Suffix tokens). + return changed ? new List(wordInfos) : wordInfos; + } + + // An OOV compound verb Sudachi normalises to X返る (のさばり返る) — the intensifier suffix 返る on a + // JMDict verb stem — has no JMDict entry of its own and drops whole. Split it into the head verb + + // 返る so both resolve; a following て is folded into 返って here so the te-form is not later mis-read + // as a quotative って. A genuine X返る (静まり返る, 呆れ返る) is a JMDict compound and is excluded by the + // lookup gate, so only the OOV coinages reach the split. + private List RepairIntensifierKaeru(List wordInfos) + { + if (HasNonNameCompoundLookup == null) return wordInfos; + var deconj = Deconjugator.Instance; + List? result = null; + for (int i = 0; i < wordInfos.Count; i++) + { + var w = wordInfos[i]; + if (w.PartOfSpeech == PartOfSpeech.Verb && w.PreMatchedWordId == null + && w.NormalizedForm.Length >= 4 && w.Text.Length >= 3 + && w.NormalizedForm.EndsWith("返る", StringComparison.Ordinal) + && !HasNonNameCompoundLookup(w.NormalizedForm)) + { + // The head is the normalised form minus 返る (a renyoukei, kana-stable): のさばり → のさばる. + var headStem = w.NormalizedForm[..^2]; + string? headDict = null; + foreach (var f in deconj.Deconjugate(headStem)) + if (f.Tags.Any(t => t.StartsWith("v", StringComparison.Ordinal)) && HasNonNameCompoundLookup(f.Text)) + { headDict = f.Text; break; } + + // The surface must open with that same kana head (のさばりかえっ = のさばり + かえっ). + if (headDict != null && w.Text.StartsWith(headStem, StringComparison.Ordinal) + && w.Text.Length > headStem.Length) + { + result ??= [..wordInfos[..i]]; + var suffixSurface = w.Text[headStem.Length..]; + int cut = w.StartOffset >= 0 ? w.StartOffset + headStem.Length : -1; + result.Add(new WordInfo(w) + { + Text = headStem, DictionaryForm = headDict, NormalizedForm = headDict, + Reading = "", PartOfSpeech = PartOfSpeech.Verb, EndOffset = cut + }); + // 返っ + a following て/た → the single 返って/返った (pinned so it is not read as 却って). + if (suffixSurface.EndsWith("っ", StringComparison.Ordinal) + && i + 1 < wordInfos.Count && wordInfos[i + 1].Text is "て" or "た") + { + result.Add(new WordInfo(w) + { + Text = suffixSurface + wordInfos[i + 1].Text, DictionaryForm = "返る", NormalizedForm = "返る", + Reading = "", PartOfSpeech = PartOfSpeech.Verb, PreMatchedWordId = 1512150, + // The chain is recovered against kana かえる: deconjugation is kana-side, + // so the kanji lemma would never match its output forms. + PreMatchedConjugations = PinnedConjugationProcess(suffixSurface + wordInfos[i + 1].Text, "かえる"), + StartOffset = cut, EndOffset = wordInfos[i + 1].EndOffset + }); + i++; + continue; + } + result.Add(new WordInfo(w) + { + Text = suffixSurface, DictionaryForm = "返る", NormalizedForm = "返る", + Reading = "", PartOfSpeech = PartOfSpeech.Verb, PreMatchedWordId = 1512150, + PreMatchedConjugations = PinnedConjugationProcess(suffixSurface, "かえる"), + StartOffset = cut, EndOffset = w.EndOffset + }); + continue; + } + } + result?.Add(w); + } + return result ?? wordInfos; + } + + // 段飛ばし (2746000, "skipping steps") is a lexicalised counter-compound. A preceding numeral + // greedily claims 段 — 三|段|飛ばし and Sudachi's fused 一段|飛ばし both leave 飛ばし stranded on the + // securities-fraud noun (1637130) — so reform the compound with 段 as its head: 三|段飛ばし, + // 一|段飛ばし. Scoped to the attested 飛ばし compound (a blanket 段+X rule would split 三段跳び). + private List RepairDanTobashi(List wordInfos) + { + List? result = null; + for (int i = 0; i < wordInfos.Count; i++) + { + var w = wordInfos[i]; + if (i + 1 < wordInfos.Count && wordInfos[i + 1] is { Text: "飛ばし" } b + && w.PreMatchedWordId == null && b.PreMatchedWordId == null + && w.Text.EndsWith("段", StringComparison.Ordinal)) + { + var head = w.Text[..^1]; + // Standalone 段 (三|段|飛ばし): merge 段+飛ばし; the numeral before it is left untouched. + if (head.Length == 0) + { + result ??= [..wordInfos[..i]]; + result.Add(new WordInfo(b) + { + Text = "段飛ばし", DictionaryForm = "段飛ばし", NormalizedForm = "段飛ばし", + Reading = "ダントバシ", PartOfSpeech = PartOfSpeech.Noun, + StartOffset = w.StartOffset, EndOffset = b.EndOffset, PreMatchedWordId = 2746000 + }); + i++; + continue; + } + // Numeral+段 fused by Sudachi (一段|飛ばし): release 段 to the compound → 一 + 段飛ばし. + // 何/数 head 何段/数段 ("how many / several steps"), so admit them alongside true numerals. + if (head.All(c => JapaneseTextHelper.IsNumeralChar(c) || c is '何' or '数')) + { + result ??= [..wordInfos[..i]]; + int cut = w.StartOffset >= 0 ? w.StartOffset + head.Length : -1; + result.Add(new WordInfo(w) + { + Text = head, DictionaryForm = head, NormalizedForm = head, Reading = "", EndOffset = cut + }); + result.Add(new WordInfo(b) + { + Text = "段飛ばし", DictionaryForm = "段飛ばし", NormalizedForm = "段飛ばし", + Reading = "ダントバシ", PartOfSpeech = PartOfSpeech.Noun, + StartOffset = cut, EndOffset = b.EndOffset, PreMatchedWordId = 2746000 + }); + i++; + continue; + } + } + result?.Add(w); + } + return result ?? wordInfos; + } + private List RepairTankaToTaNKa(List wordInfos) { var result = new List(wordInfos.Count + 4); @@ -321,7 +927,7 @@ private List RepairVowelElongation(List wordInfos) { var w = wordInfos[i]; - // Strip trailing ー from particles/conjunctions (colloquial elongation like けどー) + // Strip trailing ー from particles/conjunctions (colloquial elongation like けどー). if (w.Text.Length >= 2 && w.Text[^1] == 'ー' && w.PartOfSpeech is PartOfSpeech.Particle or PartOfSpeech.Conjunction) { @@ -911,8 +1517,27 @@ current.DictionaryForm is "の" or "ん" && } // If んだ/んで didn't match, try negative ん contraction (ませ+ん→ません) - // Only for actual negative auxiliary (DictionaryForm = "ぬ"), not explanatory ん - if (!combined && current.DictionaryForm == "ぬ" && + // Only for actual negative auxiliary (DictionaryForm = "ぬ"), not explanatory ん. + // Sudachi tags the slurred negative as ぬ after godan stems but falls back to the + // explanatory の after an ichidan stem (足り+ん) — impossible there, because the + // nominaliser ん attaches to the 連体形 (足りるん), never the bare stem. + bool misreadStemNegative = current.DictionaryForm is "の" or "ん" + && result.Count > 0 + && result[^1].PartOfSpeech == PartOfSpeech.Verb + && result[^1].Text.Length > 0 + && result[^1].Text != result[^1].DictionaryForm + // After a past/te-form (死んだ+ん, 読んで+ん) the ん is the explanatory のだ or a + // slurred ている — the negative can only follow a bare stem. + && result[^1].Text[^1] is not ('た' or 'だ' or 'て' or 'で') + // する's slurred negative is せん, never しん/すん — the mizenkei-ん deconjugation + // rule would wrongly validate し+ん (kana しん is 死ぬ material instead), and a + // bare す stem before explanatory ん is the contracted する (すんだ = するんだ). + && !(result[^1].Text is "し" or "す" && result[^1].DictionaryForm is "する" or "為る") + // ある's negative is ない/あらん, never あん — bare あ before explanatory ん is + // the contracted ある (あんだ = あるんだ). + && !(result[^1].Text == "あ" && result[^1].DictionaryForm is "ある" or "有る" or "在る"); + + if (!combined && (current.DictionaryForm == "ぬ" || misreadStemNegative) && TryCombineWithLookback(result, "ん", "ん", deconj, IsAnyVerbForm, out var negativeWord)) { negativeWord!.EndOffset = current.EndOffset; @@ -1470,28 +2095,37 @@ private List ProcessSpecialCases(List wordInfos) continue; } - // 〜つ目 that Sudachi keeps as one token (四つ目) is number + つ + the ordinal suffix 目 "-th" - // (1604890), not the noun "four-eyed". Split so it matches the split 三つ目. - if (w1.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun - && w1.Text.Length >= 3 && w1.Text.EndsWith("つ目", StringComparison.Ordinal) - && TakesOrdinalMeAfterTsu(w1.Text[0])) - { - var numPart = w1.Text[..^1]; // 四つ - int mid = w1.StartOffset >= 0 ? w1.StartOffset + numPart.Length : -1; - newList.Add(new WordInfo(w1) - { - Text = numPart, DictionaryForm = numPart, NormalizedForm = numPart, - Reading = w1.Reading.Length > 0 ? w1.Reading[..^1] : "", EndOffset = mid - }); - newList.Add(new WordInfo - { - Text = "目", DictionaryForm = "目", NormalizedForm = "目", - PartOfSpeech = PartOfSpeech.Suffix, Reading = "メ", - PreMatchedWordId = 1604890, PreMatchedReadingIndex = 0, - StartOffset = mid, EndOffset = w1.EndOffset - }); - i += 1; - continue; + // 〜つ目/〜つめ that Sudachi keeps as one token (四つ目, みっつめ→見詰める!) is + // number + つ + the ordinal suffix 目 "-th" (1604890), not the noun "four-eyed" or a + // verb. Split so it matches the split 三つ目; kana numerals are a closed set. + { + bool kanjiTsuMe = w1.Text.Length >= 3 && w1.Text.EndsWith("つ目", StringComparison.Ordinal) + && TakesOrdinalMeAfterTsu(w1.Text[0]); + bool kanaTsuMe = w1.Text.EndsWith("つめ", StringComparison.Ordinal) + && w1.Text[..^1] is "ひとつ" or "ふたつ" or "みっつ" or "よっつ" or "いつつ" + or "むっつ" or "ななつ" or "やっつ" or "ここのつ"; + if (w1.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun or PartOfSpeech.Verb + && (kanjiTsuMe || kanaTsuMe)) + { + var numPart = w1.Text[..^1]; // 四つ / みっつ + var meText = w1.Text[^1..]; // 目 / め + int mid = w1.StartOffset >= 0 ? w1.StartOffset + numPart.Length : -1; + newList.Add(new WordInfo(w1) + { + Text = numPart, DictionaryForm = numPart, NormalizedForm = numPart, + PartOfSpeech = PartOfSpeech.Noun, + Reading = w1.Reading.Length > 0 ? w1.Reading[..^1] : "", EndOffset = mid + }); + newList.Add(new WordInfo + { + Text = meText, DictionaryForm = "目", NormalizedForm = "目", + PartOfSpeech = PartOfSpeech.Suffix, Reading = "メ", + PreMatchedWordId = 1604890, PreMatchedReadingIndex = 0, HardPinned = true, + StartOffset = mid, EndOffset = w1.EndOffset + }); + i += 1; + continue; + } } // 化け物どもめ etc.: Sudachi cuts the plural+derogatory suffix run ども+め as prefix ど + @@ -1712,7 +2346,7 @@ private List ProcessSpecialCases(List wordInfos) // (カティ+ア = カティア, kept as an OOV name token.) if (w1.PartOfSpeech is PartOfSpeech.PrenounAdjectival or PartOfSpeech.Noun && w1.Text.Length >= 2 - && IsKatakanaTextChar(w1.Text[0]) && w1.Text[0] != 'ー' + && JapaneseTextHelper.IsKatakanaWordChar(w1.Text[0]) && w1.Text[0] != 'ー' && w1.Text[1..].All(c => c is >= '぀' and <= 'ゟ') && CaseParticles.Contains(w1.Text[1..]) && KanaConverter.ToHiragana(w1.Text) == w1.DictionaryForm @@ -2734,12 +3368,20 @@ w2.DictionaryForm is "いう" or "言う" bool doNiBlocked = w1.Text == "度" && w2.Text == "に" && !(i > 0 && wordInfos[i - 1].PartOfSpeech == PartOfSpeech.Verb); + // つ+か → つか (つーか) must not steal the counter つ from a preceding numeral + // (三つ|か四つ); right after a number the counter reading is the only possible one. + bool tsuKaBlocked = w1.Text == "つ" && w2.Text == "か" && i > 0 && + (wordInfos[i - 1].PartOfSpeech == PartOfSpeech.Numeral || + (wordInfos[i - 1].NormalizedForm.Length > 0 && + wordInfos[i - 1].NormalizedForm.All(char.IsAsciiDigit))); + foreach (var sc in sc2Candidates) { if (sc.Second == "い" && kaIBlocked) continue; if (sc.Second == "で" && tokoroDeBlocked) continue; if (sc.Second == "じゃ" && soreJaBlocked) continue; if (sc.Second == "に" && doNiBlocked) continue; + if (sc.Second == "か" && tsuKaBlocked) continue; if (w2.Text == sc.Second && !(sc.Pos == PartOfSpeech.Verb && w1.PartOfSpeech == PartOfSpeech.Conjunction)) { @@ -3204,8 +3846,6 @@ private static bool IsClauseFinalBeforeKana(WordInfo w) => || (w.PartOfSpeech is PartOfSpeech.Verb && w.Text == w.DictionaryForm) || (w.PartOfSpeech == PartOfSpeech.Auxiliary && w.Text is "ない" or "た" or "だ" or "です" or "ます" or "てる" or "でる"); - private static bool IsKatakanaTextChar(char c) => c is (>= 'ァ' and <= 'ヺ') or 'ー'; - // True when text deconjugates in exactly one step to a clause-final conjugation // (imperative/volitional) of a real JMDict word. Stem/infinitive chains are rejected so // genuine te-forms (信じきって) never match while quotative re-cuts (信じろ+って) do. @@ -3325,18 +3965,51 @@ int WordReattachRunLength(string add) { // An auxiliary stem in the run is a verb/polite ending (ませ+ん = ません), not a stolen noun // mora — leave it to RepairN/CombineAuxiliary, so わかってませんって is not mis-merged through ません. - // The case particle が never belongs inside a reattached noun: stop so が + 行って is not glued - // into the kana-row noun が行 (気が行って) and the verb keeps its mora. - if (result[^k].PartOfSpeech == PartOfSpeech.Auxiliary - || result[^k] is { PartOfSpeech: PartOfSpeech.Particle, Text: "が" }) break; + // The case/quotative particles が・と・を・に never belong inside a reattached noun: stop so + // が+行って/と+いって/に+かぶって keep their verbs (が行, 問い, にかぶ would otherwise attest). Topic は likewise — + // except clause-initially, where a bare は can only be a shredded word head (は|ずっ|て = はず). + // A Verb token's tail belongs to the verb machinery (生き|て|い|こうっ|て must + // reassemble as 生きていこう, not steal いこう as a noun) — never walk into it. + // An っ-final "verb" (いっ in いっ|しょっ|て) is itself a shred, not a stem — it + // passes; so does a bare す (a shredded する/すぎ stem: びびり|す|ぎっ). す can also + // be a genuine contracted する stem (すんだ), so walking through it is safe only + // because the lookup gate below must still attest the reformed word. + if ((result[^k].PartOfSpeech == PartOfSpeech.Verb + && !result[^k].Text.EndsWith('っ') && result[^k].Text != "す") + || result[^k].PartOfSpeech == PartOfSpeech.Auxiliary + || result[^k] is { PartOfSpeech: PartOfSpeech.Particle, Text: "が" or "と" or "を" or "に" }) break; + if (result[^k] is { PartOfSpeech: PartOfSpeech.Particle, Text: "は" }) + { + bool clauseInitial = result.Count <= k + || result[^(k + 1)].PartOfSpeech is PartOfSpeech.Symbol + or PartOfSpeech.SupplementarySymbol or PartOfSpeech.BlankSpace; + if (!clauseInitial) break; + } var t = result[^k].Text; if (t.Length == 0) break; + // The particle で heads exactly one shred — the copula です (で|すっ|て); anything + // else crossing it is two real tokens meeting (で+も=デモ, で+すく=デスク). + if (result[^k] is { PartOfSpeech: PartOfSpeech.Particle, Text: "で" } && t + acc != "です") break; bool kanaOrKanji = true; foreach (var c in t) if (c is not ((>= 'ぁ' and <= 'ゖ') or (>= '゠' and <= 'ヿ') or (>= '一' and <= '鿿'))) { kanaOrKanji = false; break; } if (!kanaOrKanji) break; acc = t + acc; - if (HasNonNameCompoundLookup(acc)) best = k; + if (HasNonNameCompoundLookup(acc)) + { + // A long all-kana result whose run pieces are each complete standalone words + // (どう+こう) is two words meeting, not a shred reforming — a real shred run + // has at least one fragment with no entry of its own (いっ in いっ+しょ). + // An っ-final piece (あっ, いっ) never counts as complete: it is shred-shaped + // even when an interjection homograph attests. + bool allPiecesAreWords = acc.Length >= 4 && acc.All(c => c is >= 'ぁ' and <= 'ゖ'); + if (allPiecesAreWords) + for (int m = 1; m <= k && allPiecesAreWords; m++) + allPiecesAreWords = !result[^m].Text.EndsWith('っ') + && HasNonNameCompoundLookup(result[^m].Text); + if (!allPiecesAreWords) + best = k; + } } // Decline if the chosen run is immediately preceded by a lone single-kana Noun/Symbol token: // consolidating the run into a complete word strands that kana, and the downstream stutter @@ -3446,30 +4119,30 @@ bool IsVerbReattachment(string s) => - // いたって[Adverb] homograph: after a て-form (te-iru している→していた) and before いう it is いた - // (past of いる) + quotative って, not the adverb いたって ("extremely"). Split so the て-form keeps - // its いた and って clusters into っていう. Gated on a preceding て-form, which the genuine adverb - // (いたって元気) never has, and on a following いう. + // いたって[Adverb] homograph: before a 言う-form and after a て-form (していた), a case + // particle or a topic (部署に|いた, 人が|いた) it is いた (past of いる) + quotative って, + // not the adverb いたって ("extremely") — the adverb modifies a following predicate, never + // a verb of saying. って is emitted standalone; CombineQuotativeToIu clusters it with いう. if (result.Count > 0 && wordInfos[i] is { Text: "いたって", PartOfSpeech: PartOfSpeech.Adverb } - && i + 1 < wordInfos.Count && wordInfos[i + 1] is { Text: "いう", DictionaryForm: "いう" } + && i + 1 < wordInfos.Count + && wordInfos[i + 1].DictionaryForm is "いう" or "言う" && (result[^1].Text.EndsWith("て", StringComparison.Ordinal) - || result[^1].Text.EndsWith("で", StringComparison.Ordinal))) + || result[^1].Text.EndsWith("で", StringComparison.Ordinal) + || result[^1] is { PartOfSpeech: PartOfSpeech.Particle, Text: "が" or "に" or "は" or "も" })) { var fused = wordInfos[i]; - var iu = wordInfos[i + 1]; int mid = fused.StartOffset >= 0 ? fused.StartOffset + 2 : -1; result.Add(new WordInfo(fused) { Text = "いた", DictionaryForm = "いる", NormalizedForm = "いる", Reading = "イタ", PartOfSpeech = PartOfSpeech.Verb, EndOffset = mid }); - result.Add(new WordInfo(iu) + result.Add(new WordInfo(fused) { - Text = "っていう", DictionaryForm = "っていう", NormalizedForm = "っていう", Reading = "ッテイウ", - PartOfSpeech = PartOfSpeech.Conjunction, StartOffset = mid, EndOffset = iu.EndOffset + Text = "って", DictionaryForm = "って", NormalizedForm = "って", Reading = "ッテ", + PartOfSpeech = PartOfSpeech.Particle, StartOffset = mid }); - i++; changed = true; continue; } @@ -3486,7 +4159,7 @@ bool IsVerbReattachment(string s) => { var cur = wordInfos[i].Text; int kl = 0; - while (kl < cur.Length && IsKatakanaTextChar(cur[kl])) kl++; + while (kl < cur.Length && JapaneseTextHelper.IsKatakanaWordChar(cur[kl])) kl++; if (kl >= 1 && kl < cur.Length && cur[kl] == 'っ' && HasNonNameCompoundLookup(result[^1].Text + cur[..kl])) @@ -3760,12 +4433,73 @@ or PartOfSpeech.Suffix or PartOfSpeech.Prefix } } + // Auxiliary/Interjection thieves are Sudachi's lemmatisations of stolen morae too + // (育|ちっ[Auxiliary]|て = 育ち+って, た|めっ[Interjection]|て = ため+って); the genuine + // auxiliary shapes (だっ|て, ちゃっ|て) are covered by the だって block above and the + // reattach attestation gates. + // The っつー contraction family steals morae exactly like って (びびり|す|ぎっ|つーか: + // the ぎ belongs to すぎ, the っ to っつーか). Reattach through the same run and hand + // the っ back to the contraction; only the noun-run applies — the contraction never + // continues a genuine te-form. + if (wordInfos[i].Text.Length > 1 && wordInfos[i].Text[^1] == 'っ' + && i + 1 < wordInfos.Count + && (wordInfos[i + 1].Text.StartsWith("つー", StringComparison.Ordinal) + || wordInfos[i + 1].Text.StartsWith("つう", StringComparison.Ordinal) + || wordInfos[i + 1].Text.StartsWith("ちゅう", StringComparison.Ordinal)) + && result.Count > 0) + { + var strippedMora = wordInfos[i].Text[..^1]; + if (strippedMora.All(c => c is (>= 'ぁ' and <= 'ゖ') or (>= '一' and <= '鿿'))) + { + int tsuRun = WordReattachRunLength(strippedMora); + if (tsuRun > 0) + { + MergeRunInto(tsuRun, strippedMora, wordInfos[i].EndOffset >= 0 ? wordInfos[i].EndOffset - 1 : -1); + var contraction = wordInfos[i + 1]; + result.Add(new WordInfo(contraction) + { + Text = "っ" + contraction.Text, + Reading = "ッ" + contraction.Reading, + StartOffset = wordInfos[i].EndOffset >= 0 ? wordInfos[i].EndOffset - 1 : -1, + }); + i++; + changed = true; + continue; + } + } + } + + // Fused theft, kanji head: Sudachi occasionally swallows the whole thing into one + // token (勉|強って with 強って lemmatised as たって). A kanji head before って whose + // reattach run reforms a real word (勉+強=勉強) is the same theft one merge earlier. + if (wordInfos[i].Text.Length >= 3 + && wordInfos[i].Text.EndsWith("って", StringComparison.Ordinal) + && wordInfos[i].Text[..^2].All(JapaneseTextHelper.IsKanji) + && result.Count > 0) + { + var head = wordInfos[i].Text[..^2]; + int headRun = WordReattachRunLength(head); + if (headRun > 0) + { + MergeRunInto(headRun, head, wordInfos[i].StartOffset >= 0 ? wordInfos[i].StartOffset + head.Length : -1); + result.Add(new WordInfo(wordInfos[i]) + { + Text = "って", DictionaryForm = "って", NormalizedForm = "って", Reading = "ッテ", + PartOfSpeech = PartOfSpeech.Particle, + StartOffset = wordInfos[i].StartOffset >= 0 ? wordInfos[i].StartOffset + head.Length : -1 + }); + changed = true; + continue; + } + } + if (i + 1 >= wordInfos.Count || wordInfos[i].Text.Length < 2 || wordInfos[i].Text[^1] != 'っ' || wordInfos[i + 1].Text != "て" || wordInfos[i].PartOfSpeech is not (PartOfSpeech.Verb or PartOfSpeech.Noun or PartOfSpeech.CommonNoun - or PartOfSpeech.Adverb)) + or PartOfSpeech.Adverb or PartOfSpeech.Auxiliary or PartOfSpeech.Interjection + or PartOfSpeech.Particle)) { result.Add(wordInfos[i]); continue; @@ -3846,7 +4580,7 @@ or PartOfSpeech.Suffix or PartOfSpeech.Prefix // って eating the final mora of a katakana word: デー|トっ|て → デート + って. // A mid-katakana cut before って is never a real boundary, so shape alone suffices // (covers OOV names too). - if (stripped.Length == 1 && stripped[0] != 'ー' && IsKatakanaTextChar(stripped[0]) + if (stripped.Length == 1 && stripped[0] != 'ー' && JapaneseTextHelper.IsKatakanaWordChar(stripped[0]) && prev != null && JapaneseTextHelper.IsAllKatakana(prev.Text)) { var mergedKatakana = prev.Text + stripped; @@ -3915,7 +4649,10 @@ or PartOfSpeech.Suffix or PartOfSpeech.Prefix continue; } - if (thief.PartOfSpeech is PartOfSpeech.Verb && CommonTeFormVerbs.Contains(thief.DictionaryForm)) + // する has no すっ stem (its te-form is して) — a すっ "thief" lemmatised as する is + // always a shredded word (で|すっ|て = です+って), so it doesn't earn the common-verb skip. + if (thief.PartOfSpeech is PartOfSpeech.Verb && CommonTeFormVerbs.Contains(thief.DictionaryForm) + && !(thief.DictionaryForm == "する" && thief.Text == "すっ")) { result.Add(wordInfos[i]); continue; @@ -3923,10 +4660,16 @@ or PartOfSpeech.Suffix or PartOfSpeech.Prefix // Noun-mora theft, pair shape: 繋|がりっ[Adverb]|て → 繋がり + って. The generic verb-reattach // below only accepts う-row verb endings, so a renyoukei/nominalised noun (繋がり) needs this. - // Gated to a non-Verb thief (がりっ is an Adverb; genuine verb te-forms いっ/なっ are tagged Verb) - // and to a hiragana stripped that reforms a real non-name JMDict word via lookback. - if (thief.PartOfSpeech is not PartOfSpeech.Verb - && stripped.Length >= 1 + // Verb-tagged thieves enter too (育|ちっ[散る]|て — Sudachi lemmatises the stolen mora + // as a verb): genuine te-forms are protected by the CommonTeFormVerbs skip above, the + // が/と/を/は particle stops, and the reattach attestation itself. A bare う never + // reattaches as a noun — it is the volitional of the preceding verb (生きて|いこ|うっ|て), + // which the verb machinery owns. いっ/だっ thieves are owned by their dedicated blocks + // (IuIkuDictForms, だって) — the generic run would reattach their morae onto particles + // (て+い=弟, に+だ=荷駄). + if (stripped.Length >= 1 && stripped != "う" + && !IuIkuDictForms.Contains(thief.DictionaryForm) + && thief.DictionaryForm != "だつ" && stripped.All(c => c is >= 'ぁ' and <= 'ゖ')) { int runLen = WordReattachRunLength(stripped); @@ -3957,6 +4700,36 @@ or PartOfSpeech.Suffix or PartOfSpeech.Prefix changed = true; continue; } + + // After a numeral nothing can reattach (１０ is not kana/kanji), but a counter + // single kanji standing alone is the word itself + quotative (１０|度っ|て → 度+って). + // Gated on an actual counter sense: an attested non-counter kanji after numeric-like + // material (何|言っ|て) is a genuine te-form stem, not a stolen counter. + if (stripped.Length == 1 && prev != null + && Scoring.AdjacentWordScorer.IsNumericSurface(prev.Text) + && HasCounterSenseLookup?.Invoke(stripped) == true) + { + result.Add(new WordInfo(thief) + { + Text = stripped, DictionaryForm = stripped, NormalizedForm = stripped, + Reading = thief.Reading is { Length: > 1 } r ? r[..^1] : thief.Reading, + PartOfSpeech = PartOfSpeech.Noun, + EndOffset = thief.EndOffset >= 0 ? thief.EndOffset - 1 : -1 + }); + AddTte(thief, wordInfos[i + 1]); + i++; + changed = true; + continue; + } + } + + // An Auxiliary/Interjection/Particle thief only ever repairs through a successful + // reattach — the later heuristic branches would split genuine auxiliary te-forms + // (食べ|ちゃっ|て) and particle fusions. + if (thief.PartOfSpeech is PartOfSpeech.Auxiliary or PartOfSpeech.Interjection or PartOfSpeech.Particle) + { + result.Add(wordInfos[i]); + continue; } bool shouldRepair = false; diff --git a/Jiten.Parser/Stages/MorphologicalAnalyser.RewriteRules.cs b/Jiten.Parser/Stages/MorphologicalAnalyser.RewriteRules.cs index 2ab849c5..15cd458e 100644 --- a/Jiten.Parser/Stages/MorphologicalAnalyser.RewriteRules.cs +++ b/Jiten.Parser/Stages/MorphologicalAnalyser.RewriteRules.cs @@ -11,8 +11,8 @@ namespace Jiten.Parser; // // The engine owns that boilerplate once: offsets recomputed from template text lengths, readings taken // from templates (so a re-cut head can never keep a stale reading), clone-with-PreMatched-reset, and -// conjugation recovery. Rules live in RewriteRulesTable (currently empty — migration happens in later -// steps); the table is indexed by first-token surface for a single dictionary probe per token. +// conjugation recovery. Rules live in RewriteRulesTable; the table is indexed by first-token surface +// for a single dictionary probe per token. internal enum RewritePhase { @@ -59,6 +59,9 @@ internal sealed record TokenTemplate( string? Reading = null, int? Pin = null, byte? PinReadingIndex = null, + // A hard pin is a final word decision: lookup-time compound matching must not swallow the + // token into a longer attested span (the same gate protects hand-placed hard pins). + bool HardPin = false, bool RecoverConjugations = false); // A neighbour (prev/next) guard. All specified constraints must hold (AND); Negate flips the result. @@ -66,6 +69,7 @@ internal sealed record TokenTemplate( internal sealed record ContextCond( string[]? TextAnyOf = null, string[]? TextEndsWithAnyOf = null, + string[]? TextStartsWithAnyOf = null, PartOfSpeech[]? PosAnyOf = null, bool ClauseBoundary = false, bool Negate = false); @@ -133,6 +137,33 @@ [new TokenPattern(Text: "ナシ", RequireUnpinned: false)], [new TokenTemplate("", DictForm: "なし", NormalizedForm: "なし", Pin: 1529560)]), + // Bare す stem before explanatory ん is the contracted する (すんだ = するんだ, 1157170). + // Sudachi already lemmatises it as する, but the one-mora surface cannot resolve there on + // its own and would otherwise match a junk noun (酢/素/巣). Hard: compound matching would + // re-fuse す+んだ into the attested すんだ (済んだ). + new RewriteRule("su-contracted-suru", RewritePhase.Cleanup, + [new TokenPattern(Text: "す", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["する", "為る"])], + [new TokenTemplate("", Pin: 1157170, PinReadingIndex: 1, HardPin: true)], + Next: new ContextCond(TextAnyOf: ["ん", "んだ", "んで"])), + + // Sudachi sometimes keeps the contraction fused as すん (すんの, すんな) and lemmatises it + // as する itself; unpinned, the surface can only resolve through 済む/住む lookalikes. + new RewriteRule("sun-contracted-suru", RewritePhase.Cleanup, + [new TokenPattern(Text: "すん", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["する", "為る"])], + [new TokenTemplate("", Pin: 1157170, PinReadingIndex: 1, HardPin: true)]), + + // Bare あ stem before explanatory ん is the contracted ある (あんだ = あるんだ, 1296400). + // Hard: compound matching would re-fuse あ+んだ into the attested あんだ (安打). + new RewriteRule("a-contracted-aru", RewritePhase.Cleanup, + [new TokenPattern(Text: "あ", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["ある", "有る", "在る"])], + [new TokenTemplate("", Pin: 1296400, PinReadingIndex: 2, HardPin: true)], + Next: new ContextCond(TextAnyOf: ["ん", "んだ", "んで"])), + + // The same contraction fused by Sudachi as あん (あんの, あんだろ), lemmatised as ある. + new RewriteRule("an-contracted-aru", RewritePhase.Cleanup, + [new TokenPattern(Text: "あん", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["ある", "有る", "在る"])], + [new TokenTemplate("", Pin: 1296400, PinReadingIndex: 2, HardPin: true)]), + // カンパン = the food 乾パン (1209690), not 肝斑/甲板/乾板. new RewriteRule("kanpan", RewritePhase.Cleanup, [new TokenPattern(Text: "カンパン", Pos: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun], RequireUnpinned: false)], @@ -165,6 +196,123 @@ [new TokenTemplate("", DictForm: "という", NormalizedForm: "という", Pos: [new TokenPattern(TextStartsWith: "向い", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["向く"])], [new TokenTemplate("", DictForm: "向く", NormalizedForm: "向く", Pin: 1277080, RecoverConjugations: true)]), + // 共 (noun とも) + に is the adverb 共に "together" (1234260); Sudachi leaves the two split. + new RewriteRule("tomoni", RewritePhase.Cleanup, + [new TokenPattern(Text: "共", Pos: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun], ReadingPrefix: "トモ", RequireUnpinned: false), + new TokenPattern(Text: "に", Pos: [PartOfSpeech.Particle], RequireUnpinned: false)], + [new TokenTemplate("共に", DictForm: "共に", NormalizedForm: "共に", Pos: PartOfSpeech.Adverb, Reading: "トモニ", Pin: 1234260)]), + + // 来 (来る) + the classical adnominal auxiliary たる before a noun is 来たる "coming/next" + // (1591270, きたる), not 来る taken literally with a たる aux. + new RewriteRule("kitaru", RewritePhase.Cleanup, + [new TokenPattern(Text: "来", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["来る"], RequireUnpinned: false), + new TokenPattern(Text: "たる", Pos: [PartOfSpeech.Auxiliary], RequireUnpinned: false)], + [new TokenTemplate("来たる", DictForm: "来たる", NormalizedForm: "来たる", Pos: PartOfSpeech.PrenounAdjectival, Reading: "キタル", Pin: 1591270)], + Next: new ContextCond(PosAnyOf: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun])), + + // Demonstrative adverb + した (する past) before a noun is the prenominal adnominal + // (そうした/こうした/ああした "such", 2008650/2008030/2085100), never the archaic 下物 that a + // した+もの compound merge would otherwise produce. The Next-noun guard keeps the verbal + // past (そうしたら, そうしたの?) out. + new RewriteRule("soushita", RewritePhase.Cleanup, + [new TokenPattern(Text: "そう", Pos: [PartOfSpeech.Adverb], RequireUnpinned: false), + new TokenPattern(Text: "した", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["する", "為る"], RequireUnpinned: false)], + [new TokenTemplate("そうした", DictForm: "そうした", NormalizedForm: "そうした", Pos: PartOfSpeech.PrenounAdjectival, Reading: "ソウシタ", Pin: 2008650)], + Next: new ContextCond(PosAnyOf: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun])), + new RewriteRule("koushita", RewritePhase.Cleanup, + [new TokenPattern(Text: "こう", Pos: [PartOfSpeech.Adverb], RequireUnpinned: false), + new TokenPattern(Text: "した", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["する", "為る"], RequireUnpinned: false)], + [new TokenTemplate("こうした", DictForm: "こうした", NormalizedForm: "こうした", Pos: PartOfSpeech.PrenounAdjectival, Reading: "コウシタ", Pin: 2008030)], + Next: new ContextCond(PosAnyOf: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun])), + new RewriteRule("aashita", RewritePhase.Cleanup, + [new TokenPattern(Text: "ああ", Pos: [PartOfSpeech.Adverb], RequireUnpinned: false), + new TokenPattern(Text: "した", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["する", "為る"], RequireUnpinned: false)], + [new TokenTemplate("ああした", DictForm: "ああした", NormalizedForm: "ああした", Pos: PartOfSpeech.PrenounAdjectival, Reading: "アアシタ", Pin: 2085100)], + Next: new ContextCond(PosAnyOf: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun])), + + // ほくそ笑む (2065260, to gloat): Sudachi has no entry and shreds it to ほく+そ+笑(+む); + // the mora-theft repair reforms 笑む, then this Late rule reassembles the whole verb before + // the short-kana filter would drop ほく/そ. + new RewriteRule("hokusoemu", RewritePhase.Late, + [new TokenPattern(Text: "ほく", Pos: [PartOfSpeech.Adverb], RequireUnpinned: false), + new TokenPattern(Text: "そ", RequireUnpinned: false), + new TokenPattern(Text: "笑む", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["笑む"], RequireUnpinned: false)], + [new TokenTemplate("ほくそ笑む", DictForm: "ほくそ笑む", NormalizedForm: "ほくそ笑む", Pos: PartOfSpeech.Verb, Reading: "ホクソエム", Pin: 2065260, RecoverConjugations: true)]), + + // 合 after 死 is the 合い suffix あい (死合 = しあい, a duel), not the volume unit ごう. + new RewriteRule("shiai-ai", RewritePhase.Cleanup, + [new TokenPattern(Text: "合", Pos: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun], RequireUnpinned: false)], + [new TokenTemplate("", DictForm: "合い", NormalizedForm: "合い", Reading: "アイ", Pin: 1284320)], + Prev: new ContextCond(TextAnyOf: ["死"])), + + // Bare 有り得 at a clause end is the entry 有り得 (2560320), not the verb 有り得る it deconjugates to. + new RewriteRule("ariu", RewritePhase.Cleanup, + [new TokenPattern(Text: "有り得", Pos: [PartOfSpeech.Verb], RequireUnpinned: false)], + [new TokenTemplate("", DictForm: "有り得", NormalizedForm: "有り得", Pin: 2560320)], + Next: new ContextCond(ClauseBoundary: true)), + + // 飛ばし after 首 is 飛ばす "to send flying" (1485230), not the securities-fraud noun 飛ばし (1637130). + new RewriteRule("kubi-tobashi", RewritePhase.Cleanup, + [new TokenPattern(Text: "飛ばし", Pos: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun], DictFormAnyOf: ["飛ばし"], RequireUnpinned: false)], + [new TokenTemplate("", DictForm: "飛ばす", NormalizedForm: "飛ばす", Pos: PartOfSpeech.Verb, Reading: "トバシ", Pin: 1485230, RecoverConjugations: true)], + Prev: new ContextCond(TextAnyOf: ["首"])), + + // 羽馬(surname)+車 mis-latticed a winged carriage; re-cut to 羽 + 馬車 (1471780). The exact + // two-token surface avoids treating unmarked character names as injectable fragments. + new RewriteRule("hane-basha", RewritePhase.Cleanup, + [new TokenPattern(Text: "羽馬", RequireUnpinned: false), + new TokenPattern(Text: "車", RequireUnpinned: false)], + [new TokenTemplate("羽", DictForm: "羽", NormalizedForm: "羽", Pos: PartOfSpeech.Noun, Reading: "ハネ", Pin: 1171680), + new TokenTemplate("馬車", DictForm: "馬車", NormalizedForm: "馬車", Pos: PartOfSpeech.Noun, Reading: "バシャ", Pin: 1471780)]), + + // ケダ(surname)+モノ作り mis-latticed 獣作り; re-cut to ケダモノ (獣, 1335590) + 作り. (ケダモノ alone + // resolves correctly; only the モノ作り fusion strands ケダ on the surname.) + new RewriteRule("kedamono", RewritePhase.Cleanup, + [new TokenPattern(Text: "ケダ", RequireUnpinned: false), + new TokenPattern(Text: "モノ作り", RequireUnpinned: false)], + [new TokenTemplate("ケダモノ", DictForm: "獣", NormalizedForm: "獣", Pos: PartOfSpeech.Noun, Reading: "ケダモノ", Pin: 1335590), + new TokenTemplate("作り", DictForm: "作り", NormalizedForm: "作り", Pos: PartOfSpeech.Noun, Reading: "ツクリ", Pin: 1297250)]), + + // 虚(そら)+けど+も after a demonstrative is 虚け (うつけ "fool", 2674470) + the plural suffix ども. + new RewriteRule("utsuke-domo", RewritePhase.Cleanup, + [new TokenPattern(Text: "虚", Pos: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun], RequireUnpinned: false), + new TokenPattern(Text: "けど", Pos: [PartOfSpeech.Particle], RequireUnpinned: false), + new TokenPattern(Text: "も", Pos: [PartOfSpeech.Particle], RequireUnpinned: false)], + [new TokenTemplate("虚け", DictForm: "虚け", NormalizedForm: "虚け", Pos: PartOfSpeech.Noun, Reading: "ウツケ", Pin: 2674470), + new TokenTemplate("ども", DictForm: "ども", NormalizedForm: "共", Pos: PartOfSpeech.Suffix, Reading: "ドモ")], + Prev: new ContextCond(TextAnyOf: ["この", "その", "あの", "こんな", "そんな", "あんな"])), + + // こった after an adjective is the ことだ contraction ("いいこった" = いいことだ) — resolve it + // to its own colloquial expression entry (2106260), not the verb 凝る it otherwise matches. + new RewriteRule("kotta-kotoda", RewritePhase.Cleanup, + [new TokenPattern(Text: "こった", Pos: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun], RequireUnpinned: false)], + [new TokenTemplate("", DictForm: "こった", NormalizedForm: "こった", Pos: PartOfSpeech.Expression, Reading: "コッタ", Pin: 2106260, PinReadingIndex: 0)], + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.IAdjective])), + + // ざまあみやがれ has its own expression entry ("serves you right!") that the shredded + // ざま|あみ|やがれ can never reach: compound matching probes the tail's dictionary form + // (ざまあみやがる), and only the imperative surface is attested. Reunite the whole thing. + new RewriteRule("zamaa-miyagare", RewritePhase.Late, + [new TokenPattern(Text: "ざま", RequireUnpinned: false), + new TokenPattern(Text: "あみ", RequireUnpinned: false), + new TokenPattern(Text: "やがれ", RequireUnpinned: false)], + [new TokenTemplate("ざまあみやがれ", DictForm: "ざまあみやがれ", NormalizedForm: "ざまあみやがれ", + Pos: PartOfSpeech.Expression, Reading: "ザマーミヤガレ", Pin: 2868161, PinReadingIndex: 1)]), + + // Katakana イイ is a stylistic spelling of the adjective いい — never the イラン・イラク + // abbreviation, which otherwise wins on exact surface match. + new RewriteRule("ii-katakana", RewritePhase.Cleanup, + [new TokenPattern(Text: "イイ")], + [new TokenTemplate("", DictForm: "いい", NormalizedForm: "いい", Pos: PartOfSpeech.IAdjective, + Reading: "イイ", Pin: 2820690, PinReadingIndex: 0)]), + + // 被っ* with an abstract-damage object nearby is こうむる "to suffer/incur" (損失を被った); + // the clothing かぶる keeps everything else. + new RewriteRule("koumutta", RewritePhase.Cleanup, + [new TokenPattern(TextStartsWith: "被っ", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["被る"])], + [new TokenTemplate("", DictForm: "被る", NormalizedForm: "被る", Pin: 1484340, RecoverConjugations: true)], + Window: new WindowCond(-4, -1, TextAnyOf: ["損失", "被害", "損害", "迷惑", "ダメージ", "罰", "不利益"])), + // あんた = the colloquial pronoun "you" (1979920), not the past of 編む. new RewriteRule("anta", RewritePhase.Cleanup, [new TokenPattern(Text: "あんた", RequireUnpinned: false)], @@ -227,14 +375,234 @@ [new TokenTemplate("", Pin: 1006640)], [new TokenPattern(Text: "いとおしい", DictFormAnyOf: ["いとおす", "射通す"], RequireUnpinned: false)], [new TokenTemplate("", DictForm: "いとおしい", NormalizedForm: "愛おしい", Pos: PartOfSpeech.IAdjective, Pin: 2007340)]), + // してみれば/してみりゃ after から is the discourse connective ("from …'s standpoint", + // 2407670), not the literal する conditional — 勉強をしてみれば keeps the verb. + new RewriteRule("kara-shitemireba", RewritePhase.Cleanup, + [new TokenPattern(Text: "してみれば")], + [new TokenTemplate("", DictForm: "してみれば", Pos: PartOfSpeech.Expression, Pin: 2407670, PinReadingIndex: 1)], + Prev: new ContextCond(TextAnyOf: ["から"])), + new RewriteRule("kara-shitemirya", RewritePhase.Cleanup, + [new TokenPattern(Text: "してみりゃ")], + [new TokenTemplate("", DictForm: "してみれば", Pos: PartOfSpeech.Expression, Pin: 2407670, PinReadingIndex: 1)], + Prev: new ContextCond(TextAnyOf: ["から"])), + // なかれ = the classical negative imperative 勿れ (1535750), not 無い/なし. new RewriteRule("nakare", RewritePhase.Cleanup, [new TokenPattern(Text: "なかれ", DictFormAnyOf: ["ない", "なし", "無い"], RequireUnpinned: false)], [new TokenTemplate("", DictForm: "なかれ", NormalizedForm: "なかれ", Pos: PartOfSpeech.Suffix, Pin: 1535750)]), + // Clause-initial つって/つった is the という contraction (っつう 2798260) — a quotative needs + // quoted material before it, while 釣る needs an object; mid-clause 魚をつって keeps the verb. + new RewriteRule("tsutte-quotative", RewritePhase.Cleanup, + [new TokenPattern(TextAnyOf: ["つって", "つった"], DictFormAnyOf: ["釣る", "吊る", "つる"])], + [new TokenTemplate("", DictForm: "っつう", Pos: PartOfSpeech.Particle, Pin: 2798260)], + Prev: new ContextCond(ClauseBoundary: true)), + + // っつって/っつった/つーて/っつー carry the っ/ー marks of the という contraction — never + // つて "connections" or 行く forms. + new RewriteRule("ttsutte", RewritePhase.Cleanup, + [new TokenPattern(TextAnyOf: ["っつって", "っつった", "つーて", "っつー", "っつう"])], + [new TokenTemplate("", DictForm: "っつう", Pos: PartOfSpeech.Particle, Pin: 2798260)]), + // --- Re-cuts (splits/merges). Templates carry the correct readings, so the F4 stale-reading // class cannot recur. Text is conserved (asserted at load). --- + // ない's final mora stolen by 行く in front of the という contraction or a quotative + // (できな|いっ|つー, たまんな|いっ|て): return the い and keep the contraction whole. + new RewriteRule("nai-ttsuu", RewritePhase.Early, + [new TokenPattern(Text: "な", Pos: [PartOfSpeech.Auxiliary], DictFormAnyOf: ["ない"]), + new TokenPattern(Text: "いっ", DictFormAnyOf: ["いく", "行く"]), + new TokenPattern(Text: "つー", DictFormAnyOf: ["つう"])], + [ + new TokenTemplate("ない", DictForm: "ない", NormalizedForm: "ない", Pos: PartOfSpeech.Auxiliary, Reading: "ナイ"), + new TokenTemplate("っつー", DictForm: "っつう", NormalizedForm: "っつう", Pos: PartOfSpeech.Particle, Reading: "ッツー", Pin: 2798260), + ]), + new RewriteRule("nai-tte-mora", RewritePhase.Early, + [new TokenPattern(Text: "な", Pos: [PartOfSpeech.Auxiliary], DictFormAnyOf: ["ない"]), + new TokenPattern(Text: "いっ", DictFormAnyOf: ["いく", "行く"]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("ない", DictForm: "ない", NormalizedForm: "ない", Pos: PartOfSpeech.Auxiliary, Reading: "ナイ"), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ"), + ]), + + // Clause-initial てこ+と is the てこと (ということ) contraction, not the lever 梃子 — Sudachi + // tags the contraction shape Adverb, the tool Noun. + new RewriteRule("te-koto", RewritePhase.Early, + [new TokenPattern(Text: "てこ", Pos: [PartOfSpeech.Adverb]), + new TokenPattern(Text: "と", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("て", DictForm: "て", NormalizedForm: "て", Pos: PartOfSpeech.Particle, Reading: "テ"), + new TokenTemplate("こと", DictForm: "こと", NormalizedForm: "こと", Pos: PartOfSpeech.Noun, Reading: "コト"), + ], + Prev: new ContextCond(ClauseBoundary: true)), + + // しなきゃ+って: Sudachi reads the きゃ as a scream — な(だ)+きゃっ+て(norm って) is the + // ければ-contraction なきゃ + quotative って. + new RewriteRule("nakya-tte", RewritePhase.Early, + [new TokenPattern(Text: "な", Pos: [PartOfSpeech.Auxiliary], DictFormAnyOf: ["だ"]), + new TokenPattern(Text: "きゃっ", Pos: [PartOfSpeech.Interjection]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("なきゃ", DictForm: "なきゃ", NormalizedForm: "なければ", Pos: PartOfSpeech.Auxiliary, Reading: "ナキャ"), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ"), + ]), + + // ずっと+いる shreds as ずっ[ずる]|とい[Aux] — the と belongs to the adverb, the い to いる + // (ずっといてほしい, ずっといた). + new RewriteRule("zutto-i", RewritePhase.Early, + [new TokenPattern(Text: "ずっ", DictFormAnyOf: ["ずる"]), + new TokenPattern(Text: "とい", Pos: [PartOfSpeech.Auxiliary])], + [ + new TokenTemplate("ずっと", DictForm: "ずっと", NormalizedForm: "ずっと", Pos: PartOfSpeech.Adverb, Reading: "ズット"), + new TokenTemplate("い", DictForm: "いる", NormalizedForm: "いる", Pos: PartOfSpeech.Verb, Reading: "イ"), + ]), + + // じゃ|あっ[ある]|て is the quoted interjection じゃあ + って (「じゃあってなによ」) — ある's + // common-verb protection keeps the mora-theft repair away, so the split is declared here. + new RewriteRule("jaa-tte", RewritePhase.Early, + [new TokenPattern(Text: "じゃ", Pos: [PartOfSpeech.Conjunction]), + new TokenPattern(Text: "あっ", DictFormAnyOf: ["ある", "会う"]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("じゃあ", DictForm: "じゃあ", NormalizedForm: "じゃあ", Pos: PartOfSpeech.Conjunction, + Reading: "ジャア", Pin: 1005900), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ"), + ]), + + // 連用形+てって at clause end is て + quotative って (顔出してって……); before a continuation + // (出てってくれ) the てく auxiliary survives. + new RewriteRule("tette-quotative", RewritePhase.Early, + [new TokenPattern(Text: "てっ", Pos: [PartOfSpeech.Auxiliary], DictFormAnyOf: ["てく"]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("て", DictForm: "て", NormalizedForm: "て", Pos: PartOfSpeech.Particle, Reading: "テ"), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ"), + ], + Next: new ContextCond(ClauseBoundary: true)), + + // Sudachi tags the っつ-contraction pieces with dict つう/ちゅう; the leading っ rides on the + // previous token (かっ|つー, どもっ|つっ|て, えっ|つっ|た). Give the っ back to the contraction + // before the combine stages fuse the pair into 買う/どもる lookalikes. + new RewriteRule("ka-ttsuu", RewritePhase.Early, + [new TokenPattern(Text: "かっ"), + new TokenPattern(Text: "つー", DictFormAnyOf: ["つう"])], + [ + new TokenTemplate("か", DictForm: "か", NormalizedForm: "か", Pos: PartOfSpeech.Particle, Reading: "カ"), + new TokenTemplate("っつー", DictForm: "っつう", NormalizedForm: "っつう", Pos: PartOfSpeech.Particle, Reading: "ッツー", Pin: 2798260), + ]), + new RewriteRule("ka-cchuu", RewritePhase.Early, + [new TokenPattern(Text: "かっ"), + new TokenPattern(Text: "ちゅう", Pos: [PartOfSpeech.Auxiliary], DictFormAnyOf: ["ちゅう"])], + [ + new TokenTemplate("か", DictForm: "か", NormalizedForm: "か", Pos: PartOfSpeech.Particle, Reading: "カ"), + new TokenTemplate("っちゅう", DictForm: "っちゅう", NormalizedForm: "っちゅう", Pos: PartOfSpeech.Conjunction, Reading: "ッチュウ", Pin: 2757620), + ]), + // ども stays a Suffix so a preceding noun can reclaim it (子+ども → 子ども); standalone + // it resolves as the plural suffix, which is the reading these frames carry. + new RewriteRule("domo-ttsutte", RewritePhase.Early, + [new TokenPattern(Text: "どもっ", DictFormAnyOf: ["どもる", "吃る"]), + new TokenPattern(Text: "つっ", DictFormAnyOf: ["つう"]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("ども", DictForm: "ども", NormalizedForm: "ども", Pos: PartOfSpeech.Suffix, Reading: "ドモ"), + new TokenTemplate("っつって", DictForm: "っつう", NormalizedForm: "っつう", Pos: PartOfSpeech.Particle, Reading: "ッツッテ", Pin: 2798260), + ]), + + // ねえ's stretched え read as a standalone interjection before the contraction + // (いらねえ|っつった → いらね|えっ|つっ|た): the えっ run is え + っつった. + new RewriteRule("e-ttsutta", RewritePhase.Early, + [new TokenPattern(Text: "えっ", Pos: [PartOfSpeech.Interjection]), + new TokenPattern(Text: "つっ", DictFormAnyOf: ["つう"]), + new TokenPattern(Text: "た", Pos: [PartOfSpeech.Auxiliary])], + [ + new TokenTemplate("え", DictForm: "え", NormalizedForm: "え", Pos: PartOfSpeech.Interjection, Reading: "エ"), + new TokenTemplate("っつった", DictForm: "っつう", NormalizedForm: "っつう", Pos: PartOfSpeech.Particle, Reading: "ッツッタ", Pin: 2798260), + ]), + + // ずつ+って: the ず arrives as the negative auxiliary, so the function-word gate on the + // generic rule below can't see the theft — give ずつ (2829645) its つ back. + new RewriteRule("zutsu-tte", RewritePhase.Early, + [new TokenPattern(Text: "ず"), + new TokenPattern(Text: "つっ", DictFormAnyOf: ["つう"]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("ずつ", DictForm: "ずつ", NormalizedForm: "ずつ", Pos: PartOfSpeech.Particle, Reading: "ズツ", Pin: 2829645), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ"), + ]), + + // Bare つっ(つう)+て/た is the same contraction conjugated (つっても, いらねえっつった). + // A quotative follows a completed clause, so the previous token must be a boundary or a + // function word — a content-word fragment before つっ means the つ was stolen from it + // (待|つっ|て, 撃|つっ|て, ず|つっ|て), repaired elsewhere. + new RewriteRule("tsutte-raw", RewritePhase.Early, + [new TokenPattern(Text: "つっ", DictFormAnyOf: ["つう"]), + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [new TokenTemplate("つって", DictForm: "っつう", NormalizedForm: "っつう", Pos: PartOfSpeech.Particle, Reading: "ツッテ", Pin: 2798260)], + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.Verb, PartOfSpeech.Noun, PartOfSpeech.CommonNoun, + PartOfSpeech.Name, PartOfSpeech.Pronoun, PartOfSpeech.NaAdjective, PartOfSpeech.Prefix, + PartOfSpeech.Suffix, PartOfSpeech.Numeral, PartOfSpeech.Counter], Negate: true)), + new RewriteRule("tsutta-raw", RewritePhase.Early, + [new TokenPattern(Text: "つっ", DictFormAnyOf: ["つう"]), + new TokenPattern(Text: "た", Pos: [PartOfSpeech.Auxiliary])], + [new TokenTemplate("つった", DictForm: "っつう", NormalizedForm: "っつう", Pos: PartOfSpeech.Particle, Reading: "ツッタ", Pin: 2798260)], + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.Verb, PartOfSpeech.Noun, PartOfSpeech.CommonNoun, + PartOfSpeech.Name, PartOfSpeech.Pronoun, PartOfSpeech.NaAdjective, PartOfSpeech.Prefix, + PartOfSpeech.Suffix, PartOfSpeech.Numeral, PartOfSpeech.Counter], Negate: true)), + + // 〜だって before a quote verb is copula だ + quotative って (大袈裟だって言いたい), not the + // concessive conjunction だって. Cleanup phase: both CombineTte (だっ+て) and the combine group + // (言い+たい) have finished, so the merged だって is a single token and the following quote verb + // 言う/思う is settled as one 言いたい/思う — gate the recut on a nominal host and that verb head. + new RewriteRule("datte-quotative", RewritePhase.Cleanup, + [new TokenPattern(Text: "だって", RequireUnpinned: false)], + [new TokenTemplate("だ", DictForm: "だ", NormalizedForm: "だ", Pos: PartOfSpeech.Auxiliary, Reading: "ダ"), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ", Pin: 2086960)], + // Restricted to a na-adjective predicate: 大袈裟だ/危険だ + って is copula + quotative, and + // "even exaggerated" is not a reading, so the split is unambiguous. A noun/pronoun + だって is + // the "even/too" particle far too often to split safely (子供だって思ってる = "even children + // think", 俺だって = "I too"), so those keep だって whole even before a quote verb. + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.NaAdjective]), + // The quote-verb class the copula's quotative って attaches to (言う/思う/聞く/考える/感じる). + Next: new ContextCond(PosAnyOf: [PartOfSpeech.Verb], TextStartsWithAnyOf: ["言", "思", "聞", "考", "感"])), + + // Slang ねえ (= ない) before a quotative って: Sudachi shreds it to ね + えっ(interjection) + て, + // stealing the え. After a verb, reclaim ねえ as the negative and hand て back as って + // (堪らねえって, 食えねえって); the verb then folds 〜ねえ into the plain negative. + new RewriteRule("nee-tte", RewritePhase.Early, + [new TokenPattern(Text: "ね"), + new TokenPattern(Text: "えっ"), // Sudachi tags the stolen え as Interjection or Verb by context + new TokenPattern(Text: "て", Pos: [PartOfSpeech.Particle])], + [new TokenTemplate("ねえ", DictForm: "ない", NormalizedForm: "ない", Pos: PartOfSpeech.Auxiliary, Reading: "ネエ"), + new TokenTemplate("って", DictForm: "って", NormalizedForm: "って", Pos: PartOfSpeech.Particle, Reading: "ッテ", Pin: 2086960)], + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.Verb])), + + // 連用形+たっちゅう fused by the lattice into 塔頭: the dialectal という after past た + // (起こしたっちゅう); the temple noun never follows a bare 連用形. + new RewriteRule("ta-cchuu", RewritePhase.Early, + [new TokenPattern(Text: "たっちゅう")], + [ + new TokenTemplate("た", DictForm: "た", NormalizedForm: "た", Pos: PartOfSpeech.Auxiliary, Reading: "タ"), + new TokenTemplate("っちゅう", DictForm: "っちゅう", NormalizedForm: "っちゅう", Pos: PartOfSpeech.Conjunction, Reading: "ッチュウ", Pin: 2757620), + ], + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.Verb])), + + // じゃ (conjunction "well then") + a bare interjection あ is the drawn-out conjunction じゃあ + // (1005900) — a genuine interjection あ after じゃ is set off by punctuation. + new RewriteRule("jaa", RewritePhase.Cleanup, + [new TokenPattern(Text: "じゃ", Pos: [PartOfSpeech.Conjunction]), + new TokenPattern(Text: "あ", Pos: [PartOfSpeech.Interjection])], + [new TokenTemplate("じゃあ", DictForm: "じゃあ", NormalizedForm: "じゃあ", Pos: PartOfSpeech.Conjunction, + Reading: "ジャア", Pin: 1005900, PinReadingIndex: 0)]), + + // Copula や + emphatic ばい only exists in Kyushu dialect after a full predicate; directly + // after a noun the sequence is the i-adjective やばい (1012840) cut by the lattice. + new RewriteRule("ya-bai", RewritePhase.Cleanup, + [new TokenPattern(Text: "や", Pos: [PartOfSpeech.Auxiliary]), + new TokenPattern(Text: "ばい", Pos: [PartOfSpeech.Particle])], + [new TokenTemplate("やばい", DictForm: "やばい", NormalizedForm: "やばい", Pos: PartOfSpeech.IAdjective, + Reading: "ヤバイ", Pin: 1012840, PinReadingIndex: 0)]), + // 何かって is 何か + quotative って, not the adverb かつて. new RewriteRule("nani-katte", RewritePhase.Cleanup, [new TokenPattern(Text: "かって", RequireUnpinned: false)], @@ -295,6 +663,30 @@ [new TokenPattern(Text: "左手首", RequireUnpinned: false)], new TokenTemplate("す", DictForm: "する", NormalizedForm: "する", Pos: PartOfSpeech.Verb, Reading: "ス", Pin: 1157170), ]), + // A sentence-final ね(え) can only follow a finite form; after the 仮定形 なら the sequence + // is the slang negative of 成る (鼻持ちならねえ, 我慢ならねえ). Restore the IAdjective shape + // Sudachi itself produces for other ねえ negatives, so tail deconjugation and expression + // matching see 〜ならない. Early phase so the combine stages treat it like any negative. + new RewriteRule("nara-nee", RewritePhase.Early, + [new TokenPattern(Text: "なら", Pos: [PartOfSpeech.Auxiliary], DictFormAnyOf: ["だ"]), + new TokenPattern(Text: "ねえ", Pos: [PartOfSpeech.Particle])], + [ + new TokenTemplate("なら", DictForm: "成る", NormalizedForm: "成る", Pos: PartOfSpeech.Verb, Reading: "ナラ"), + new TokenTemplate("ねえ", DictForm: "ねえ", NormalizedForm: "無い", Pos: PartOfSpeech.IAdjective, Reading: "ネエ"), + ]), + + // 面さ lemmatised as 面す is the する-verb mizenkei, which is real only before a passive/ + // causative auxiliary; after a noun with no れる/せる continuation the cut is the face + // suffix 面 (づら) + particle さ. Early phase so the suffix can rejoin its noun downstream. + new RewriteRule("tsura-sa", RewritePhase.Early, + [new TokenPattern(Text: "面さ", Pos: [PartOfSpeech.Verb], DictFormAnyOf: ["面す", "面する"])], + [ + new TokenTemplate("面", DictForm: "面", NormalizedForm: "面", Pos: PartOfSpeech.Suffix, Reading: "ヅラ"), + new TokenTemplate("さ", DictForm: "さ", NormalizedForm: "さ", Pos: PartOfSpeech.Particle, Reading: "サ"), + ], + Prev: new ContextCond(PosAnyOf: [PartOfSpeech.Noun, PartOfSpeech.CommonNoun]), + Next: new ContextCond(TextAnyOf: ["れ", "れる", "れた", "せ", "せる"], Negate: true)), + // あ (interjection) directly against いつも is the stolen-mora あいつ + も, not an exclamation // (a genuine interjection あ is set off by punctuation). Runs in the Late phase, where the // mora-theft repairs live — replaces the RepairInterjectionPronounTheft stage. @@ -484,6 +876,9 @@ private static bool MatchesContext(WordInfo? neighbour, ContextCond cond) if (cond.TextEndsWithAnyOf != null) ok &= neighbour != null && Array.Exists(cond.TextEndsWithAnyOf, s => neighbour.Text.EndsWith(s, StringComparison.Ordinal)); + if (cond.TextStartsWithAnyOf != null) + ok &= neighbour != null && Array.Exists(cond.TextStartsWithAnyOf, + s => neighbour.Text.StartsWith(s, StringComparison.Ordinal)); if (cond.PosAnyOf != null) ok &= neighbour != null && Array.IndexOf(cond.PosAnyOf, neighbour.PartOfSpeech) >= 0; if (cond.ClauseBoundary) @@ -578,6 +973,8 @@ private List BuildOutputs(RewriteRule rule, List tokens, int PreMatchedReadingIndex = t.Pin != null ? t.PinReadingIndex : null, PreMatchedCandidateWordIds = null, PreMatchedConjugations = null, + PinnedByRewriteRule = t.Pin != null, + HardPinned = t.Pin != null && t.HardPin, }; if (t.RecoverConjugations && t.Pin != null) w.PreMatchedConjugations = PinnedConjugationProcess(surface, w.DictionaryForm); @@ -619,7 +1016,7 @@ private static void AssignOffsets(List outputs, int spanStart, int spa } // Test hook: run an arbitrary rule table over a token list (validated first), bypassing the static - // empty table so the engine can be exercised before any rules are migrated. + // table so the engine can be exercised with focused synthetic rules. internal List ApplyRewriteRulesForTesting(List input, RewriteRule[] rules, RewritePhase phase) { var index = BuildRewriteIndex(rules).GetValueOrDefault(phase); diff --git a/Jiten.Parser/Stages/MorphologicalAnalyser.SplitStages.cs b/Jiten.Parser/Stages/MorphologicalAnalyser.SplitStages.cs index cc3ed698..50ee1258 100644 --- a/Jiten.Parser/Stages/MorphologicalAnalyser.SplitStages.cs +++ b/Jiten.Parser/Stages/MorphologicalAnalyser.SplitStages.cs @@ -327,7 +327,11 @@ private List SplitUnresolvablePrefixedAdjectives(List wordIn if (word.PartOfSpeech == PartOfSpeech.IAdjective && !string.IsNullOrEmpty(dictForm) && dictForm.Length >= 3 && !HasCompoundLookup(dictForm) && - (dictForm == word.Text || !HasCompoundLookup(word.Text))) + (dictForm == word.Text || !HasCompoundLookup(word.Text)) && + // A slang surface whose NormalizedForm is the attested compound (どでけえ → どでかい) + // resolves through deconjugation — splitting would strand the prefix. + (string.IsNullOrEmpty(word.NormalizedForm) || word.NormalizedForm == dictForm || + !HasCompoundLookup(word.NormalizedForm))) { foreach (var p in AdjectivePrefixes) { @@ -881,7 +885,9 @@ private static bool IsSuruNegationOrPoliteContinuation(string text) => || text.StartsWith("ねえ", StringComparison.Ordinal) || text.StartsWith("ねぇ", StringComparison.Ordinal); - private static readonly string[] OovGrammarMarkers = ["って", "った", "のは", "のが", "のに", "ので", "んだ", "んで", "わけ", "ない"]; + // りゃ qualifies a noun blob as garbage on its own: no real noun contains that mora sequence — + // it is always a shredded conditional contraction (て|りゃもろ…) or ありゃ/こりゃ material. + private static readonly string[] OovGrammarMarkers = ["って", "った", "のは", "のが", "のに", "ので", "んだ", "んで", "わけ", "ない", "りゃ"]; private static readonly (string text, string reading, PartOfSpeech pos, PartOfSpeechSection sec)[] GrammarTokenTable = [ @@ -918,6 +924,11 @@ private static readonly (string text, string reading, PartOfSpeech pos, PartOfSp ("だろう", "ダロウ", PartOfSpeech.Auxiliary, PartOfSpeechSection.None), ("だろ", "ダロ", PartOfSpeech.Auxiliary, PartOfSpeechSection.None), ("だ", "ダ", PartOfSpeech.Auxiliary, PartOfSpeechSection.None), + // Conditional copula closing a quoted clause (…るって|なれば, …って|ならば). Without these the + // ば-final conditional is dumped as a leftover CommonNoun and aborts the verb reattachment, + // so every 〜るってなれば blob dropped whole; longest-match keeps them ahead of bare な. + ("なれば", "ナレバ", PartOfSpeech.Conjunction, PartOfSpeechSection.None), + ("ならば", "ナラバ", PartOfSpeech.Conjunction, PartOfSpeechSection.None), ("な", "ナ", PartOfSpeech.Particle, PartOfSpeechSection.SentenceEndingParticle), ("ね", "ネ", PartOfSpeech.Particle, PartOfSpeechSection.SentenceEndingParticle), ("よ", "ヨ", PartOfSpeech.Particle, PartOfSpeechSection.SentenceEndingParticle), @@ -972,6 +983,7 @@ private List TokenizeGrammarRemainder(string text, int startOffset) // (のはだな → の|肌|な, ってんだ → って|んだ). The candidate must end in a う-row kana — the // deconjugator alone over-generates (んだ/はだ both "deconjugate" to verb pasts). int verbLen = 0; + var tailPos = PartOfSpeech.Verb; if (HasNonNameCompoundLookup != null) for (int len = Math.Min(text.Length - i, 6); len > markerLen && len >= 2; len--) { @@ -983,6 +995,26 @@ private List TokenizeGrammarRemainder(string text, int startOffset) break; } + // A conjugated i-adjective tail (よかった inside りゃあ|よかった|から) would otherwise + // shred into markers (よ|か|った). Only past/conditional shapes with an attested + // deconjugated dictionary form qualify — the marker table still wins for + // grammatical clusters (んだ, はだ end in だ and never reach here). The window must + // reach a full かった form (ありがたかった is 7 chars) — a shorter cut strands かった. + if (verbLen == 0 && HasNonNameCompoundLookup != null) + for (int len = Math.Min(text.Length - i, 8); len > markerLen && len >= 3 && verbLen == 0; len--) + { + var cand = text.Substring(i, len); + if (cand[^1] is not ('た' or 'ば')) continue; + foreach (var f in Deconjugator.Instance.Deconjugate(cand)) + { + if (f.Text == cand || !HasNonNameCompoundLookup(f.Text)) continue; + if (!f.Tags.Any(t => t.StartsWith("adj", StringComparison.Ordinal))) continue; + verbLen = len; + tailPos = PartOfSpeech.IAdjective; + break; + } + } + if (verbLen > 0) { var w = text.Substring(i, verbLen); @@ -990,7 +1022,7 @@ private List TokenizeGrammarRemainder(string text, int startOffset) { Text = w, DictionaryForm = w, NormalizedForm = w, Reading = WanaKanaShaapu.WanaKana.ToKatakana(NormalizeToHiragana(w)), - PartOfSpeech = PartOfSpeech.Verb, + PartOfSpeech = tailPos, StartOffset = startOffset >= 0 ? startOffset + i : -1, EndOffset = startOffset >= 0 ? startOffset + i + verbLen : -1 }); @@ -1077,6 +1109,61 @@ private List SplitOovGarbageTokens(List wordInfos) continue; } + // ている-conditional contraction shredded into an OOV blob: [verb-stem][て/で][りゃ(あ)…blob]. + // Reform て+りゃ(あ) as one auxiliary token (てりゃ = ていれば, which the deconjugator knows), + // so CombineInflections folds it into the verb chain (相手にし|て|りゃあ… → 相手にしてりゃあ), + // and re-tokenize the rest of the blob. Same shape as the るって repair above. + if (word.Text.StartsWith("りゃ", StringComparison.Ordinal) + && prev is { PartOfSpeech: PartOfSpeech.Particle, Text: "て" or "で" } + && result.Count >= 2 + && result[^2].PartOfSpeech is PartOfSpeech.Verb or PartOfSpeech.IAdjective or PartOfSpeech.Auxiliary) + { + int cut = word.Text.Length >= 3 && word.Text[2] == 'あ' ? 3 : 2; + var remainder = word.Text[cut..]; + int boundary = word.StartOffset >= 0 ? word.StartOffset + cut : -1; + // A remainder the lookups attest whole (もろ in 見て|りゃもろ) is one word — the + // grammar tokenizer would eat its particle-homograph first mora (も|ろ). + var grammarTail = remainder.Length == 0 + ? [] + : HasNonNameCompoundLookup?.Invoke(remainder) == true + ? + [ + new WordInfo + { + Text = remainder, DictionaryForm = remainder, NormalizedForm = remainder, + Reading = WanaKanaShaapu.WanaKana.ToKatakana(NormalizeToHiragana(remainder)), + PartOfSpeech = PartOfSpeech.Noun, + StartOffset = boundary, + EndOffset = word.EndOffset + } + ] + : TokenizeGrammarRemainder(remainder, boundary); + // A leftover the lookups attest (もろ in 見て|りゃもろ) is a real word, not evidence + // of a bad cut — only unattested leftovers abort. + bool badLeftover = remainder.Length > 0 && + (grammarTail.Count == 0 || grammarTail.Any(t => + t.PartOfSpeech == PartOfSpeech.Noun && + t.PartOfSpeechSection1 == PartOfSpeechSection.CommonNoun && + t.Text is not ("わけ" or "こと") && + HasNonNameCompoundLookup?.Invoke(t.Text) != true)); + if (!badLeftover) + { + var contraction = word.Text[..cut]; + result[^1] = new WordInfo(prev) + { + Text = prev.Text + contraction, + DictionaryForm = prev.Text + contraction, + NormalizedForm = prev.Text + contraction, + Reading = prev.Reading + WanaKanaShaapu.WanaKana.ToKatakana(contraction), + PartOfSpeech = PartOfSpeech.Auxiliary, + EndOffset = boundary + }; + result.AddRange(grammarTail); + changed = true; + continue; + } + } + // When prev is a bound Suffix (Sudachi split a verb's kanji stem, e.g. 頑|張), reattaching the // stolen mora to the suffix alone strands the leading kanji (張れ, 頑 orphaned). If the FULL run // prev2+prev+leadingMora is a real JMDict compound verb (頑張れ→頑張る 1217700), reform it across diff --git a/Jiten.Parser/Stages/TokenStage.cs b/Jiten.Parser/Stages/TokenStage.cs index 82acccde..065ff71b 100644 --- a/Jiten.Parser/Stages/TokenStage.cs +++ b/Jiten.Parser/Stages/TokenStage.cs @@ -1,3 +1,4 @@ +using Jiten.Core; using Jiten.Core.Data; namespace Jiten.Parser; @@ -45,6 +46,12 @@ internal enum TokenFeatures : uint DictKiru = 1 << 22, HiraganaOovBlob = 1 << 23, AdverbEndsTo = 1 << 24, + TextTobashi = 1 << 25, + VerbKaeru = 1 << 26, + GeminateSuffixShape = 1 << 27, + KatakanaRun = 1 << 28, + CompoundBoundaryShape = 1 << 29, + SingleKanjiNoun = 1u << 30, // Composite InflectableBase = 1 << 18, @@ -54,23 +61,57 @@ internal sealed class TokenStage( string name, TokenStageGroup group, Func, List> process, - TokenFeatures requiredFeatures = TokenFeatures.None) + TokenFeatures requiredFeatures = TokenFeatures.None, + Func, IReadOnlyList, List>? candidateProcess = null) { public string Name { get; } = name; public TokenStageGroup Group { get; } = group; public TokenFeatures RequiredFeatures { get; } = requiredFeatures; - public List Apply(List input) => process(input); + public bool UsesCandidatePositions => candidateProcess != null; + public List Apply(List input, TokenFeatureScan? scan = null) => + candidateProcess == null + ? process(input) + : candidateProcess(input, + (scan ?? TokenFeatureScanner.ScanWithCandidates(input)).Candidates(RequiredFeatures)); +} + +internal sealed class TokenFeatureScan(TokenFeatures features, Dictionary> candidates) +{ + public TokenFeatures Features { get; } = features; + + public IReadOnlyList Candidates(TokenFeatures feature) => + candidates.TryGetValue(feature, out var positions) ? positions : []; } internal static class TokenFeatureScanner { - public static TokenFeatures Scan(List tokens) + public static TokenFeatures Scan(List tokens) => ScanCore(tokens, null); + + public static TokenFeatureScan ScanWithCandidates(List tokens) + { + var candidates = new Dictionary>(4); + return new TokenFeatureScan(ScanCore(tokens, candidates), candidates); + } + + private static TokenFeatures ScanCore(List tokens, + Dictionary>? candidates) { var f = TokenFeatures.None; string prevText = ""; - foreach (var w in tokens) + void AddCandidate(TokenFeatures feature, int position) { + f |= feature; + if (candidates == null) return; + if (!candidates.TryGetValue(feature, out var positions)) + candidates[feature] = positions = []; + if (positions.Count == 0 || positions[^1] != position) + positions.Add(position); + } + + for (int index = 0; index < tokens.Count; index++) + { + var w = tokens[index]; switch (w.PartOfSpeech) { case PartOfSpeech.Prefix: f |= TokenFeatures.Prefix; break; @@ -92,6 +133,30 @@ public static TokenFeatures Scan(List tokens) f |= TokenFeatures.LongVowelMark; if (text.Length > 0 && text[^1] == 'っ') f |= TokenFeatures.EndsWithTsu; + + // Candidate positions are needed only immediately before the structural block. Normal + // feature rescans skip these extra string walks; the pipeline requests a candidate scan + // before it evaluates the four stages. + if (candidates != null) + { + if (text.EndsWith('っ') || text.EndsWith("っぱ", StringComparison.Ordinal) + || text.EndsWith("っぷ", StringComparison.Ordinal)) + AddCandidate(TokenFeatures.GeminateSuffixShape, index); + if (text.Length > 0 && text.All(JapaneseTextHelper.IsKatakanaWordChar)) + AddCandidate(TokenFeatures.KatakanaRun, index); + if (text.Length is 2 or 3 && "上中内外前後間下先際的".Contains(text[^1])) + { + if (index > 0) + AddCandidate(TokenFeatures.CompoundBoundaryShape, index - 1); + } + if (w.PartOfSpeech == PartOfSpeech.Expression && text.Length == 3 + && text[..2] is "その" or "この" or "あの" or "どの" + && JapaneseTextHelper.IsKanji(text[2])) + AddCandidate(TokenFeatures.CompoundBoundaryShape, index); + if (text.Length == 1 && JapaneseTextHelper.IsKanji(text[0]) + && w.PartOfSpeech is PartOfSpeech.Noun or PartOfSpeech.CommonNoun) + AddCandidate(TokenFeatures.SingleKanjiNoun, index); + } // Fused-mora theft where って rides inside a kana/kanji-headed token (ケン|カって, エリ|アっての, // 結|果って[果て], 偶|然って[然て], 寒|さって[さて], 婆|さ|んって[んて]), handled by RepairQuotativeTte // alongside the two-token Xっ|て shape. Contains (not EndsWith) catches an idiom-fused tail too @@ -137,11 +202,25 @@ public static TokenFeatures Scan(List tokens) case "さっ": f |= TokenFeatures.TextSakki; break; + // 飛ばし heads the counter-compound 段飛ばし; RepairDanTobashi re-checks the numeral+段 head. + case "飛ばし": + f |= TokenFeatures.TextTobashi; + break; + // The emphatic prefix ど arrives as Adverb (truncated どう) — CombinePrefixes + // re-checks precisely. + case "ど" when w.PartOfSpeech == PartOfSpeech.Adverb: + f |= TokenFeatures.Prefix; + break; } if (w.DictionaryForm == "切る") f |= TokenFeatures.DictKiru; + // A verb normalised to X返る is a candidate intensifier compound; RepairIntensifierKaeru + // splits only the OOV ones (a real 静まり返る is a JMDict entry and is excluded there). + if (w.PartOfSpeech == PartOfSpeech.Verb && w.NormalizedForm.EndsWith("返る", StringComparison.Ordinal)) + f |= TokenFeatures.VerbKaeru; + // SplitUnattestedToAdverbs only acts on adverb tokens ending in と (凛と, 堂々と). if (w.PartOfSpeech == PartOfSpeech.Adverb && text.Length >= 2 && text[^1] == 'と') f |= TokenFeatures.AdverbEndsTo; diff --git a/Jiten.Tests/FormSelectionTests.cs b/Jiten.Tests/FormSelectionTests.cs index 4dede32a..f43312b1 100644 --- a/Jiten.Tests/FormSelectionTests.cs +++ b/Jiten.Tests/FormSelectionTests.cs @@ -1646,6 +1646,187 @@ public static IEnumerable FormSelectionCases() yield return ["ピアノはもう弾かない。", "弾かない", 1419370, (byte)0]; // negative = play yield return ["自力で飛んだ。", "飛んだ", 1429700, (byte)0]; // 飛ぶ past after で yield return ["麻雀に負けて札を渡した。", "札", 1298960, (byte)0]; // banknote: no card verb + + // 話 after an ordinal/numeral is the episode counter, not はなし + yield return ["聞き耳ラジオ、第二話", "話", 2100460, (byte)0]; + yield return ["１０度って言われた", "度", 1445160, (byte)0]; + yield return ["小さな石を拾った", "石", 1382440, (byte)0]; + // Bare す stem before explanatory ん is contracted する; the lexical 済んだ/澄んだ stay themselves + yield return ["なんでロボット相手に八つ当たりなんかすんだよ！", "す", 1157170, (byte)1]; + yield return ["もう済んだことだ", "済んだ", 1295070, (byte)0]; + yield return ["空気が澄んだ朝", "澄んだ", 1373680, (byte)0]; + // Bare-りゃ contracted conditionals on single-mora stems (みりゃ = 見れば, すりゃ = すれば) + yield return ["見りゃわかるだろ", "見りゃ", 1259290, (byte)0]; + yield return ["そんなことすりゃ怒られる", "すりゃ", 1157170, (byte)1]; + yield return ["安くうりゃ売れる", "うりゃ", 1473950, (byte)1]; + // Attested mimetic adverbs in the Xと+verb frame survive the SFX gate: katakana spellings + // of hiragana-listed entries attest via the script fold, and an adverbial sense counts + // even when the entry lists a noun POS first + yield return ["ルーシィは玄関までスタスタと出迎えに行った。", "スタスタ", 1006090, (byte)0]; + yield return ["ぽつぽつと店頭に明かりを灯す", "ぽつぽつ", 1012010, (byte)1]; + yield return ["からからと笑う", "からから", 1003000, (byte)0]; + yield return ["ぽんぽんと飛び出る", "ぽんぽん", 2849068, (byte)0]; + // OOV katakana boundaries: attested tail word survives the blob; prefix re-cuts keep + // attested remainders + yield return ["彼のガゾルニアレベルは高い", "レベル", 1145910, (byte)0]; + yield return ["おバカメーターが振り切れた", "メーター", 1132530, (byte)0]; + // Contracted ある before explanatory ん (あんだ = あるんだ, never 安打) + yield return ["可愛いとこあんだな", "あ", 1296400, (byte)2]; + yield return ["そんなことあんの？", "あん", 1296400, (byte)2]; + yield return ["日本国内の話", "国内", 1286930, (byte)0]; + // ど+えれえ deconjugates through the slang ee-form to どえらい (kana form) + yield return ["うむ何しろどえれえ大仕事だ", "どえれえ", 1623940, (byte)1]; + // に越したことはない matches through its slang ねえ tail + yield return ["用心するに越したことはねえからな", "越したことはねえ", 2195810, (byte)3]; + yield return ["その間抜け面さ", "間抜け面", 1715260, (byte)0]; + yield return ["鼻持ちならねえ", "鼻持ちならねえ", 1626080, (byte)0]; + // Particle-cluster expressions attested for the full surface match whole + yield return ["ひと足先に参るぞ", "ひと足先に", 1164420, (byte)4]; + yield return ["時には胸が痛みます", "時には", 1315860, (byte)0]; + yield return ["俺の素晴らしさを天下に称えるためだとしたら、百八ごときじゃ足りんだろう。", "としたら", 2100750, (byte)0]; + // Emphatic small-vowel insertion resolves to たっぷり, never the cross-script タプル + yield return ["お楽しみはたぁっぷり時間をかけて行こう。", "たぁっぷり", 1007240, (byte)0]; + yield return ["「確か百八発鳴らすんだよな。じゃあ百八って何の数字よ？」", "じゃあ", 1005900, (byte)0]; + // Stem+ん is the negative ぬ: 足りん folds into 足りる + yield return ["俺の素晴らしさを天下に称えるためだとしたら、百八ごときじゃ足りんだろう。", "足りん", 1404740, (byte)0]; + // Contracted conditional てりゃあ (ていれば) keeps the verb chain on 相手にする — never テリア + yield return ["今まで田舎のシケた術屋を相手にしてりゃあよかったから、何とか隠し通せていただけで……", + "相手にしてりゃあ", 2572400, (byte)0]; + yield return ["見てりゃもろバレっていうか", "見てりゃ", 1259290, (byte)0]; + yield return ["見てりゃもろバレっていうか", "もろバレ", 2670820, (byte)1]; + yield return ["鼓膜やばい", "やばい", 1012840, (byte)0]; + // やべえ resolves to its own colloquial entry (and the following よ must survive) + yield return ["やべえよ母ちゃん", "やべえ", 2827244, (byte)1]; + // Exact-surface expression entry beats the deconjugated する analysis + yield return ["もっとも僕からしてみれば、それはそれでイイんだけどね。", "してみれば", 2407670, (byte)1]; + yield return ["もっとも僕からしてみれば、それはそれでイイんだけどね。", "イイ", 2820690, (byte)0]; + yield return ["言い訳をするな。", "言い訳", 1587030, (byte)0]; + // 弾き語り is its own lexeme — the flick-reading gate must not touch it + yield return ["彼は弾き語りが得意だ。", "弾き語り", 1801680, (byte)0]; + + // という-contraction family resolves to its own entry (っつう 2798260 / っちゅう 2757620), + // never 行く/釣る/買う/つて; the fishing verb keeps 釣る behind を + yield return ["いくら僕でも、そんな芸当できないっつーの", "っつー", 2798260, (byte)0]; + yield return ["つっても、力仕事は勘弁な", "つって", 2798260, (byte)1]; + yield return ["つーかこれ、クラウドっつっても、かなり限定的", "っつって", 2798260, (byte)1]; + yield return ["ロボットが反乱を起こしたっちゅうわけじゃ", "っちゅう", 2757620, (byte)0]; + // Contraction slang resolves to the standard lexeme + yield return ["ケガさせられちゃたまんないって思うだろ", "たまんない", 1211340, (byte)1]; + yield return ["クリスがいるかもしんねえよな", "かもしんねえ", 1002970, (byte)1]; + yield return ["実家が力仕事の連中ばっかだったから", "ばっか", 2857403, (byte)0]; + + // Numeral+分/日/月 before a temporal anchor takes the duration reading; without the + // anchor (or across punctuation) the idiom/fraction homograph survives + yield return ["十分後にまた来てください。", "十分", 1335070, (byte)1]; + yield return ["五分待ってくれ。", "五分", 2039350, (byte)1]; + yield return ["三分待ったら教えて。", "三分", 1814040, (byte)1]; + yield return ["あれから一月経った。", "一月", 1162130, (byte)0]; + yield return ["三十日間の猶予がある。", "三十日", 1300670, (byte)1]; + yield return ["何分後に着くの?", "何分", 1189320, (byte)0]; + yield return ["十分に注意してください。", "十分", 1335080, (byte)0]; + + // 弾く outside a music window is はじく in every conjugation, not just the passive + yield return ["この装甲は銃弾を弾く。", "弾く", 1419360, (byte)0]; + + yield return ["三時に集合だ。", "三時", 1300520, (byte)1]; + yield return ["損失を被った投資家。", "被った", 1484340, (byte)0]; + yield return ["科学技術の粋を集めた工場。", "粋を集めた", 2836958, (byte)0]; + yield return ["粋な計らいだ。", "粋な", 1372410, (byte)0]; + + // Fraction frames resolve to the 分の一 suffix (2269760), never the N-minutes homograph; + // attested whole-fraction entries and plain durations are unaffected + yield return ["クリスの七分の一。", "分の一", 2269760, (byte)0]; + yield return ["看板の４分の１の欠片だけなのだ。", "分の１", 2269760, (byte)1]; + yield return ["全体の三分の一を占める。", "三分の一", 1949730, (byte)0]; + yield return ["収入の十分の一を貯金する。", "十分の一", 1335100, (byte)0]; + yield return ["４分後に着く。", "４分", 2863218, (byte)0]; + // Hours of the clock merge under the hour entry with the clock reading (四時=よじ, + // 九時=くじ); the four-seasons and ふたとき homographs never survive a numeral context + yield return ["そして、昼の十二時十分前に、株を買った。", "十二時", 1334960, (byte)1]; + yield return ["そして、午後四時四十七分だ。", "四時", 1307230, (byte)1]; + yield return ["そして、午後四時四十七分だ。", "分", 1502840, (byte)0]; + yield return ["九時に寝るつもりだ。", "九時", 2845349, (byte)1]; + yield return ["一時的な処置だ。", "一時的な", 1162980, (byte)0]; + + // Geminate suffixes っぱなし/っぷり/っぽい reassemble their stems (Sudachi bleeds the っ: + // 流|しっ|ぱなし, 上|が|りっぱ|な|し); 立派/ちっぽけ/やっぱなし adversarials stay intact + yield return ["「ラジオ流しっぱなしで寝てんなよ」", "っぱなし", 1008020, (byte)1]; + yield return ["不動産は上がりっぱなしだったのだ。", "っぱなし", 1008020, (byte)1]; + yield return ["彼女の気力の失いっぷりはかなり激しいみたいだ。", "っぷり", 2202980, (byte)0]; + yield return ["ざまあみやがれ山犬野郎", "ざまあみやがれ", 2868161, (byte)1]; + yield return ["「そりゃいいこった」", "こった", 2106260, (byte)0]; + yield return ["ミッチーのあまりの脳天気っぷりに面食らった。", "脳天気", 2027320, (byte)2]; + yield return ["俺の皮肉っぽい言い方に、むっとしたのがわかった。", "っぽい", 2083720, (byte)0]; + yield return ["立派な人になりたい。", "立派な", 1551790, (byte)0]; + yield return ["ちっぽけな家に住んでいる。", "ちっぽけな", 1007570, (byte)1]; + yield return ["やっぱなしだな。", "やっぱ", 2772780, (byte)0]; + + // Quotative-って mora theft: stolen morae reattach to reform the word (温室育ち, 勉強, + // 打ち上げ, はず); genuine te-forms and auxiliary fusions stay intact + yield return ["俺は温室育ちってわけだ。", "温室育ち", 1781110, (byte)0]; + yield return ["「勉強って数学だろ？」", "勉強", 1512670, (byte)0]; + yield return ["ロケットの打ち上げっていうイベントだ。", "打ち上げ", 1408690, (byte)0]; + yield return ["はずって……。これでよく引率が務まるなあ。", "はず", 1476430, (byte)2]; + yield return ["１０度？１０度って。", "度", 1445160, (byte)0]; + yield return ["「じゃあってなによ、じゃあって」", "じゃあ", 1005900, (byte)0]; + yield return ["と言ってくれた。", "言ってくれた", 1587040, (byte)0]; + yield return ["花が散っていた。", "散っていた", 1303490, (byte)0]; + yield return ["食べちゃってごめん。", "食べちゃって", 1358280, (byte)0]; + yield return ["手を取って歩いた。", "手を取って", 2402970, (byte)0]; + + // って-final auxiliary before a quote-taking verb stays quotative (大袈裟だ+って+言いたい, + // never the 大袈裟だった-style fold); だって "even/because" survives everywhere else + yield return ["「要は大袈裟だって言いたいんだよ」", "大袈裟", 1588890, (byte)1]; + // 〜だって before 言いたい is copula だ + quotative って (2086960), not the concessive conjunction. + yield return ["「要は大袈裟だって言いたいんだよ」", "って", 2086960, (byte)0]; + yield return ["静かだって聞いた。", "静か", 1381820, (byte)0]; + yield return ["だって嫌だもん。", "だって", 2643970, (byte)0]; + yield return ["本だってさ。", "だって", 2643970, (byte)0]; + // The っつー contraction family steals morae like って; the stem reattaches + yield return ["びびりすぎっつーか、逆効果っつーか", "びびりすぎ", 2096500, (byte)1]; + yield return ["びびりすぎっつーか、逆効果っつーか", "逆効果", 1227020, (byte)0]; + + // Katakana-run repair: shredded OOV compounds re-merge or drop; attested compounds and + // injected-name-adjacent tokens are untouched + yield return ["[name]フラウ[line]“リア充マジ爆発すべき”", "リア充", 2625510, (byte)0]; + yield return ["システムエラーが発生した。", "システムエラー", 2300460, (byte)0]; + yield return ["ロケットエンジンの点火実験だ。", "ロケットエンジン", 2828078, (byte)0]; + yield return ["テレビカメラが並んでいる。", "テレビカメラ", 1927120, (byte)0]; + yield return ["コーヒーメーカーを買った。", "コーヒーメーカー", 2152950, (byte)0]; + + yield return ["彼女はピアノを弾いた", "弾いた", 1419370, (byte)0]; // piano context keeps ひく + + // Demonstrative-adverb + する past before a noun is the adnominal (そうした/こうした/ああした + // "such"), not the した+もの archaic compound. Adversarial: そうしたら/そうしたの stay verbal. + yield return ["これはそうしたものに耐性がない", "そうした", 2008650, (byte)1]; + yield return ["こうした問題が起きる", "こうした", 2008030, (byte)2]; + yield return ["そうしたら困る", "そうしたら", 2084700, (byte)0]; + // 共 (とも) + に is the adverb 共に "together" + yield return ["爾子が共におらぬので寂しい", "共に", 1234260, (byte)0]; + // 来 + classical adnominal たる before a noun is 来たる "coming/next" + yield return ["来たる東征に先駆けて", "来たる", 1591270, (byte)1]; + // ほくそ笑む: Sudachi lacks the compound and shreds it; the verb reassembles whole + yield return ["こういうのをほくそ笑むってんだろう", "ほくそ笑む", 2065260, (byte)0]; + // Classical polite negative ませぬ deconjugates like ません → する, not the noun 揣摩 + yield return ["否定はしませぬ", "しませぬ", 1157170, (byte)1]; + // Polite かもしれません is its own expression (1002975); the plain かもしれない (1002970) still combines + yield return ["同じなのかもしれませんしね", "かもしれません", 1002975, (byte)1]; + // 合 after 死 is the 合い suffix あい (死合=しあい), not the volume unit ごう + yield return ["竜胆はこの死合の果てに", "合", 1284320, (byte)1]; + // 直+copula に is 直に じかに (1430690), not 直(ちょく)+に + yield return ["その熱量を直に感じ取ってみる", "直に", 1430690, (byte)0]; + // bare 有り得 at a clause end is its own entry (2560320), not 有り得る + yield return ["終了することも充分有り得", "有り得", 2560320, (byte)0]; + // 飛ばし after 首 is the verb 飛ばす (1485230), not the securities-fraud noun (1637130) + yield return ["首飛ばしの颶風", "飛ばし", 1485230, (byte)0]; + // 段飛ばし (2746000): the numeral must release 段 to the compound, not swallow it (三段|飛ばし) + yield return ["三段飛ばしで登る", "段飛ばし", 2746000, (byte)0]; + // 羽馬(surname)+車 re-cut to 羽 + 馬車 (1471780) + yield return ["羽馬車が降りるのを見た", "馬車", 1471780, (byte)0]; + // ケダ(surname)+モノ作り re-cut to ケダモノ (獣, 1335590) + 作り + yield return ["こんなケダモノ作りやがった", "ケダモノ", 1335590, (byte)5]; + // 虚+けど+も re-cut to 虚け (うつけ, 2674470) + ども + yield return ["舐めるなよこの虚けども", "虚け", 2674470, (byte)1]; } public static IEnumerable FormSelectionShouldNotMatchCases() @@ -1700,4 +1881,4 @@ public async Task FormSelectionShouldNotMatchTest(string input, string tokenText var match = results.FirstOrDefault(w => w.OriginalText == tokenText); match.Should().BeNull($"token '{tokenText}' should not match any word in '{input}'"); } -} \ No newline at end of file +} diff --git a/Jiten.Tests/MorphologicalAnalyserTests.cs b/Jiten.Tests/MorphologicalAnalyserTests.cs index ec2066f8..7adef711 100644 --- a/Jiten.Tests/MorphologicalAnalyserTests.cs +++ b/Jiten.Tests/MorphologicalAnalyserTests.cs @@ -1021,6 +1021,38 @@ public static IEnumerable SegmentationCases() yield return ["この期に及んでとぼける俺である", new[] { "この期に及んで", "とぼける", "俺", "である" }]; yield return ["何ねぼけてんのよ", new[] { "何", "ねぼけてん", "の", "よ" }]; yield return ["何ねぼけた事言ってんだ", new[] { "何", "ねぼけた", "事", "言って", "んだ" }]; + // 言っ directly after the numeric-like 何 must stay a te-form: 言っ is 言う's 促音便 stem, + // not a counter shred to re-cut as 言+って. + yield return ["いいって、何言ってんだ？", new[] { "いい", "って", "何", "言って", "んだ" }]; + // Bare す before explanatory ん is contracted する — never a slurred-negative merge (すん) + yield return ["なんでロボット相手に八つ当たりなんかすんだよ！", new[] { "なんで", "ロボット", "相手", "に", "八つ当たり", "なんか", "す", "んだ", "よ" }]; + // Bare-りゃ contracted conditionals survive on single-mora stems, including kana-written + // 売りゃ (the bare-りゃ deconjugation rule) + yield return ["見りゃわかるだろ", new[] { "見りゃ", "わかる", "だろ" }]; + yield return ["そんなことすりゃ怒られる", new[] { "そんな", "こと", "すりゃ", "怒られる" }]; + yield return ["安くうりゃ売れる", new[] { "安く", "うりゃ", "売れる" }]; + // A counter kanji standing alone after a numeral is the word itself + quotative って + yield return ["１０度って言われた", new[] { "度", "って", "言われた" }]; + yield return ["１０回って言われた", new[] { "回", "って", "言われた" }]; + // Attested mimetic adverbs in the Xと+verb frame survive the SFX gate; the unattested + // ふるふる still drops (it cannot be vocabulary) + yield return ["ぽつぽつと店頭に明かりを灯す", new[] { "ぽつぽつ", "と", "店頭", "に", "明かり", "を", "灯す" }]; + yield return ["からからと笑う", new[] { "からから", "と", "笑う" }]; + yield return ["ふるふると震える", new[] { "と", "震える" }]; + // The OOV name's trailing small vowel is its identity — it never resolves through the + // stripped form (and drops here) + yield return ["ルーシィは玄関までスタスタと出迎えに行った。", new[] { "は", "玄関", "まで", "スタスタ", "と", "出迎え", "に", "行った" }]; + // An OOV katakana blob must not swallow an attested tail word (レベル splits off, the + // unattested name head drops) + yield return ["彼のガゾルニアレベルは高い", new[] { "彼", "の", "レベル", "は", "高い" }]; + yield return ["ガゾルニアレベル", new[] { "レベル" }]; + // A prefix boundary re-cut must leave an attested remainder: the blob falls to + // resegmentation (バカ+メーター), never おバ[小母] + unattested junk + yield return ["おバカメーターが振り切れた", new[] { "お", "バカ", "メーター", "が", "振り切れた" }]; + // Two genuine long loanwords meeting resolve as the attested compound, not shred material + yield return ["システムエラーが発生", new[] { "システムエラー", "が", "発生" }]; + // A two-piece run of coincidentally-attested halves is an OOV name: merge whole and drop + yield return ["――大国スティオードの王都。", new[] { "大国", "の", "王都" }]; // 第一次/一次/第二次/二次 — ordinal counters should not be split yield return ["第一次魔王討伐", new[] { "第一次", "魔王", "討伐" }]; @@ -1736,6 +1768,27 @@ public static IEnumerable SegmentationCases() yield return ["魔法使いでもない", new[] { "魔法使い", "でもない" }]; // X史|上 re-cuts to X|史上, and 史上+初 merges into the JMDict entry yield return ["人類史上初", new[] { "人類", "史上初" }]; + // The boundary-theft re-cut needs the frequency guard: 国内 far outranks 日本国, so the + // Sudachi cut stands; 滑走路 holds its own against 路上, so the theft re-cuts. + yield return ["日本国内の話", new[] { "日本", "国内", "の", "話" }]; + yield return ["飛行機は滑走路上で待機した", new[] { "飛行機", "は", "滑走路", "上", "で", "待機した" }]; + // Bare あ / fused あん before explanatory ん is contracted ある (あんだ = あるんだ) — the + // attested homographs 安打/餡 must not re-fuse or replace it + yield return ["可愛いとこあんだな", new[] { "可愛い", "とこ", "あ", "んだ", "な" }]; + yield return ["そんなことあんの？", new[] { "そんな", "こと", "あん", "の" }]; + // Pure SFX line: the attested interjection うっ survives, the unattested ひっく drops + yield return ["うっ…ひっく…", new[] { "うっ" }]; + + // Kanji-verb shred repair: a conjugated verb split as [single kanji noun][kana tail] + // merges back when the concatenation deconjugates to an attested verb (覚|ませ → 覚ませ, + // with the quotative って preserved) + yield return ["眼を覚ませって言われた", new[] { "眼", "を", "覚ませ", "って", "言われた" }]; + yield return ["彼の言葉を信じろ", new[] { "彼", "の", "言葉", "を", "信じろ" }]; + yield return ["また明日会おう", new[] { "また明日", "会おう" }]; + // Copula tails belong to the sentence, not the noun (目+だった must not become 目立った), + // and a complete dictionary-form verb after a noun is its own word (今+いる) + yield return ["それは彼の目だった", new[] { "それ", "は", "彼", "の", "目", "だった" }]; + yield return ["彼は今いる", new[] { "彼", "は", "今", "いる" }]; // === batch 3: whitelist one-liners === yield return ["綺麗さっぱりとした気分で", new[] { "綺麗さっぱり", "と", "した", "気分", "で" }]; @@ -1972,6 +2025,27 @@ public static IEnumerable SegmentationCases() // Sudachi strands 続ける's final る onto an OOV blob るってことだろ; the verb is reformed and the // blob split — だろ must be a known grammar token or the trailing ろ aborts the whole split. yield return ["続けるってことだろ", new[] { "続ける", "って", "こと", "だろ" }]; + // Same stranding with the conditional なれば tail: the ば-final copula must be a known grammar + // token or the reattachment aborts and the whole blob drops (any verb, not just 出る). + yield return ["見るってなれば", new[] { "見る", "って", "なれば" }]; + yield return ["出るってなれば", new[] { "出る", "って", "なれば" }]; + // Sudachi shreds ざま|あみ(編む)|やがれ; the whole surface is its own expression entry + // ("serves you right!"), unreachable via compound matching (only the imperative is attested). + yield return ["ざまあみやがれ", new[] { "ざまあみやがれ" }]; + // An OOV X返る intensifier compound (Sudachi norm のさばり返る) has no JMDict entry and dropped + // whole; split into head verb + 返る, folding the following て into 返って (not a quotative って). + yield return ["のさばりかえっている", new[] { "のさばり", "かえっている" }]; + // A na-adjective predicate + だって before a quote verb (言う/思う/聞く/考える/感じる) is + // copula だ + quotative って — "even exaggerated" is not a reading, so the split is unambiguous. + yield return ["大袈裟だって言いたい", new[] { "大袈裟", "だ", "って", "言いたい" }]; + yield return ["大袈裟だって聞いた", new[] { "大袈裟", "だ", "って", "聞いた" }]; + // …but a noun/pronoun + だって is the "even/too" particle far too often to split (子供だって = + // "even children", 俺だって = "I too"), so those keep だって whole even before a quote verb. + yield return ["子供だって思ってる", new[] { "子供", "だって", "思ってる" }]; + yield return ["俺だって思うことがある", new[] { "俺", "だって", "思う", "ことがある" }]; + yield return ["子供だって知ってる", new[] { "子供", "だって", "知ってる" }]; + // Slang ねえ (= ない) shredded by a following quotative って — reclaim ねえ, hand て back as って. + yield return ["堪らねえって", new[] { "堪らねえ", "って" }]; // SFX/onomatopoeia fragments: an isolated kana burst (quote/exclamation-bounded, sokuon-clipped) // or the quotative-mimetic frame X！と / X、と is phonetic material — a content-noun homograph @@ -2088,7 +2162,8 @@ public static IEnumerable SegmentationCases() yield return ["ずがんっ！！", new string[] { }]; // ない + quotative って must not let いく steal ない's final mora (出られな|いって) - yield return ["「出られないってことか？」", new[] { "出られ", "ない", "って", "こと", "か" }]; + // ない folds into the potential like any negative; って must still not steal its い + yield return ["「出られないってことか？」", new[] { "出られない", "って", "こと", "か" }]; // Kana からだ directly after a predicate is から + だ "because it is"; 体つき compounds // and the body noun elsewhere stay intact yield return ["逃げたのは怖かったからだ。", new[] { "逃げた", "の", "は", "怖かった", "から", "だ" }]; @@ -2116,6 +2191,62 @@ public static IEnumerable SegmentationCases() // いじらしい思いで = 思い + で after an adjective, not the memory noun 思い出 yield return ["ゆにが、いじらしい思いで私を励ましてくれたように。", new[] { "に", "が", "いじらしい", "思い", "で", "私", "を", "励ましてくれた", "ように" }]; + + // Emphatic prefix ど + i-adjective forms one lexeme when JMDict attests the compound + // (ど+えれえ → どえらい); the bare ど must not drop as noise + yield return ["うむ何しろどえれえ大仕事だ", new[] { "うむ", "何しろ", "どえれえ", "大仕事", "だ" }]; + // 面さ is a Sudachi 面す(verb) lattice error before a sentence-final さ — in a nominal + // clause the reading is the compound noun 間抜け面 + particle さ + yield return ["その間抜け面さ", new[] { "その", "間抜け面", "さ" }]; + // ねえ after the 仮定形 なら is the slang negative of 成る (sentence-final ね can only + // follow a finite form); the expression matches through its ねえ tail + yield return ["鼻持ちならねえ", new[] { "鼻持ちならねえ" }]; + // Noun/particle-cluster sequences whose full surface attests a JMDict expression match whole + yield return ["ひと足先に参るぞ", new[] { "ひと足先に", "参る", "ぞ" }]; + yield return ["時には胸が痛みます", new[] { "時には", "胸が痛みます" }]; + // Emphatic small-vowel insertion collapses to the base word (たぁっぷり → たっぷり), + // never to a cross-script loanword via kana stripping + yield return ["お楽しみはたぁっぷり時間をかけて行こう。", + new[] { "お楽しみ", "は", "たぁっぷり", "時間をかけて行こう" }]; + // じゃ+あ after a clause boundary is the conjunction じゃあ, not じゃ + interjection あ + yield return ["「確か百八発鳴らすんだよな。じゃあ百八って何の数字よ？」", + new[] { "確か", "百八", "発", "鳴らす", "んだ", "よ", "な", "じゃあ", "百八", "って", "何の", "数字", "よ" }]; + // ん directly after a verb stem is the negative ぬ (the nominaliser ん needs the plain + // form); と+したら after だ is the suppositional conjunction としたら + yield return ["俺の素晴らしさを天下に称えるためだとしたら、百八ごときじゃ足りんだろう。", + new[] { "俺", "の", "素晴らしさ", "を", "天下", "に", "称える", "ため", "だ", "としたら", "百八", "ごとき", "じゃ", "足りん", "だろう" }]; + // The counter つ after a numeral must not feed the つ+か (つーか) merge + yield return ["当時の御身は三つか四つであったかな。", + new[] { "当時", "の", "御身", "は", "三つ", "か", "四つ", "であった", "かな" }]; + // An OOV katakana noun ending in a particle-shaped mora must not strip it to reveal a + // name entry (ブリタニカ is not the surname ブリタニ + か) + yield return ["ザ・ブリタニカを検索した。", new[] { "を", "検索した" }]; + // てりゃ(あ) is the contracted conditional ていれば and stays on the verb chain; the + // following よかった must survive the OOV-blob re-cut + yield return ["今まで田舎のシケた術屋を相手にしてりゃあよかったから、何とか隠し通せていただけで……", + new[] { "今まで", "田舎", "の", "シケた", "術", "屋", "を", "相手にしてりゃあ", "よかった", "から", "何とか", "隠し通せていただけ", "で" }]; + yield return ["見てりゃもろバレっていうか", new[] { "見てりゃ", "もろバレ", "っていう", "か" }]; + // や+ばい after a bare noun is the i-adjective やばい, not copula や + emphatic ばい + // A sentence-final particle after a resolved word must reach the output + yield return ["やべえよ母ちゃん", new[] { "やべえ", "よ", "母ちゃん" }]; + + // という-contraction family: っつー/っちゅう and conjugated つって keep the contraction + // whole (Sudachi's dict=つう rules the fishing verb out); ない gets its stolen い back + yield return ["いくら僕でも、そんな芸当できないっつーの", + new[] { "いくら", "僕", "でも", "そんな", "芸当", "できない", "っつー", "の" }]; + yield return ["ロボットアニメが語れるかっつー話", new[] { "ロボット", "アニメ", "が", "語れる", "か", "っつー", "話" }]; + yield return ["いやぁ、子どもっつっても、まだ幼い", new[] { "いやぁ", "子ども", "っつって", "も", "まだ", "幼い" }]; + yield return ["ロボットが反乱を起こしたっちゅうわけじゃ", + new[] { "ロボット", "が", "反乱", "を", "起こした", "っちゅう", "わけ", "じゃ" }]; + // Clause-initial てこ+と is ということ, not the lever 梃子 + yield return ["要は、皆が同じ予想している、てことか", new[] { "要は", "皆", "が", "同じ", "予想している", "て", "こと", "か" }]; + // な(だ)+きゃっ+て is the ければ-contraction なきゃ + quotative って + yield return ["なんかしなきゃって気にもなる", new[] { "なんか", "し", "なきゃ", "って", "気", "に", "も", "なる" }]; + // いたって before a 言う-form after a case particle is いた + って + yield return ["確か、今の部署にいたって言ってましたよね", + new[] { "確か", "今", "の", "部署", "に", "いた", "って", "言ってました", "よね" }]; + // 連用形+てって at clause end is て + quotative って + yield return ["一度でいいから顔出してって……", new[] { "一度", "で", "いい", "から", "顔出し", "て", "って" }]; } [Theory] diff --git a/Jiten.Tests/PipelineStagePropertyTests.cs b/Jiten.Tests/PipelineStagePropertyTests.cs index 726d1e30..19c75185 100644 --- a/Jiten.Tests/PipelineStagePropertyTests.cs +++ b/Jiten.Tests/PipelineStagePropertyTests.cs @@ -50,6 +50,42 @@ public void Pipeline_ShouldRecordAllStages_InDiagnostics() diagnostics.TokenStages.Should().HaveCount(analyser.GetPipelineStageNamesForTesting().Count); } + [Fact] + public void StructuralRepairFeatures_ShouldRecognizeEachCandidateShape() + { + var features = TokenFeatureScanner.ScanWithCandidates( + [ + Token("流しっ", PartOfSpeech.Noun), + Token("ポン", PartOfSpeech.Noun), + Token("路上", PartOfSpeech.Noun), + Token("会", PartOfSpeech.CommonNoun) + ]).Features; + + features.Should().HaveFlag(TokenFeatures.GeminateSuffixShape); + features.Should().HaveFlag(TokenFeatures.KatakanaRun); + features.Should().HaveFlag(TokenFeatures.CompoundBoundaryShape); + features.Should().HaveFlag(TokenFeatures.SingleKanjiNoun); + } + + [Fact] + public void StructuralRepairStages_ShouldBeSkipped_WhenNoCandidateShapeExists() + { + var analyser = new MorphologicalAnalyser(); + var diagnostics = new ParserDiagnostics(); + + analyser.RunPipelineForTesting([Token("ordinary", PartOfSpeech.Noun)], diagnostics); + + string[] stages = + [ + "RepairGeminateSuffixTheft", + "RepairKatakanaShreds", + "RepairCompoundBoundaryTheft", + "RepairKanjiVerbShred" + ]; + diagnostics.TokenStages.Where(s => stages.Contains(s.StageName)) + .Should().OnlyContain(s => s.Skipped); + } + [Fact] public void Pipeline_ShouldPreservePunctuationBoundary_AsStandaloneToken() { diff --git a/Jiten.Tests/TransitionRuleEngineTests.cs b/Jiten.Tests/TransitionRuleEngineTests.cs index 9662898f..38cdb3ba 100644 --- a/Jiten.Tests/TransitionRuleEngineTests.cs +++ b/Jiten.Tests/TransitionRuleEngineTests.cs @@ -327,6 +327,17 @@ public void Sfp_MergesWithPrevious_WhenLookupSucceeds() words[1].word.Text.Should().Be("明日"); } + [Fact] + public void Sfp_AfterPlainPredicate_BeforeVocative_Preserved() + { + var words = Sentence( + W("やべえ", PartOfSpeech.IAdjective), + W("よ", PartOfSpeech.Particle, "よ", PartOfSpeechSection.SentenceEndingParticle), + W("母ちゃん", PartOfSpeech.CommonNoun)); + TransitionRuleEngine.ApplyHardRules(words, NoLookup); + words.Select(w => w.word.Text).Should().Equal("やべえ", "よ", "母ちゃん"); + } + [Fact] public void Sfp_MergesWithNounPrev_PreservesNounPos_NotForcedToVerb() { diff --git a/Shared/resources/deconjugator.json b/Shared/resources/deconjugator.json index 79e54bb1..ffcc88fe 100644 --- a/Shared/resources/deconjugator.json +++ b/Shared/resources/deconjugator.json @@ -950,6 +950,14 @@ "con_tag": ["uninflectable"], "detail": "negative polite" }, + { + "type": "onlyfinalrule", + "dec_end": [""], + "con_end": ["ませぬ"], + "dec_tag": ["stem-ren"], + "con_tag": ["uninflectable"], + "detail": "negative polite (classical ぬ)" + }, { "type": "onlyfinalrule", "dec_end": [""], @@ -1542,12 +1550,20 @@ }, { "type": "stdrule", - "dec_end": ["ていれば", "でいれば"], - "con_end": ["てりゃ", "でりゃ"], + "dec_end": ["ていれば", "でいれば", "ていれば", "でいれば"], + "con_end": ["てりゃ", "でりゃ", "てりゃあ", "でりゃあ"], "dec_tag": ["uninflectable"], "con_tag": ["uninflectable"], "detail": "contracted conditional (te-ireba)" }, + { + "type": "stdrule", + "dec_end": ["れば", "れば"], + "con_end": ["りゃ", "りゃあ"], + "dec_tag": ["uninflectable"], + "con_tag": ["uninflectable"], + "detail": "contracted conditional (-reba)" + }, { "type": "stdrule", "dec_end": ["でしまう"], @@ -1636,6 +1652,14 @@ "con_tag": ["adj-i"], "detail": "slang negative" }, + { + "type": "stdrule", + "dec_end": ["しれない"], + "con_end": ["しんない"], + "dec_tag": ["adj-i"], + "con_tag": ["adj-i"], + "detail": "slang negative (shirenai)" + }, { "type": "stdrule", "dec_end": ["らなければ"],