diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 6e3ba60c..6d7542e2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -66,3 +66,8 @@ fixes). Newest entries at the bottom. already-working Windows `cp314` installs. Completing the wheel matrix is the correct fix; the next 0.3.8 patch release will carry full `cp314` coverage once CI confirms the new platforms build cleanly. + +## 2026-09-06: Preserve mixed-script words in BM25 search + +- Keep Latin words and numbers separate from adjacent Chinese, Japanese, and Korean n-grams when building SQLite FTS5 indexes and parsing keyword queries. For example, `Python数据库SQL` can be retrieved by `Python`, `SQL`, or a mixed-script query. +- Existing BM25 artifacts remain readable. Rebuild indexes containing mixed-script text to regenerate tokens that were previously joined at script boundaries. diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py index e374a46a..0ff88e65 100644 --- a/packages/leann-core/src/leann/api.py +++ b/packages/leann-core/src/leann/api.py @@ -45,12 +45,15 @@ def _fts5_cjk_ngrams(match: re.Match[str]) -> str: """Expand a CJK run into unigram and bigram tokens for SQLite FTS5.""" text = match.group() - return " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))]) + tokens = " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))]) + # Keep adjacent non-CJK words separate from the first and last n-grams. + return f" {tokens} " def _fts5_cjk_query(query: str) -> str: """Build a safe FTS5 query that requires every CJK bigram in each term.""" - tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|\w+", query.lower()) + # Unicode \w includes CJK, so exclude it from the non-CJK alternative. + tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|[^\W{_CJK_CHARACTERS}]+", query.lower()) terms = [] for token in tokens: if _CJK_RUN.fullmatch(token): diff --git a/tests/test_fts5_bm25.py b/tests/test_fts5_bm25.py index 3561a8b9..0f00aa6a 100644 --- a/tests/test_fts5_bm25.py +++ b/tests/test_fts5_bm25.py @@ -1,3 +1,4 @@ +import pytest from leann.api import Fts5BM25Index @@ -30,3 +31,45 @@ def test_fts5_bm25_keeps_legacy_database_query_format(tmp_path): assert [result.id for result in reopened.search("database")] == ["database"] finally: reopened.close() + + +@pytest.mark.parametrize("cjk_text", ["数据库", "データベース", "데이터베이스"]) +@pytest.mark.parametrize("query", ["Python", "SQL"]) +def test_fts5_bm25_preserves_words_adjacent_to_cjk(tmp_path, cjk_text, query): + db_path = tmp_path / "mixed.sqlite" + index = Fts5BM25Index(str(db_path)) + index.fit( + [ + {"id": "mixed", "text": f"Python{cjk_text}SQL"}, + {"id": "unrelated", "text": "unrelated document"}, + ] + ) + index.close() + + reopened = Fts5BM25Index(str(db_path)) + try: + assert [result.id for result in reopened.search(query)] == ["mixed"] + finally: + reopened.close() + + +@pytest.mark.parametrize("query", ["Python数据库", "数据库Python", "Python数据库SQL", "2026数据库"]) +def test_fts5_bm25_splits_mixed_script_query_terms(tmp_path, query): + db_path = tmp_path / "queries.sqlite" + index = Fts5BM25Index(str(db_path)) + index.fit( + [ + {"id": "database", "text": "数据库检索系统"}, + {"id": "partial", "text": "数据分析"}, + {"id": "unrelated", "text": "image classification"}, + ] + ) + index.close() + + reopened = Fts5BM25Index(str(db_path)) + try: + # Match the CJK term independently of the Latin/number terms, while + # still requiring all its bigrams (the partial match must be excluded). + assert [result.id for result in reopened.search(query)] == ["database"] + finally: + reopened.close()