fix: preserve mixed-script token boundaries in BM25 search

docs/CHANGELOG.md

@@ -66,3 +66,8 @@ fixes). Newest entries at the bottom.
   already-working Windows `cp314` installs. Completing the wheel matrix is the
   correct fix; the next 0.3.8 patch release will carry full `cp314` coverage
   once CI confirms the new platforms build cleanly.
+
+## 2026-09-06: Preserve mixed-script words in BM25 search
+
+- Keep Latin words and numbers separate from adjacent Chinese, Japanese, and Korean n-grams when building SQLite FTS5 indexes and parsing keyword queries. For example, `Python数据库SQL` can be retrieved by `Python`, `SQL`, or a mixed-script query.
+- Existing BM25 artifacts remain readable. Rebuild indexes containing mixed-script text to regenerate tokens that were previously joined at script boundaries.

packages/leann-core/src/leann/api.py

@@ -45,12 +45,15 @@
 def _fts5_cjk_ngrams(match: re.Match[str]) -> str:
     """Expand a CJK run into unigram and bigram tokens for SQLite FTS5."""
     text = match.group()
-    return " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])
+    tokens = " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])
+    # Keep adjacent non-CJK words separate from the first and last n-grams.
+    return f" {tokens} "
 
 
 def _fts5_cjk_query(query: str) -> str:
     """Build a safe FTS5 query that requires every CJK bigram in each term."""
-    tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|\w+", query.lower())
+    # Unicode \w includes CJK, so exclude it from the non-CJK alternative.
+    tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|[^\W{_CJK_CHARACTERS}]+", query.lower())
     terms = []
     for token in tokens:
         if _CJK_RUN.fullmatch(token):

tests/test_fts5_bm25.py

@@ -1,3 +1,4 @@
+import pytest
 from leann.api import Fts5BM25Index
 
 
@@ -30,3 +31,45 @@ def test_fts5_bm25_keeps_legacy_database_query_format(tmp_path):
         assert [result.id for result in reopened.search("database")] == ["database"]
     finally:
         reopened.close()
+
+
+@pytest.mark.parametrize("cjk_text", ["数据库", "データベース", "데이터베이스"])
+@pytest.mark.parametrize("query", ["Python", "SQL"])
+def test_fts5_bm25_preserves_words_adjacent_to_cjk(tmp_path, cjk_text, query):
+    db_path = tmp_path / "mixed.sqlite"
+    index = Fts5BM25Index(str(db_path))
+    index.fit(
+        [
+            {"id": "mixed", "text": f"Python{cjk_text}SQL"},
+            {"id": "unrelated", "text": "unrelated document"},
+        ]
+    )
+    index.close()
+
+    reopened = Fts5BM25Index(str(db_path))
+    try:
+        assert [result.id for result in reopened.search(query)] == ["mixed"]
+    finally:
+        reopened.close()
+
+
+@pytest.mark.parametrize("query", ["Python数据库", "数据库Python", "Python数据库SQL", "2026数据库"])
+def test_fts5_bm25_splits_mixed_script_query_terms(tmp_path, query):
+    db_path = tmp_path / "queries.sqlite"
+    index = Fts5BM25Index(str(db_path))
+    index.fit(
+        [
+            {"id": "database", "text": "数据库检索系统"},
+            {"id": "partial", "text": "数据分析"},
+            {"id": "unrelated", "text": "image classification"},
+        ]
+    )
+    index.close()
+
+    reopened = Fts5BM25Index(str(db_path))
+    try:
+        # Match the CJK term independently of the Latin/number terms, while
+        # still requiring all its bigrams (the partial match must be excluded).
+        assert [result.id for result in reopened.search(query)] == ["database"]
+    finally:
+        reopened.close()