fix(pdf): remove NUL characters from parsed text

docling/backend/docling_parse_backend.py

@@ -79,6 +79,18 @@ def _make_docling_parse_page_content_config(
     )
 
 
+def _sanitize_text_cells(cells: Iterable[TextCell]) -> None:
+    for cell in cells:
+        cell.text = cell.text.replace("\x00", "")
+        cell.orig = cell.orig.replace("\x00", "")
+
+
+def _sanitize_segmented_page_text(seg_page: SegmentedPdfPage) -> None:
+    _sanitize_text_cells(seg_page.char_cells)
+    _sanitize_text_cells(seg_page.word_cells)
+    _sanitize_text_cells(seg_page.textline_cells)
+
+
 class DoclingParsePageBackend(ManagedPdfiumPageBackend):
     def __init__(
         self,
@@ -130,6 +142,7 @@ def _ensure_parsed(self) -> None:
             self._page_no + 1,
             content_config=content_config,
         )
+        _sanitize_segmented_page_text(seg_page)
 
         # In Docling, all TextCell instances are expected with top-left origin.
         [
@@ -412,6 +425,7 @@ def get_segmented_page(self) -> Optional[SegmentedPdfPage]:
             return None
         if self._seg_page is None:
             seg_page = self._result.get_page()
+            _sanitize_segmented_page_text(seg_page)
             page_height = seg_page.dimension.height
             for tc in seg_page.textline_cells:
                 tc.to_top_left_origin(page_height)

tests/test_backend_docling_parse.py

@@ -4,6 +4,7 @@
 
 import pytest
 from docling_core.types.doc import CoordOrigin
+from docling_core.types.doc.page import BoundingRectangle, TextCell
 from docling_parse.pdf_parser import ContentLevel
 from PIL import Image, ImageDraw, ImageStat
 
@@ -13,6 +14,7 @@
     DoclingParsePageBackend,
     ThreadedDoclingParseDocumentBackend,
     ThreadedDoclingParsePageBackend,
+    _sanitize_text_cells,
 )
 from docling.backend.pdf_backend import PdfDocumentBackend
 from docling.datamodel.backend_options import ThreadedDoclingParseBackendOptions
@@ -38,6 +40,38 @@ def _get_backend(pdf_doc):
     return doc_backend
 
 
+def test_sanitize_text_cells_removes_nul_characters():
+    rect = BoundingRectangle(
+        r_x0=0,
+        r_y0=0,
+        r_x1=1,
+        r_y1=0,
+        r_x2=1,
+        r_y2=1,
+        r_x3=0,
+        r_y3=1,
+    )
+    cell = TextCell(
+        rect=rect,
+        text="30.45 mg.mL \x00 1",
+        orig="30.45 mg.mL \x00 1",
+        from_ocr=False,
+    )
+    unicode_cell = TextCell(
+        rect=rect,
+        text="30.45 mg.mL\u207b\u00b9",
+        orig="30.45 mg.mL\u207b\u00b9",
+        from_ocr=False,
+    )
+
+    _sanitize_text_cells([cell, unicode_cell])
+
+    assert cell.text == "30.45 mg.mL  1"
+    assert cell.orig == "30.45 mg.mL  1"
+    assert unicode_cell.text == "30.45 mg.mL\u207b\u00b9"
+    assert unicode_cell.orig == "30.45 mg.mL\u207b\u00b9"
+
+
 def test_text_cell_counts():
     pdf_doc = Path("./tests/data/pdf/sources/redp5110_sampled.pdf")
 
@@ -554,6 +588,9 @@ def test_non_threaded_page_backend_disables_bitmap_byte_materialization() -> Non
     captured_content_config: Any | None = None
 
     class _FakeCell:
+        text = "text\x00"
+        orig = "orig\x00"
+
         def to_top_left_origin(self, _page_height: float) -> "_FakeCell":
             return self
 
@@ -596,6 +633,8 @@ def close(self) -> None:
         page_backend.unload()
 
     assert len(cells) == 1
+    assert cells[0].text == "text"
+    assert cells[0].orig == "orig"
     assert captured_content_config is not None
     assert captured_content_config.include_bitmap_bytes is False
 
@@ -648,6 +687,9 @@ def test_threaded_page_backend_delegates_image_access() -> None:
 
 def test_threaded_page_backend_disables_bitmap_materialization() -> None:
     class _FakeCell:
+        text = "text\x00"
+        orig = "orig\x00"
+
         def to_top_left_origin(self, _page_height: float) -> "_FakeCell":
             return self
 
@@ -669,6 +711,8 @@ class _FakeSegmentedPage:
     cells = list(page_backend.get_text_cells())
 
     assert len(cells) == 1
+    assert cells[0].text == "text"
+    assert cells[0].orig == "orig"
 
 
 def _create_black_square_pdf(path: Path) -> None: