Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions docling_core/transforms/serializer/doclang.py
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,17 @@ def _get_delim(*, params: DocLangParams) -> str:
return "" if params.pretty_indentation is None else "\n"


_XML_10_ILLEGAL_CHARACTER_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\uD800-\uDFFF\uFFFE\uFFFF]")


def _replace_xml_illegal_character(match: re.Match[str]) -> str:
"""Return a visible representation for an XML 1.0-illegal character."""
return f"[U+{ord(match.group()):04X}]"


def _escape_text(text: str, params: DocLangParams) -> str:
"""Escape text for DocLang while preserving XML 1.0 validity."""
text = _XML_10_ILLEGAL_CHARACTER_RE.sub(_replace_xml_illegal_character, text)
do_wrap = params.content_wrapping_mode == WrapMode.ALWAYS or (
params.content_wrapping_mode == WrapMode.AUTO and (text != text.strip() or "\n" in text)
)
Expand Down
25 changes: 25 additions & 0 deletions test/test_serialization_doclang.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Unit tests for Doclang create_closing_token helper."""

import warnings
import xml.etree.ElementTree as ET
from itertools import chain
from pathlib import Path
from typing import Optional
Expand Down Expand Up @@ -328,6 +329,30 @@ def _create_escape_test_doc(inp_doc: DoclingDocument):
return doc


@pytest.mark.parametrize("pretty_indentation", [None, " "])
def test_doclang_replaces_xml_illegal_characters(pretty_indentation: Optional[str]):
"""DocLang output remains valid when document text contains XML-illegal controls."""
doc = DoclingDocument(name="xml_illegal_character")
doc.add_text(label=DocItemLabel.TEXT, text="Before break\x0bAfter break")

result = (
DocLangDocSerializer(
doc=doc,
params=DocLangParams(
include_version=False,
add_location=False,
pretty_indentation=pretty_indentation,
),
)
.serialize()
.text
)

ET.fromstring(result)
assert "Before break[U+000B]After break" in result
assert "\x0b" not in result


def test_cdata_always(sample_doc: DoclingDocument):
"""Test escape_mode=ALWAYS."""
doc = _create_escape_test_doc(sample_doc)
Expand Down
Loading