diff --git a/docling_core/transforms/serializer/doclang.py b/docling_core/transforms/serializer/doclang.py index 20c18414..1293a382 100644 --- a/docling_core/transforms/serializer/doclang.py +++ b/docling_core/transforms/serializer/doclang.py @@ -434,7 +434,17 @@ def _get_delim(*, params: DocLangParams) -> str: return "" if params.pretty_indentation is None else "\n" +_XML_10_ILLEGAL_CHARACTER_RE = re.compile(r"[\x00-\x08\x0B\x0C\x0E-\x1F\uD800-\uDFFF\uFFFE\uFFFF]") + + +def _replace_xml_illegal_character(match: re.Match[str]) -> str: + """Return a visible representation for an XML 1.0-illegal character.""" + return f"[U+{ord(match.group()):04X}]" + + def _escape_text(text: str, params: DocLangParams) -> str: + """Escape text for DocLang while preserving XML 1.0 validity.""" + text = _XML_10_ILLEGAL_CHARACTER_RE.sub(_replace_xml_illegal_character, text) do_wrap = params.content_wrapping_mode == WrapMode.ALWAYS or ( params.content_wrapping_mode == WrapMode.AUTO and (text != text.strip() or "\n" in text) ) diff --git a/test/test_serialization_doclang.py b/test/test_serialization_doclang.py index 5327bd8f..44775415 100644 --- a/test/test_serialization_doclang.py +++ b/test/test_serialization_doclang.py @@ -1,6 +1,7 @@ """Unit tests for Doclang create_closing_token helper.""" import warnings +import xml.etree.ElementTree as ET from itertools import chain from pathlib import Path from typing import Optional @@ -328,6 +329,30 @@ def _create_escape_test_doc(inp_doc: DoclingDocument): return doc +@pytest.mark.parametrize("pretty_indentation", [None, " "]) +def test_doclang_replaces_xml_illegal_characters(pretty_indentation: Optional[str]): + """DocLang output remains valid when document text contains XML-illegal controls.""" + doc = DoclingDocument(name="xml_illegal_character") + doc.add_text(label=DocItemLabel.TEXT, text="Before break\x0bAfter break") + + result = ( + DocLangDocSerializer( + doc=doc, + params=DocLangParams( + include_version=False, + add_location=False, + pretty_indentation=pretty_indentation, + ), + ) + .serialize() + .text + ) + + ET.fromstring(result) + assert "Before break[U+000B]After break" in result + assert "\x0b" not in result + + def test_cdata_always(sample_doc: DoclingDocument): """Test escape_mode=ALWAYS.""" doc = _create_escape_test_doc(sample_doc)