diff --git a/docling_core/transforms/serializer/markdown.py b/docling_core/transforms/serializer/markdown.py index 955fd080..aaa0418e 100644 --- a/docling_core/transforms/serializer/markdown.py +++ b/docling_core/transforms/serializer/markdown.py @@ -427,6 +427,46 @@ def serialize( ) +def _serialize_image_part( + item: FloatingItem, + doc: DoclingDocument, + image_mode: ImageRefMode, + image_placeholder: str, + **kwargs: Any, +) -> SerializationResult: + """Render a floating item's image per the image mode. + + Shared by the picture and table serializers so both honor image_mode + identically for any FloatingItem that carries an image. + """ + error_response = "" + if image_mode == ImageRefMode.PLACEHOLDER: + text_res = image_placeholder + elif image_mode == ImageRefMode.EMBEDDED: + # short-cut: we already have the image in base64 + if isinstance(item.image, ImageRef) and isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data": + text_res = f"![Image]({item.image.uri})" + else: + # get the item.image._pil or crop it out of the page-image + img = item.get_image(doc=doc) + if img is not None: + imgb64 = item._image_to_base64(img) + text_res = f"![Image](data:image/png;base64,{imgb64})" + else: + text_res = error_response + elif image_mode == ImageRefMode.REFERENCED: + if not isinstance(item.image, ImageRef) or ( + isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data" + ): + text_res = image_placeholder + else: + text_res = f"![Image]({item.image.uri!s})" + else: + text_res = image_placeholder + + return create_ser_result(text=text_res, span_source=item) + + class MarkdownTableSerializer(BaseTableSerializer): """Markdown-specific table item serializer.""" @@ -584,6 +624,18 @@ def serialize( if table_text: res_parts.append(create_ser_result(text=table_text, span_source=item)) + # Emit the table image alongside the text grid when one was generated. + # Tables without an image (the common case) stay a strict no-op. + if item.image is not None and params.image_mode != ImageRefMode.PLACEHOLDER: + img_res = _serialize_image_part( + item=item, + doc=doc, + image_mode=params.image_mode, + image_placeholder=params.image_placeholder, + ) + if img_res.text: + res_parts.append(img_res) + text_res = "\n\n".join([r.text for r in res_parts]) return create_ser_result(text=text_res, span_source=res_parts) @@ -622,7 +674,7 @@ def serialize( if ann_res.text: res_parts.append(ann_res) - img_res = self._serialize_image_part( + img_res = _serialize_image_part( item=item, doc=doc, image_mode=params.image_mode, @@ -648,51 +700,6 @@ def serialize( return create_ser_result(text=text_res, span_source=res_parts) - def _serialize_image_part( - self, - item: PictureItem, - doc: DoclingDocument, - image_mode: ImageRefMode, - image_placeholder: str, - **kwargs: Any, - ) -> SerializationResult: - error_response = ( - "" - ) - if image_mode == ImageRefMode.PLACEHOLDER: - text_res = image_placeholder - elif image_mode == ImageRefMode.EMBEDDED: - # short-cut: we already have the image in base64 - if ( - isinstance(item.image, ImageRef) - and isinstance(item.image.uri, AnyUrl) - and item.image.uri.scheme == "data" - ): - text = f"![Image]({item.image.uri})" - text_res = text - else: - # get the item.image._pil or crop it out of the page-image - img = item.get_image(doc=doc) - - if img is not None: - imgb64 = item._image_to_base64(img) - text = f"![Image](data:image/png;base64,{imgb64})" - - text_res = text - else: - text_res = error_response - elif image_mode == ImageRefMode.REFERENCED: - if not isinstance(item.image, ImageRef) or ( - isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data" - ): - text_res = image_placeholder - else: - text_res = f"![Image]({item.image.uri!s})" - else: - text_res = image_placeholder - - return create_ser_result(text=text_res, span_source=item) - class MarkdownKeyValueSerializer(BaseKeyValueSerializer): """Markdown-specific key-value item serializer.""" diff --git a/docling_core/types/doc/items/node.py b/docling_core/types/doc/items/node.py index 54f68b02..60e52ad4 100644 --- a/docling_core/types/doc/items/node.py +++ b/docling_core/types/doc/items/node.py @@ -1,6 +1,8 @@ """Base document-tree node items: NodeItem, DocItem, FloatingItem.""" +import base64 from collections.abc import Sequence +from io import BytesIO from typing import TYPE_CHECKING, Annotated, Optional from PIL import Image as PILImage @@ -231,3 +233,12 @@ def get_image(self, doc: "DoclingDocument", prov_index: int = 0) -> Optional[PIL if self.image is not None: return self.image.pil_image return super().get_image(doc=doc, prov_index=prov_index) + + # Convert the image to Base64 + def _image_to_base64(self, pil_image, format="PNG"): + """Base64 representation of the image.""" + buffered = BytesIO() + pil_image.save(buffered, format=format) # Save the image to the byte stream + img_bytes = buffered.getvalue() # Get the byte data + img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string + return img_base64 diff --git a/docling_core/types/doc/items/picture/picture.py b/docling_core/types/doc/items/picture/picture.py index 924c5c2c..06a14d69 100644 --- a/docling_core/types/doc/items/picture/picture.py +++ b/docling_core/types/doc/items/picture/picture.py @@ -1,12 +1,10 @@ """Picture item and its annotation-data union.""" -import base64 import hashlib import logging import typing import warnings from collections.abc import Sequence -from io import BytesIO from typing import TYPE_CHECKING, Annotated, Optional, Union from PIL import Image as PILImage @@ -143,15 +141,6 @@ def _migrate_annotations_to_meta(self) -> Self: return self - # Convert the image to Base64 - def _image_to_base64(self, pil_image, format="PNG"): - """Base64 representation of the image.""" - buffered = BytesIO() - pil_image.save(buffered, format=format) # Save the image to the byte stream - img_bytes = buffered.getvalue() # Get the byte data - img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string - return img_base64 - @staticmethod def _image_to_hexhash(img: Optional[PILImage.Image]) -> Optional[str]: """Hexash from the image.""" diff --git a/test/test_serialization.py b/test/test_serialization.py index b6bd4b1d..9f6ecca3 100644 --- a/test/test_serialization.py +++ b/test/test_serialization.py @@ -6,6 +6,7 @@ from xml.etree import ElementTree as ET import pytest +from PIL import Image from docling_core.transforms.serializer.common import _DEFAULT_LABELS from docling_core.transforms.serializer.html import ( @@ -32,12 +33,14 @@ DescriptionAnnotation, EntitiesMetaField, EntityMention, + ImageRef, LanguageMetaField, PictureClassificationMetaField, PictureClassificationPrediction, PictureMeta, RefItem, RichTableCell, + Size, SummaryMetaField, TableCell, TableData, @@ -352,6 +355,82 @@ def test_md_single_row_table(): verify(exp_file=exp_file, actual=actual) +def test_md_table_with_image_embedded(): + doc = DoclingDocument(name="") + table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) + doc.add_table_cell( + table_item=table, + cell=TableCell( + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="foo", + ), + ) + table.image = ImageRef.from_pil(image=Image.new("RGB", (2, 2)), dpi=72) + + ser = MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED), + ) + actual = ser.serialize().text + assert "| foo |" in actual + assert "![Image](data:image/png;base64," in actual + + +def test_md_table_with_image_referenced(): + doc = DoclingDocument(name="") + table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) + doc.add_table_cell( + table_item=table, + cell=TableCell( + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="foo", + ), + ) + table.image = ImageRef( + mimetype="image/png", + dpi=72, + size=Size(width=2, height=2), + uri="table_0.png", + ) + + ser = MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(image_mode=ImageRefMode.REFERENCED), + ) + actual = ser.serialize().text + assert actual == "| foo |\n|-------|\n\n![Image](table_0.png)" + + +def test_md_table_without_image_unaffected_by_image_mode(): + """A table with no generated image must not gain an image line, even if + image_mode is set — this must remain a strict no-op for the common case.""" + doc = DoclingDocument(name="") + table = doc.add_table(data=TableData(num_rows=1, num_cols=1)) + doc.add_table_cell( + table_item=table, + cell=TableCell( + start_row_offset_idx=0, + end_row_offset_idx=1, + start_col_offset_idx=0, + end_col_offset_idx=1, + text="foo", + ), + ) + + ser = MarkdownDocSerializer( + doc=doc, + params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED), + ) + actual = ser.serialize().text + assert actual == "| foo |\n|-------|" + + def test_md_pipe_in_table(): doc = DoclingDocument(name="Pipe in Table") table = doc.add_table(data=TableData(num_rows=1, num_cols=1))