Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 53 additions & 46 deletions docling_core/transforms/serializer/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,46 @@ def serialize(
)


def _serialize_image_part(
item: FloatingItem,
doc: DoclingDocument,
image_mode: ImageRefMode,
image_placeholder: str,
**kwargs: Any,
) -> SerializationResult:
"""Render a floating item's image per the image mode.

Shared by the picture and table serializers so both honor image_mode
identically for any FloatingItem that carries an image.
"""
error_response = "<!-- 🖼️❌ Image not available. Please use `PdfPipelineOptions(generate_picture_images=True)` -->"
if image_mode == ImageRefMode.PLACEHOLDER:
text_res = image_placeholder
elif image_mode == ImageRefMode.EMBEDDED:
# short-cut: we already have the image in base64
if isinstance(item.image, ImageRef) and isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data":
text_res = f"![Image]({item.image.uri})"
else:
# get the item.image._pil or crop it out of the page-image
img = item.get_image(doc=doc)
if img is not None:
imgb64 = item._image_to_base64(img)
text_res = f"![Image](data:image/png;base64,{imgb64})"
else:
text_res = error_response
elif image_mode == ImageRefMode.REFERENCED:
if not isinstance(item.image, ImageRef) or (
isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data"
):
text_res = image_placeholder
else:
text_res = f"![Image]({item.image.uri!s})"
else:
text_res = image_placeholder

return create_ser_result(text=text_res, span_source=item)


class MarkdownTableSerializer(BaseTableSerializer):
"""Markdown-specific table item serializer."""

Expand Down Expand Up @@ -584,6 +624,18 @@ def serialize(
if table_text:
res_parts.append(create_ser_result(text=table_text, span_source=item))

# Emit the table image alongside the text grid when one was generated.
# Tables without an image (the common case) stay a strict no-op.
if item.image is not None and params.image_mode != ImageRefMode.PLACEHOLDER:
img_res = _serialize_image_part(
item=item,
doc=doc,
image_mode=params.image_mode,
image_placeholder=params.image_placeholder,
)
if img_res.text:
res_parts.append(img_res)

text_res = "\n\n".join([r.text for r in res_parts])

return create_ser_result(text=text_res, span_source=res_parts)
Expand Down Expand Up @@ -622,7 +674,7 @@ def serialize(
if ann_res.text:
res_parts.append(ann_res)

img_res = self._serialize_image_part(
img_res = _serialize_image_part(
item=item,
doc=doc,
image_mode=params.image_mode,
Expand All @@ -648,51 +700,6 @@ def serialize(

return create_ser_result(text=text_res, span_source=res_parts)

def _serialize_image_part(
self,
item: PictureItem,
doc: DoclingDocument,
image_mode: ImageRefMode,
image_placeholder: str,
**kwargs: Any,
) -> SerializationResult:
error_response = (
"<!-- 🖼️❌ Image not available. Please use `PdfPipelineOptions(generate_picture_images=True)` -->"
)
if image_mode == ImageRefMode.PLACEHOLDER:
text_res = image_placeholder
elif image_mode == ImageRefMode.EMBEDDED:
# short-cut: we already have the image in base64
if (
isinstance(item.image, ImageRef)
and isinstance(item.image.uri, AnyUrl)
and item.image.uri.scheme == "data"
):
text = f"![Image]({item.image.uri})"
text_res = text
else:
# get the item.image._pil or crop it out of the page-image
img = item.get_image(doc=doc)

if img is not None:
imgb64 = item._image_to_base64(img)
text = f"![Image](data:image/png;base64,{imgb64})"

text_res = text
else:
text_res = error_response
elif image_mode == ImageRefMode.REFERENCED:
if not isinstance(item.image, ImageRef) or (
isinstance(item.image.uri, AnyUrl) and item.image.uri.scheme == "data"
):
text_res = image_placeholder
else:
text_res = f"![Image]({item.image.uri!s})"
else:
text_res = image_placeholder

return create_ser_result(text=text_res, span_source=item)


class MarkdownKeyValueSerializer(BaseKeyValueSerializer):
"""Markdown-specific key-value item serializer."""
Expand Down
11 changes: 11 additions & 0 deletions docling_core/types/doc/items/node.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Base document-tree node items: NodeItem, DocItem, FloatingItem."""

import base64
from collections.abc import Sequence
from io import BytesIO
from typing import TYPE_CHECKING, Annotated, Optional

from PIL import Image as PILImage
Expand Down Expand Up @@ -231,3 +233,12 @@ def get_image(self, doc: "DoclingDocument", prov_index: int = 0) -> Optional[PIL
if self.image is not None:
return self.image.pil_image
return super().get_image(doc=doc, prov_index=prov_index)

# Convert the image to Base64
def _image_to_base64(self, pil_image, format="PNG"):
"""Base64 representation of the image."""
buffered = BytesIO()
pil_image.save(buffered, format=format) # Save the image to the byte stream
img_bytes = buffered.getvalue() # Get the byte data
img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string
return img_base64
11 changes: 0 additions & 11 deletions docling_core/types/doc/items/picture/picture.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
"""Picture item and its annotation-data union."""

import base64
import hashlib
import logging
import typing
import warnings
from collections.abc import Sequence
from io import BytesIO
from typing import TYPE_CHECKING, Annotated, Optional, Union

from PIL import Image as PILImage
Expand Down Expand Up @@ -143,15 +141,6 @@ def _migrate_annotations_to_meta(self) -> Self:

return self

# Convert the image to Base64
def _image_to_base64(self, pil_image, format="PNG"):
"""Base64 representation of the image."""
buffered = BytesIO()
pil_image.save(buffered, format=format) # Save the image to the byte stream
img_bytes = buffered.getvalue() # Get the byte data
img_base64 = base64.b64encode(img_bytes).decode("utf-8") # Encode to Base64 and decode to string
return img_base64

@staticmethod
def _image_to_hexhash(img: Optional[PILImage.Image]) -> Optional[str]:
"""Hexash from the image."""
Expand Down
79 changes: 79 additions & 0 deletions test/test_serialization.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from xml.etree import ElementTree as ET

import pytest
from PIL import Image

from docling_core.transforms.serializer.common import _DEFAULT_LABELS
from docling_core.transforms.serializer.html import (
Expand All @@ -32,12 +33,14 @@
DescriptionAnnotation,
EntitiesMetaField,
EntityMention,
ImageRef,
LanguageMetaField,
PictureClassificationMetaField,
PictureClassificationPrediction,
PictureMeta,
RefItem,
RichTableCell,
Size,
SummaryMetaField,
TableCell,
TableData,
Expand Down Expand Up @@ -352,6 +355,82 @@ def test_md_single_row_table():
verify(exp_file=exp_file, actual=actual)


def test_md_table_with_image_embedded():
doc = DoclingDocument(name="")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="foo",
),
)
table.image = ImageRef.from_pil(image=Image.new("RGB", (2, 2)), dpi=72)

ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED),
)
actual = ser.serialize().text
assert "| foo |" in actual
assert "![Image](data:image/png;base64," in actual


def test_md_table_with_image_referenced():
doc = DoclingDocument(name="")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="foo",
),
)
table.image = ImageRef(
mimetype="image/png",
dpi=72,
size=Size(width=2, height=2),
uri="table_0.png",
)

ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(image_mode=ImageRefMode.REFERENCED),
)
actual = ser.serialize().text
assert actual == "| foo |\n|-------|\n\n![Image](table_0.png)"


def test_md_table_without_image_unaffected_by_image_mode():
"""A table with no generated image must not gain an image line, even if
image_mode is set — this must remain a strict no-op for the common case."""
doc = DoclingDocument(name="")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
doc.add_table_cell(
table_item=table,
cell=TableCell(
start_row_offset_idx=0,
end_row_offset_idx=1,
start_col_offset_idx=0,
end_col_offset_idx=1,
text="foo",
),
)

ser = MarkdownDocSerializer(
doc=doc,
params=MarkdownParams(image_mode=ImageRefMode.EMBEDDED),
)
actual = ser.serialize().text
assert actual == "| foo |\n|-------|"


def test_md_pipe_in_table():
doc = DoclingDocument(name="Pipe in Table")
table = doc.add_table(data=TableData(num_rows=1, num_cols=1))
Expand Down
Loading