diff --git a/docling_core/types/doc/__init__.py b/docling_core/types/doc/__init__.py index 4625f740..380ef1fa 100644 --- a/docling_core/types/doc/__init__.py +++ b/docling_core/types/doc/__init__.py @@ -21,6 +21,10 @@ BaseMeta, BasePrediction, CodeMetaField, + DataPointDirection, + DataPointMention, + DataPointPrecision, + DataPointScale, DescriptionMetaField, EntitiesMetaField, EntityMention, diff --git a/docling_core/types/doc/common/meta.py b/docling_core/types/doc/common/meta.py index fd7be423..8b6abd37 100644 --- a/docling_core/types/doc/common/meta.py +++ b/docling_core/types/doc/common/meta.py @@ -1,7 +1,7 @@ """Metadata models attached to document nodes.""" from enum import Enum -from typing import Annotated, Any, Final, Optional +from typing import Annotated, Any, Final, Literal, Optional, Union from pydantic import ( BaseModel, @@ -10,6 +10,7 @@ Field, FieldSerializationInfo, field_serializer, + field_validator, model_validator, ) from typing_extensions import Self @@ -159,10 +160,161 @@ class EntityMention(BasePrediction): ] = None +DataPointPrecision = Literal["exact", "approximate", "lower_bound", "upper_bound", "range_low", "range_high"] +"""Valid values for DataPointMention.precision.""" + +DataPointDirection = Literal["increase", "decrease", "neutral"] +"""Valid values for DataPointMention.direction.""" + +DataPointScale = Literal[ + "hundred", + "thousand", + "k", + "million", + "m", + "billion", + "b", + "bn", + "trillion", + "t", + "quadrillion", +] +"""Valid values for DataPointMention.scale. Each value maps to a known numeric multiplier.""" + + +class DataPointMention(EntityMention): + """A quantitative data point mentioned in text, extending EntityMention with a numeric breakdown. + + Inherited fields: text (normalised form), orig (source text), label (category), + charspan, confidence, created_by. + """ + + value: Annotated[ + Optional[float], + Field( + description="Numeric value as written, before scale application. E.g. 4.0 for '$4B'.", + examples=[4.0, 30.0, 3.0], + ), + ] = None + + unit: Annotated[ + Optional[str], + Field( + description="Dimensional unit, e.g. 'USD', '°C', '%'. ISO 4217 recommended for currencies.", + examples=["USD", "°C", "%"], + ), + ] = None + + scale: Annotated[ + Optional[DataPointScale], + Field( + description="Magnitude multiplier from the source text, e.g. 'billion', 'million', 'k'.", + examples=["billion", "million", "k"], + ), + ] = None + + normalized_value: Annotated[ + Optional[float], + Field( + description="value * scale_factor in base units, e.g. 4e9 for '$4B'. Derived convenience field.", + examples=[4_000_000_000.0, 30.0], + ), + ] = None + + range_end: Annotated[ + Optional[float], + Field( + description=( + "Upper bound when the data point expresses a range, e.g. 20.0 for 'between 10 and 20°C'. " + "Use with precision='range_low'; value holds the lower bound." + ), + examples=[20.0], + ), + ] = None + + display_dp: Annotated[ + Optional[int], + Field( + ge=0, + description=( + "Decimal places as written in the source: 0 for '3%', 1 for '3.0%', 2 for '3.00%'. " + "None when not determinable." + ), + examples=[0, 1, 2], + ), + ] = None + + precision: Annotated[ + Optional[DataPointPrecision], + Field( + description=( + "Author's epistemic qualification: 'exact', 'approximate', " + "'lower_bound', 'upper_bound', 'range_low', 'range_high'." + ), + examples=["approximate", "lower_bound"], + ), + ] = None + + direction: Annotated[ + Optional[DataPointDirection], + Field( + description=( + "Direction of change for delta/growth values: 'increase', 'decrease', 'neutral'. " + "E.g. 'revenue grew 3%' → 'increase'." + ), + examples=["increase", "decrease"], + ), + ] = None + + @field_validator("range_end", mode="after") + @classmethod + def _validate_range_end(cls, v: Optional[float], info: Any) -> Optional[float]: + if v is not None and info.data.get("value") is None: + raise ValueError("range_end requires value to be set") + return v + + @property + def scale_factor(self) -> Optional[float]: + """Numeric multiplier for scale, or None if absent or unrecognised.""" + if self.scale is None: + return None + _SCALE_MAP: dict[str, float] = { + "hundred": 1e2, + "thousand": 1e3, + "k": 1e3, + "million": 1e6, + "m": 1e6, + "billion": 1e9, + "b": 1e9, + "bn": 1e9, + "trillion": 1e12, + "t": 1e12, + "quadrillion": 1e15, + } + return _SCALE_MAP.get(self.scale) + + def compute_normalized_value(self) -> Optional[float]: + """Return value * scale_factor without caching. + + Returns None when value is absent or scale is present but unrecognised. + Use this to derive or refresh normalized_value after construction. + """ + if self.value is None: + return None + factor = self.scale_factor + if factor is None and self.scale is not None: + return None + return self.value * (factor if factor is not None else 1.0) + + class EntitiesMetaField(_ExtraAllowingModel): - """Container for extracted entity mentions.""" + """Container for extracted entity mentions. + + The mentions list accepts both plain EntityMention objects and the richer + DataPointMention subclass; use isinstance(m, DataPointMention) to filter. + """ - mentions: Annotated[list[EntityMention], Field(min_length=1)] + mentions: Annotated[list[Union[DataPointMention, EntityMention]], Field(min_length=1)] class KeywordsMetaField(_ExtraAllowingModel): diff --git a/docs/DoclingDocument.json b/docs/DoclingDocument.json index 74f4d839..a98f88fe 100644 --- a/docs/DoclingDocument.json +++ b/docs/DoclingDocument.json @@ -590,6 +590,273 @@ "title": "CoordOrigin", "type": "string" }, + "DataPointMention": { + "additionalProperties": true, + "description": "A quantitative data point mentioned in text, extending EntityMention with a numeric breakdown.\n\nInherited fields: text (normalised form), orig (source text), label (category),\ncharspan, confidence, created_by.", + "properties": { + "confidence": { + "anyOf": [ + { + "maximum": 1, + "minimum": 0, + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The confidence of the prediction.", + "examples": [ + 0.9, + 0.42 + ], + "title": "Confidence" + }, + "created_by": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "The origin of the prediction.", + "examples": [ + "ibm-granite/granite-docling-258M" + ], + "title": "Created By" + }, + "text": { + "description": "Normalized text of the entity mention.", + "title": "Text", + "type": "string" + }, + "orig": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Exact source text extracted from the original charspan, analogous to TextItem.orig. This may differ from 'text' when the mention has been normalized.", + "title": "Orig" + }, + "label": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Entity type or category.", + "title": "Label" + }, + "charspan": { + "anyOf": [ + { + "description": "Character span (0-indexed)", + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "integer" + } + ], + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Character span (0-indexed) of the entity mention in the source text.", + "title": "Charspan" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Numeric value as written, before scale application. E.g. 4.0 for '$4B'.", + "examples": [ + 4.0, + 30.0, + 3.0 + ], + "title": "Value" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dimensional unit, e.g. 'USD', '°C', '%'. ISO 4217 recommended for currencies.", + "examples": [ + "USD", + "°C", + "%" + ], + "title": "Unit" + }, + "scale": { + "anyOf": [ + { + "enum": [ + "hundred", + "thousand", + "k", + "million", + "m", + "billion", + "b", + "bn", + "trillion", + "t", + "quadrillion" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Magnitude multiplier from the source text, e.g. 'billion', 'million', 'k'.", + "examples": [ + "billion", + "million", + "k" + ], + "title": "Scale" + }, + "normalized_value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "value * scale_factor in base units, e.g. 4e9 for '$4B'. Derived convenience field.", + "examples": [ + 4000000000.0, + 30.0 + ], + "title": "Normalized Value" + }, + "range_end": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Upper bound when the data point expresses a range, e.g. 20.0 for 'between 10 and 20°C'. Use with precision='range_low'; value holds the lower bound.", + "examples": [ + 20.0 + ], + "title": "Range End" + }, + "display_dp": { + "anyOf": [ + { + "minimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Decimal places as written in the source: 0 for '3%', 1 for '3.0%', 2 for '3.00%'. None when not determinable.", + "examples": [ + 0, + 1, + 2 + ], + "title": "Display Dp" + }, + "precision": { + "anyOf": [ + { + "enum": [ + "exact", + "approximate", + "lower_bound", + "upper_bound", + "range_low", + "range_high" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Author's epistemic qualification: 'exact', 'approximate', 'lower_bound', 'upper_bound', 'range_low', 'range_high'.", + "examples": [ + "approximate", + "lower_bound" + ], + "title": "Precision" + }, + "direction": { + "anyOf": [ + { + "enum": [ + "increase", + "decrease", + "neutral" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Direction of change for delta/growth values: 'increase', 'decrease', 'neutral'. E.g. 'revenue grew 3%' → 'increase'.", + "examples": [ + "increase", + "decrease" + ], + "title": "Direction" + } + }, + "required": [ + "text" + ], + "title": "DataPointMention", + "type": "object" + }, "DescriptionAnnotation": { "description": "DescriptionAnnotation.", "properties": { @@ -707,11 +974,18 @@ }, "EntitiesMetaField": { "additionalProperties": true, - "description": "Container for extracted entity mentions.", + "description": "Container for extracted entity mentions.\n\nThe mentions list accepts both plain EntityMention objects and the richer\nDataPointMention subclass; use isinstance(m, DataPointMention) to filter.", "properties": { "mentions": { "items": { - "$ref": "#/$defs/EntityMention" + "anyOf": [ + { + "$ref": "#/$defs/DataPointMention" + }, + { + "$ref": "#/$defs/EntityMention" + } + ] }, "minItems": 1, "title": "Mentions", diff --git a/test/test_metadata.py b/test/test_metadata.py index 45abf8a3..b7e1de2c 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -15,6 +15,7 @@ ) from docling_core.types.doc import ( BaseMeta, + DataPointMention, DocItem, DocItemLabel, DoclingDocument, @@ -379,3 +380,130 @@ def test_keywords_topics_required_values() -> None: TopicsMetaField(values=34) with pytest.raises(ValidationError, match="valid string"): TopicsMetaField(values=[34]) + + +# --------------------------------------------------------------------------- +# DataPointMention tests +# --------------------------------------------------------------------------- + + +def test_data_point_roundtrip_and_html_rendering() -> None: + doc = DoclingDocument(name="data-point-meta") + item = doc.add_text(label=DocItemLabel.TEXT, text="IBM revenue was $4B in Q3 2024.") + item.meta = BaseMeta( + entities=EntitiesMetaField( + mentions=[ + EntityMention(text="IBM", label="ORG", charspan=(0, 3)), + DataPointMention( + text="4 billion USD", + orig="$4B", + label="REVENUE", + charspan=(16, 19), + value=4.0, + unit="USD", + scale="billion", + normalized_value=4_000_000_000.0, + display_dp=0, + precision="exact", + ), + ] + ) + ) + + roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json")) + meta = roundtrip.texts[0].meta + assert meta is not None and meta.entities is not None + assert meta.has_content() + + plain, dp = meta.entities.mentions + # plain EntityMention comes back as DataPointMention (subclass tried first in Union) + # but carries no numeric fields + assert isinstance(plain, DataPointMention) + assert plain.value is None + + assert isinstance(dp, DataPointMention) + assert dp.orig == "$4B" + assert dp.value == 4.0 + assert dp.unit == "USD" + assert dp.scale == "billion" + assert dp.normalized_value == 4_000_000_000.0 + assert dp.display_dp == 0 + assert dp.precision == "exact" + assert dp.scale_factor == 1e9 + assert dp.compute_normalized_value() == dp.normalized_value + + html = HTMLDocSerializer(doc=doc, params=HTMLParams()).serialize().text + assert 'data-meta-name="entities"' in html + assert "IBM (ORG, [0,3])" in html + assert "4 billion USD (REVENUE, [16,19])" in html + + +def test_data_point_range_and_direction() -> None: + doc = DoclingDocument(name="range-meta") + item = doc.add_text(label=DocItemLabel.TEXT, text="The growth rate is approximately 3.0%.") + item.meta = BaseMeta( + entities=EntitiesMetaField( + mentions=[ + DataPointMention( + text="3.0%", + orig="3.0%", + label="GROWTH_RATE", + charspan=(32, 36), + value=3.0, + unit="%", + display_dp=1, + precision="approximate", + direction="increase", + ), + DataPointMention( + text="10 °C", + label="TEMPERATURE", + value=10.0, + unit="°C", + precision="range_low", + range_end=20.0, + display_dp=0, + ), + ] + ) + ) + + roundtrip = DoclingDocument.model_validate(doc.model_dump(mode="json")) + meta = roundtrip.texts[0].meta + assert meta is not None and meta.entities is not None + mentions = meta.entities.mentions + + growth = mentions[0] + assert isinstance(growth, DataPointMention) + assert growth.value == 3.0 + assert growth.display_dp == 1 # "3.0%" — one decimal place + assert growth.precision == "approximate" + assert growth.direction == "increase" + + temp = mentions[1] + assert isinstance(temp, DataPointMention) + assert temp.value == 10.0 + assert temp.range_end == 20.0 + assert temp.precision == "range_low" + assert temp.display_dp == 0 # "10" — no decimal places + + +def test_data_point_numeric_helpers() -> None: + # scale_factor maps canonical scale strings to their multipliers + assert DataPointMention(text="4B", scale="billion").scale_factor == 1e9 + assert DataPointMention(text="5k", scale="k").scale_factor == 1e3 + assert DataPointMention(text="x").scale_factor is None + + # compute_normalized_value applies the scale factor on demand + assert DataPointMention(text="4B", value=4.0, scale="billion").compute_normalized_value() == 4_000_000_000.0 + assert DataPointMention(text="30", value=30.0).compute_normalized_value() == 30.0 + assert DataPointMention(text="n/a", scale="billion").compute_normalized_value() is None + + # display_dp is independent of the numeric value: "3%" and "3.0%" share value=3.0 + assert DataPointMention(text="3%", value=3.0, display_dp=0).display_dp == 0 + assert DataPointMention(text="3.0%", value=3.0, display_dp=1).display_dp == 1 + + +def test_data_point_range_end_requires_value() -> None: + with pytest.raises(ValidationError, match="range_end requires value"): + DataPointMention(text="x", range_end=20.0)