diff --git a/openviking/parse/parsers/media/large_image_processor.py b/openviking/parse/parsers/media/large_image_processor.py index 6b2bde001d..9af149a128 100644 --- a/openviking/parse/parsers/media/large_image_processor.py +++ b/openviking/parse/parsers/media/large_image_processor.py @@ -18,6 +18,12 @@ from PIL import Image, ImageDraw, ImageFont +from openviking.utils.media_limits import ( + DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX, + DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION, + DEFAULT_IMAGE_TILE_OVERLAP_PX, + is_large_image_by_size, +) from openviking_cli.utils.config.parser_config import ImageConfig from openviking_cli.utils.logger import get_logger @@ -28,21 +34,10 @@ # Configuration # ============================================================================= -# Thresholds for triggering large image processing (fallback defaults when no config) -MAX_FILE_SIZE_MB = 10.0 # 10 MB -LARGE_IMAGE_THRESHOLD_DIMENSION = 4096 # 4096 pixels - -# Target limits for individual tiles -MAX_TILE_DIMENSION_PX = 2048 # 2048 pixels -TILE_OVERLAP_PX = 2 # 2 pixels of overlap on each side - # Quality settings for JPEG compression PREVIEW_QUALITY_START = 85 TILE_QUALITY = 90 -# Preview dimension fallback (when no config provides preview_max_dimension) -PREVIEW_MAX_DIMENSION = 2048 - @dataclass class TileInfo: @@ -108,11 +103,12 @@ def needs_large_image_processing( Returns: True if large image processing is needed """ - max_file_size_mb = config.max_file_size_mb if config else MAX_FILE_SIZE_MB - max_dimension_px = config.large_image_threshold_dimension if config else LARGE_IMAGE_THRESHOLD_DIMENSION - - file_size_mb = get_image_size_mb(file_path) - return file_size_mb > max_file_size_mb or width > max_dimension_px or height > max_dimension_px + return is_large_image_by_size( + file_size_bytes=file_path.stat().st_size, + width=width, + height=height, + config=config, + ) def create_low_res_preview( @@ -131,7 +127,7 @@ def create_low_res_preview( Returns: Preview image bytes in JPEG format """ - max_dimension_px = config.preview_max_dimension if config else PREVIEW_MAX_DIMENSION + max_dimension_px = config.preview_max_dimension if config else DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION # Work on a copy img = img.copy() @@ -171,8 +167,10 @@ def calculate_grid_dimensions( Returns: Tuple of (num_rows, num_cols) """ - max_tile_dimension_px = config.max_tile_dimension_px if config else MAX_TILE_DIMENSION_PX - tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX + max_tile_dimension_px = ( + config.max_tile_dimension_px if config else DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX + ) + tile_overlap_px = config.tile_overlap_px if config else DEFAULT_IMAGE_TILE_OVERLAP_PX effective_tile = max_tile_dimension_px - tile_overlap_px * 2 if effective_tile <= 0: @@ -204,7 +202,7 @@ def calculate_tile_positions( Returns: List of (x1, y1, x2, y2) tuples """ - tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX + tile_overlap_px = config.tile_overlap_px if config else DEFAULT_IMAGE_TILE_OVERLAP_PX # Calculate base tile size without overlap base_tile_width = ceil(width / cols) @@ -353,8 +351,9 @@ def create_grid_overlay( Returns: Grid overlay image bytes """ - tile_overlap_px = config.tile_overlap_px if config else TILE_OVERLAP_PX - preview_max_dimension = config.preview_max_dimension if config else PREVIEW_MAX_DIMENSION + preview_max_dimension = ( + config.preview_max_dimension if config else DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION + ) # Resize to a reasonable size for overlay drawing ow, oh = img.size @@ -442,9 +441,6 @@ def process_large_image( Returns: LargeImageResult with processing results """ - # Use config values or defaults - max_file_size_mb = config.max_file_size_mb if config else MAX_FILE_SIZE_MB - # Load image if not provided own_img = False if img is None: diff --git a/openviking/parse/parsers/media/utils.py b/openviking/parse/parsers/media/utils.py index daeb471ebc..7ce8e53069 100644 --- a/openviking/parse/parsers/media/utils.py +++ b/openviking/parse/parsers/media/utils.py @@ -12,6 +12,7 @@ from openviking.parse.parsers.constants import TYPESCRIPT_MPEG_TS_EXTENSION from openviking.prompts import render_prompt from openviking.storage.viking_fs import get_viking_fs +from openviking.utils.image_search import prepare_image_bytes_for_model from openviking.utils.media_limits import MAX_MEDIA_FILE_BYTES from openviking_cli.utils.config import get_openviking_config from openviking_cli.utils.logger import get_logger @@ -264,7 +265,8 @@ async def generate_image_summary( Dictionary with "name" and "summary" keys """ viking_fs = get_viking_fs() - vlm = get_openviking_config().vlm + config = get_openviking_config() + vlm = config.vlm file_name = original_filename try: @@ -295,7 +297,12 @@ async def generate_image_summary( async with llm_sem or asyncio.Semaphore(1): response = await vlm.get_vision_completion_async( prompt=prompt, - images=[image_bytes], + images=[ + prepare_image_bytes_for_model( + image_bytes, + config=getattr(config, "image", None), + ) + ], ) logger.info( diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py index a34b463d36..24b1930306 100644 --- a/openviking/utils/embedding_utils.py +++ b/openviking/utils/embedding_utils.py @@ -31,7 +31,7 @@ from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs from openviking.telemetry.request_wait_tracker import get_request_wait_tracker from openviking.utils.embedding_input import truncate_embedding_input -from openviking.utils.image_search import image_bytes_to_data_uri +from openviking.utils.image_search import image_bytes_to_model_data_uri from openviking.utils.ingest_options import IngestOptions from openviking.utils.time_utils import parse_iso_datetime from openviking_cli.utils import VikingURI, get_logger @@ -272,11 +272,14 @@ async def _build_image_data_uri( ) -> Optional[str]: """Read an image file and encode it as a base64 ``data:`` URI. + Oversized images are downsampled only for the embedding request. The + original resource bytes in VikingFS are left unchanged. Returns None if the image cannot be read. """ try: content = await viking_fs.read_file_bytes(file_path, ctx=ctx) - return image_bytes_to_data_uri(content, file_name) + image_config = getattr(get_openviking_config(), "image", None) + return image_bytes_to_model_data_uri(content, file_name, config=image_config) except Exception as e: logger.warning(f"Failed to read image for multimodal vectorization {file_path}: {e}") return None @@ -571,7 +574,9 @@ async def vectorize_file( if summary: context.set_vectorize(Vectorize(text=summary)) else: - logger.warning(f"No summary available for {file_path}, skipping vectorization") + logger.warning( + f"No summary available for {file_path}, skipping vectorization" + ) return False else: embedding_text = truncate_embedding_input( diff --git a/openviking/utils/image_search.py b/openviking/utils/image_search.py index e44eb279c0..d5a8b7d1f9 100644 --- a/openviking/utils/image_search.py +++ b/openviking/utils/image_search.py @@ -5,11 +5,17 @@ from __future__ import annotations import base64 +import io import mimetypes import os from pathlib import Path from typing import Any, Dict, List, Optional +from PIL import Image, UnidentifiedImageError + +from openviking.utils.media_limits import is_large_image_by_size +from openviking_cli.utils.config.parser_config import ImageConfig + def image_mime_type(file_name: str = "") -> str: mime_type, _ = mimetypes.guess_type(file_name or "") @@ -23,6 +29,57 @@ def image_bytes_to_data_uri(data: bytes | bytearray | memoryview, file_name: str return f"data:{image_mime_type(file_name)};base64,{encoded}" +def _prepare_image_bytes_for_model( + data: bytes | bytearray | memoryview, + config: ImageConfig | None = None, +) -> tuple[bytes, bool]: + """Return model-ready image bytes and whether they differ from the input.""" + content = bytes(data) + try: + with Image.open(io.BytesIO(content)) as img: + width, height = img.size + image_config = config or ImageConfig() + if not is_large_image_by_size( + file_size_bytes=len(content), + width=width, + height=height, + config=image_config, + ): + return content, False + + preview = img.convert("RGB") + + max_dimension = image_config.preview_max_dimension + if width > max_dimension or height > max_dimension: + ratio = min(max_dimension / width, max_dimension / height) + new_size = (max(1, int(width * ratio)), max(1, int(height * ratio))) + preview = preview.resize(new_size, Image.Resampling.LANCZOS) + + buf = io.BytesIO() + preview.save(buf, format="JPEG", quality=85, optimize=True) + return buf.getvalue(), True + except (UnidentifiedImageError, OSError, ValueError): + return content, False + + +def prepare_image_bytes_for_model( + data: bytes | bytearray | memoryview, + config: ImageConfig | None = None, +) -> bytes: + """Downsample oversized image bytes for model requests without changing storage.""" + model_bytes, _ = _prepare_image_bytes_for_model(data, config=config) + return model_bytes + + +def image_bytes_to_model_data_uri( + data: bytes | bytearray | memoryview, + file_name: str = "", + config: ImageConfig | None = None, +) -> str: + model_bytes, changed = _prepare_image_bytes_for_model(data, config=config) + return image_bytes_to_data_uri(model_bytes, "model_input.jpg" if changed else file_name) + + def is_data_image_uri(value: str) -> bool: return value.startswith("data:image/") and ";base64," in value diff --git a/openviking/utils/media_limits.py b/openviking/utils/media_limits.py index d960387309..c64fbba3dd 100644 --- a/openviking/utils/media_limits.py +++ b/openviking/utils/media_limits.py @@ -1,5 +1,37 @@ # Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. # SPDX-License-Identifier: AGPL-3.0 -"""Shared resource limits for audio and video processing.""" +"""Shared media limits and limit checks.""" + +from typing import Protocol MAX_MEDIA_FILE_BYTES = 512 * 1024 * 1024 + +DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB = 10.0 +DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION = 4096 +DEFAULT_IMAGE_PREVIEW_MAX_DIMENSION = 2048 +DEFAULT_IMAGE_MAX_TILE_DIMENSION_PX = 2048 +DEFAULT_IMAGE_TILE_OVERLAP_PX = 2 + + +class ImageLimitConfig(Protocol): + max_file_size_mb: float + large_image_threshold_dimension: int + + +def is_large_image_by_size( + *, + file_size_bytes: int, + width: int, + height: int, + config: ImageLimitConfig | None = None, +) -> bool: + """Return whether image bytes exceed the configured large-image limits.""" + max_file_size_mb = config.max_file_size_mb if config else DEFAULT_LARGE_IMAGE_MAX_FILE_SIZE_MB + max_dimension_px = ( + config.large_image_threshold_dimension + if config + else DEFAULT_LARGE_IMAGE_THRESHOLD_DIMENSION + ) + + file_size_mb = file_size_bytes / (1024 * 1024) + return file_size_mb > max_file_size_mb or width > max_dimension_px or height > max_dimension_px diff --git a/tests/parse/test_media_understanding_summary.py b/tests/parse/test_media_understanding_summary.py index 1ebe34cec8..7d5fdbce6a 100644 --- a/tests/parse/test_media_understanding_summary.py +++ b/tests/parse/test_media_understanding_summary.py @@ -1,11 +1,14 @@ import asyncio +import io from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock import pytest +from PIL import Image from openviking.parse.parsers.media import utils as media_utils +from openviking_cli.utils.config.parser_config import ImageConfig from openviking_cli.utils.config.vlm_config import VLMConfig @@ -24,6 +27,9 @@ async def read(self, _uri, offset=0, size=-1, ctx=None): return b"" return self.content[offset : offset + size] + async def read_file_bytes(self, _uri, ctx=None): + return self.content + class _BlockingReadFS: def __init__(self): @@ -102,6 +108,15 @@ async def get_media_completion_async( return "# Clip\n\nUseful summary." +class _ImageVLM: + def __init__(self): + self.images = [] + + async def get_vision_completion_async(self, *, prompt, images): + self.images = images + return "image summary" + + def _config(model_config, *, max_chars=4000): if hasattr(model_config, "get_client_instance"): vlm = model_config.get_client_instance() @@ -118,6 +133,17 @@ def _config(model_config, *, max_chars=4000): ) +def _jpeg_bytes(width: int, height: int) -> bytes: + buf = io.BytesIO() + Image.new("RGB", (width, height), "white").save(buf, format="JPEG") + return buf.getvalue() + + +def _image_size(data: bytes) -> tuple[int, int]: + with Image.open(io.BytesIO(data)) as img: + return img.size + + def _lazy_client(*, return_value=None, side_effect=None): async def invoke(prepare_media=None, **_kwargs): if prepare_media is not None: @@ -176,6 +202,32 @@ async def test_media_concurrency_bounds_staging_and_inference(monkeypatch): assert fs.peak_reads == 2 +async def test_image_summary_downsamples_large_model_input(monkeypatch): + original = _jpeg_bytes(80, 220) + fs = _FS(original) + vlm = _ImageVLM() + config = SimpleNamespace( + vlm=vlm, + image=ImageConfig( + preview_max_dimension=64, + max_file_size_mb=100.0, + large_image_threshold_dimension=100, + ), + ) + monkeypatch.setattr(media_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr(media_utils, "get_openviking_config", lambda: config) + + result = await media_utils.generate_image_summary( + "viking://resources/docs/large.jpg", + "large.jpg", + ) + + assert result == {"name": "large.jpg", "summary": "image summary"} + assert len(vlm.images) == 1 + assert max(_image_size(vlm.images[0])) <= 64 + assert fs.content == original + + async def test_unknown_size_media_stops_at_hard_staging_limit( monkeypatch, ): diff --git a/tests/unit/test_image_search.py b/tests/unit/test_image_search.py index 070cdcad18..ddcf4d7d54 100644 --- a/tests/unit/test_image_search.py +++ b/tests/unit/test_image_search.py @@ -1,9 +1,16 @@ +import base64 +import io + +from PIL import Image + from openviking.models.embedder.base import DenseEmbedderBase, EmbedResult from openviking.utils.image_search import ( build_multimodal_embedding_input, image_bytes_to_data_uri, + image_bytes_to_model_data_uri, normalize_client_image_input, ) +from openviking_cli.utils.config.parser_config import ImageConfig class TextOnlyEmbedder(DenseEmbedderBase): @@ -36,3 +43,24 @@ def test_image_helpers_accept_base64_bytes_and_image_uris(tmp_path): assert image_bytes_to_data_uri(b"png").startswith("data:image/png;base64,") assert normalize_client_image_input(image_path).startswith("data:image/jpeg;base64,") + + +def test_model_image_input_downsamples_jpeg_incompatible_modes(): + buf = io.BytesIO() + Image.new("LA", (80, 220), color=(255, 255)).save(buf, format="PNG") + + data_uri = image_bytes_to_model_data_uri( + buf.getvalue(), + "large-la.png", + config=ImageConfig( + preview_max_dimension=64, + max_file_size_mb=100.0, + large_image_threshold_dimension=100, + ), + ) + + assert data_uri.startswith("data:image/jpeg;base64,") + _, encoded = data_uri.split(";base64,", 1) + with Image.open(io.BytesIO(base64.b64decode(encoded))) as img: + assert img.mode == "RGB" + assert max(img.size) <= 64 diff --git a/tests/unit/test_vectorize_file_strategy.py b/tests/unit/test_vectorize_file_strategy.py index 0e8c5716ee..c5ff22d0a8 100644 --- a/tests/unit/test_vectorize_file_strategy.py +++ b/tests/unit/test_vectorize_file_strategy.py @@ -1,6 +1,9 @@ +import base64 +import io import types import pytest +from PIL import Image from openviking.core.context import ResourceContentType from openviking.parse.parsers.media.utils import ( @@ -9,6 +12,7 @@ ) from openviking.utils import embedding_utils from openviking.utils.ingest_options import IngestOptions +from openviking_cli.utils.config.parser_config import ImageConfig class DummyQueue: @@ -88,6 +92,18 @@ def __init__(self): self.account_id = "default" +def _jpeg_bytes(width: int, height: int) -> bytes: + buf = io.BytesIO() + Image.new("RGB", (width, height), "white").save(buf, format="JPEG") + return buf.getvalue() + + +def _data_uri_image_size(data_uri: str) -> tuple[int, int]: + _, encoded = data_uri.split(";base64,", 1) + with Image.open(io.BytesIO(base64.b64decode(encoded))) as img: + return img.size + + def test_get_resource_content_type_recognizes_media_extensions(): expected = { "recording.ogg": ResourceContentType.AUDIO, @@ -172,6 +188,42 @@ async def test_vectorize_file_uses_summary_first(monkeypatch): assert "content" not in queue.items[0].context_data +@pytest.mark.asyncio +async def test_vectorize_image_downsamples_large_embedding_input(monkeypatch): + queue = DummyQueue() + original = _jpeg_bytes(80, 220) + fs = DummyFS(original) + monkeypatch.setattr(embedding_utils, "get_queue_manager", lambda: DummyQueueManager(queue)) + monkeypatch.setattr(embedding_utils, "get_viking_fs", lambda: fs) + monkeypatch.setattr( + embedding_utils, + "get_openviking_config", + lambda: types.SimpleNamespace( + embedding=types.SimpleNamespace(text_source="content_only", max_input_tokens=1000), + image=ImageConfig( + preview_max_dimension=64, + max_file_size_mb=100.0, + large_image_threshold_dimension=100, + ), + ), + ) + + await embedding_utils.vectorize_file( + file_path="viking://resources/docs/large.jpg", + summary_dict={"name": "large.jpg", "summary": "large image"}, + parent_uri="viking://resources/docs", + ctx=DummyReq(), + ) + + assert len(queue.items) == 1 + message = queue.items[0].message + image_part = next(part for part in message if part["type"] == "image_url") + width, height = _data_uri_image_size(image_part["image_url"]["url"]) + assert max(width, height) <= 64 + assert fs.content == original + assert fs.read_file_bytes_calls == 1 + + @pytest.mark.asyncio async def test_vectorize_file_registers_request_wait_with_embedding_msg_id(monkeypatch): queue = DummyQueueWithId()