diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md index 4319fe6c76..b7ce9dabb3 100644 --- a/docs/en/api/02-resources.md +++ b/docs/en/api/02-resources.md @@ -12,7 +12,7 @@ OpenViking supports various resource types, categorized by functionality: | Type | Extensions | Description | |------|------------|-------------| -| PDF | `.pdf` | Supports local parsing and MinerU API conversion | +| PDF | `.pdf` | Locally extracts text, tables, images, and document structure | | Markdown | `.md`, `.markdown`, `.mdown`, `.mkd` | Native support, extracts structure and stores in segments | | HTML | `.html`, `.htm` | Cleans navigation/ads and extracts content, converts to Markdown | | Word | `.docx` | Extracts text, headings, tables and converts to Markdown | diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md index 6eaf34ce1a..757d0c01bd 100644 --- a/docs/zh/api/02-resources.md +++ b/docs/zh/api/02-resources.md @@ -11,7 +11,7 @@ OpenViking 支持多种资源类型，按照功能分类如下： 文档类 | 类型 | 扩展名 | 说明 | |------|--------|------| -| PDF | `.pdf` | 支持本地解析和 MinerU API 转换 | +| PDF | `.pdf` | 本地提取文本、表格、图片和文档结构 | | Markdown | `.md`, `.markdown`, `.mdown`, `.mkd` | 原生支持，会提取结构并分段存储 | | HTML | `.html`, `.htm` | 清理导航/广告后提取内容，转换为 Markdown | | Word | `.docx` | 提取文本、标题、表格并转换为 Markdown | diff --git a/examples/ov.conf.example b/examples/ov.conf.example index 9056a72865..5777ea6259 100644 --- a/examples/ov.conf.example +++ b/examples/ov.conf.example @@ -223,14 +223,12 @@ }, "parsers": { "pdf": { - "strategy": "auto", "max_content_length": 100000, "max_section_size": 4000, "section_size_flexibility": 0.3, "max_section_chars": 6000, - "mineru_endpoint": "https://mineru.example.com/api/v1", - "mineru_api_key": "{your-mineru-api-key}", - "mineru_timeout": 300.0, + "heading_detection": "auto", + "image_resolution": 300, }, "code": { "github_raw_domain": "raw.githubusercontent.com", diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.py index 4d7acca204..e91ee54f2c 100644 --- a/openviking/parse/parsers/pdf.py +++ b/openviking/parse/parsers/pdf.py @@ -3,13 +3,8 @@ """ PDF parser for OpenViking. -Unified parser that converts PDF to Markdown then parses the result. -Supports dual strategy: -- Local: pdfplumber for direct conversion -- Remote: MinerU API for advanced conversion - -This design simplifies PDF handling by delegating structure analysis -to the MarkdownParser after conversion. +Converts PDF to Markdown with pdfplumber, then delegates structure analysis +to the MarkdownParser. """ import asyncio @@ -37,30 +32,15 @@ class PDFParser(BaseParser): """ - PDF parser with dual conversion strategy. + PDF parser backed by pdfplumber. Converts PDF → Markdown → ParseResult using MarkdownParser. When available, extracts PDF bookmarks/outlines and injects them as markdown headings so MarkdownParser can build a hierarchical directory structure instead of flat numbered files. - Strategies: - - "local": Use pdfplumber for text and table extraction - - "mineru": Use MinerU API for advanced PDF processing - - "auto": Try local first, fallback to MinerU if configured - Examples: - >>> # Local parsing - >>> parser = PDFParser(PDFConfig(strategy="local")) - >>> result = await parser.parse("document.pdf") - - >>> # Remote API parsing - >>> config = PDFConfig( - ... strategy="mineru", - ... mineru_endpoint="https://api.example.com/convert", - ... mineru_api_key="key" - ... ) - >>> parser = PDFParser(config) + >>> parser = PDFParser(PDFConfig()) >>> result = await parser.parse("document.pdf") """ @@ -69,7 +49,7 @@ def __init__(self, config: Optional[PDFConfig] = None): Initialize PDF parser. Args: - config: PDFConfig instance (defaults to auto strategy) + config: PDFConfig instance """ self.config = config or PDFConfig() self.config.validate() @@ -103,7 +83,7 @@ async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) Raises: FileNotFoundError: If PDF file doesn't exist - ValueError: If conversion fails with all strategies + ValueError: If PDF conversion fails """ start_time = time.time() pdf_path = Path(source) @@ -146,10 +126,9 @@ async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) # Step 3: Update metadata for PDF origin result.source_format = "pdf" # Override markdown format result.parser_name = "PDFParser" - result.parser_version = "2.0" + result.parser_version = "3.0" result.parse_time = time.time() - start_time result.meta.update(conversion_meta) - result.meta["pdf_strategy"] = self.config.strategy result.meta["intermediate_markdown_length"] = len(markdown_content) result.meta["intermediate_markdown_preview"] = markdown_content[:500] @@ -175,60 +154,31 @@ async def parse(self, source: Union[str, Path], instruction: str = "", **kwargs) async def _convert_to_markdown( self, pdf_path: Path, + storage=None, resource_name: Optional[str] = None, ) -> tuple[str, Dict[str, Any]]: """ - Convert PDF to Markdown using configured strategy. + Convert PDF to Markdown with pdfplumber without blocking the event loop. Args: pdf_path: Path to PDF file + storage: Optional storage instance for extracted images resource_name: Optional resource name for organizing saved images Returns: Tuple of (markdown_content, metadata_dict) - - Raises: - ValueError: If all conversion strategies fail """ - if self.config.strategy == "local": - return await self._convert_local(pdf_path, resource_name=resource_name) - - elif self.config.strategy == "mineru": - return await self._convert_mineru(pdf_path, resource_name=resource_name) - - elif self.config.strategy == "auto": - # Try local first - try: - return await self._convert_local(pdf_path, resource_name=resource_name) - except Exception as e: - logger.warning(f"Local conversion failed: {e}") - - # Fallback to MinerU if configured - if self.config.mineru_endpoint: - logger.info("Falling back to MinerU API") - return await self._convert_mineru(pdf_path, resource_name=resource_name) - else: - raise ValueError( - f"Local conversion failed and no MinerU endpoint configured: {e}" - ) - - else: - raise ValueError(f"Unknown strategy: {self.config.strategy}") - - async def _convert_local( - self, pdf_path: Path, storage=None, resource_name: Optional[str] = None - ) -> tuple[str, Dict[str, Any]]: - # pdfplumber / pdfminer 的解析与图片/表格提取通常是 CPU/IO 密集且为同步实现， - # 放到线程池中执行，避免阻塞事件循环。 - return await asyncio.to_thread(self._convert_local_sync, pdf_path, storage, resource_name) + return await asyncio.to_thread( + self._convert_to_markdown_sync, + pdf_path, + storage, + resource_name, + ) - def _convert_local_sync( + def _convert_to_markdown_sync( self, pdf_path: Path, storage=None, resource_name: Optional[str] = None ) -> tuple[str, Dict[str, Any]]: - """同步版：用 pdfplumber 将 PDF 转 Markdown。 - - 该方法会在 :meth:`_convert_local` 中通过 asyncio.to_thread 调用。 - """ + """Convert PDF to Markdown synchronously with pdfplumber.""" pdfplumber = lazy_import("pdfplumber") # Import storage utilities @@ -242,7 +192,6 @@ def _convert_local_sync( parts = [] meta = { - "strategy": "local", "library": "pdfplumber", "pages_processed": 0, "images_extracted": 0, @@ -679,83 +628,6 @@ def _extract_image_from_page(self, page, img_info: dict) -> Optional[bytes]: logger.debug(f"Image extraction error: {e}") return None - async def _convert_mineru( - self, - pdf_path: Path, - resource_name: Optional[str] = None, - ) -> tuple[str, Dict[str, Any]]: - """ - Convert PDF to Markdown using MinerU API. - - Args: - pdf_path: Path to PDF file - resource_name: Optional resource name (unused in MinerU conversion) - - Returns: - Tuple of (markdown_content, metadata) - - Raises: - ImportError: If httpx not installed - Exception: If API call fails - """ - httpx = lazy_import("httpx") - - if not self.config.mineru_endpoint: - raise ValueError("MinerU endpoint not configured") - - meta = { - "strategy": "mineru", - "endpoint": self.config.mineru_endpoint, - "api_version": None, - } - - try: - async with httpx.AsyncClient(timeout=self.config.mineru_timeout) as client: - # Prepare file upload - with open(pdf_path, "rb") as f: - files = {"file": (pdf_path.name, f, "application/pdf")} - - # Prepare headers - headers = {} - if self.config.mineru_api_key: - headers["Authorization"] = f"Bearer {self.config.mineru_api_key}" - - # Prepare request params - params = self.config.mineru_params or {} - - # Make API request - logger.info(f"Calling MinerU API: {self.config.mineru_endpoint}") - response = await client.post( - self.config.mineru_endpoint, - files=files, - headers=headers, - params=params, - ) - response.raise_for_status() - - # Parse response - result = response.json() - markdown_content = result.get("markdown", "") - - # Extract metadata from response - meta["api_version"] = result.get("version") - meta["processing_time"] = result.get("processing_time") - meta["total_pages"] = result.get("total_pages") - - if not markdown_content: - logger.warning(f"MinerU returned empty content for {pdf_path}") - - logger.info( - f"MinerU conversion: {meta.get('total_pages', '?')} pages → " - f"{len(markdown_content)} chars" - ) - - return markdown_content, meta - - except Exception as e: - logger.error(f"MinerU API call failed: {e}") - raise - def _format_table_markdown(self, table: List[List[Optional[str]]]) -> str: """ Convert table data to Markdown table format. diff --git a/openviking_cli/utils/config/parser_config.py b/openviking_cli/utils/config/parser_config.py index 83fd271c20..a1ad41d60d 100644 --- a/openviking_cli/utils/config/parser_config.py +++ b/openviking_cli/utils/config/parser_config.py @@ -140,27 +140,13 @@ class PDFConfig(ParserConfig): """ Configuration for PDF parsing. - Supports three strategies: - - "local": Use pdfplumber for local PDF→Markdown conversion - - "mineru": Use MinerU API for remote PDF→Markdown conversion - - "auto": Try local first, fallback to MinerU if available - Attributes: - strategy: Parsing strategy ("local" | "mineru" | "auto") - mineru_endpoint: MinerU API endpoint URL - mineru_api_key: MinerU API authentication key - mineru_timeout: MinerU request timeout in seconds - mineru_params: Additional MinerU API parameters + heading_detection: Heading detection mode + font_heading_min_delta: Minimum font size delta from body text + max_heading_levels: Maximum heading levels for font analysis + image_resolution: Rendering DPI for extracted image regions """ - strategy: str = "auto" # "local" | "mineru" | "auto" - - # MinerU API configuration - mineru_endpoint: Optional[str] = None # API endpoint URL - mineru_api_key: Optional[str] = None # API authentication key - mineru_timeout: float = 300.0 # Request timeout in seconds (5 minutes) - mineru_params: Optional[dict] = None # Additional API parameters - # Heading detection configuration heading_detection: str = "auto" # "bookmarks" | "font" | "auto" | "none" font_heading_min_delta: float = 1.5 # Minimum font size delta from body text (pt) @@ -180,18 +166,6 @@ def validate(self) -> None: super().validate() # Validate PDF-specific fields - if self.strategy not in ("local", "mineru", "auto"): - raise ValueError( - f"Invalid strategy '{self.strategy}'. Must be 'local', 'mineru', or 'auto'" - ) - - if self.strategy == "mineru": - if not self.mineru_endpoint: - raise ValueError("mineru_endpoint is required when strategy='mineru'") - - if self.mineru_timeout <= 0: - raise ValueError("mineru_timeout must be positive") - if self.heading_detection not in ("bookmarks", "font", "auto", "none"): raise ValueError(f"Invalid heading_detection: {self.heading_detection}") @@ -840,7 +814,7 @@ def load_parser_configs_from_dict(config_dict: Dict[str, Any]) -> Dict[str, Pars Examples: >>> configs = load_parser_configs_from_dict({ - ... "pdf": {"strategy": "auto"}, + ... "pdf": {"heading_detection": "auto"}, ... "code": {"github_raw_domain": "raw.githubusercontent.com"} ... }) >>> pdf_config = configs["pdf"] diff --git a/tests/parse/test_markdown_no_split.py b/tests/parse/test_markdown_no_split.py index 62f5d18a6f..5d4adfca2e 100644 --- a/tests/parse/test_markdown_no_split.py +++ b/tests/parse/test_markdown_no_split.py @@ -102,9 +102,7 @@ async def test_pdf_no_split_converts_to_one_complete_markdown( source.write_bytes(b"%PDF-fixture") content = _long_markdown(with_headings=True) fake_fs = _FakeVikingFS() - parser = PDFParser( - PDFConfig(strategy="local", max_section_size=32, max_section_chars=128) - ) + parser = PDFParser(PDFConfig(max_section_size=32, max_section_chars=128)) markdown_parser = parser._get_markdown_parser() monkeypatch.setattr(markdown_parser, "_get_viking_fs", lambda: fake_fs) monkeypatch.setattr( diff --git a/tests/parse/test_parser_config_wiring.py b/tests/parse/test_parser_config_wiring.py index aaae5cc4c5..fe199a89a2 100644 --- a/tests/parse/test_parser_config_wiring.py +++ b/tests/parse/test_parser_config_wiring.py @@ -9,7 +9,7 @@ def test_pdf_parser_passes_its_config_to_nested_markdown_parser(): - parser = PDFParser(PDFConfig(strategy="local", max_section_size=2222, max_section_chars=5555)) + parser = PDFParser(PDFConfig(max_section_size=2222, max_section_chars=5555)) markdown_parser = parser._get_markdown_parser() diff --git a/tests/parse/test_pdf_bookmark_extraction.py b/tests/parse/test_pdf_bookmark_extraction.py index 441540e516..7b24bed189 100644 --- a/tests/parse/test_pdf_bookmark_extraction.py +++ b/tests/parse/test_pdf_bookmark_extraction.py @@ -4,7 +4,7 @@ Tests for PDF bookmark/outline extraction in PDFParser. Verifies that _extract_bookmarks correctly extracts bookmark entries -and that _convert_local injects them as markdown headings. +and that PDF conversion injects them as markdown headings. """ from contextlib import nullcontext @@ -90,7 +90,7 @@ def crop(self, bbox): class _FakePage: - """Minimal pdfplumber page stub for _convert_local tests.""" + """Minimal pdfplumber page stub for PDF conversion tests.""" def __init__(self, text: str): self._text = text @@ -267,11 +267,11 @@ def test_extract_bookmarks_exception_returns_empty(self): assert bookmarks == [] -class TestConvertLocalBookmarks: - """Test bookmark injection behavior in local PDF conversion.""" +class TestConvertPDFBookmarks: + """Test bookmark injection behavior in PDF conversion.""" @pytest.mark.asyncio - async def test_convert_local_skips_unresolved_bookmarks(self): + async def test_convert_to_markdown_skips_unresolved_bookmarks(self): parser = PDFParser() fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")]) fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) @@ -287,7 +287,7 @@ async def test_convert_local_skips_unresolved_bookmarks(self): ], ), ): - markdown, meta = await parser._convert_local( + markdown, meta = await parser._convert_to_markdown( "dummy.pdf", storage=MagicMock(), resource_name="dummy" ) @@ -300,7 +300,7 @@ async def test_convert_local_skips_unresolved_bookmarks(self): assert meta["heading_source"] == "bookmarks" @pytest.mark.asyncio - async def test_convert_local_falls_back_to_font_when_bookmarks_unresolved(self): + async def test_convert_to_markdown_falls_back_to_font_when_bookmarks_unresolved(self): parser = PDFParser() fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")]) fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf)) @@ -318,7 +318,7 @@ async def test_convert_local_falls_back_to_font_when_bookmarks_unresolved(self): return_value=[{"level": 1, "title": "Font Heading", "page_num": 2}], ), ): - markdown, meta = await parser._convert_local( + markdown, meta = await parser._convert_to_markdown( "dummy.pdf", storage=MagicMock(), resource_name="dummy" ) @@ -331,7 +331,7 @@ async def test_convert_local_falls_back_to_font_when_bookmarks_unresolved(self): assert meta["heading_source"] == "font_analysis" @pytest.mark.asyncio - async def test_convert_local_skips_images_stacked_on_same_bbox(self): + async def test_convert_to_markdown_skips_images_stacked_on_same_bbox(self): """Two image XObjects at the same spot must render (and save) only once.""" parser = PDFParser() stacked = {"x0": 0.0, "top": 0.1, "x1": 595.0, "bottom": 841.9} @@ -349,7 +349,7 @@ async def test_convert_local_skips_images_stacked_on_same_bbox(self): patch.object(parser, "_detect_headings_by_font", return_value=[]), patch.object(parser, "_extract_image_from_page", return_value=b"png") as extract, ): - markdown, meta = await parser._convert_local( + markdown, meta = await parser._convert_to_markdown( "dummy.pdf", storage=storage, resource_name="dummy" ) @@ -361,7 +361,7 @@ async def test_convert_local_skips_images_stacked_on_same_bbox(self): assert markdown.count("![Page 1 Image") == 1 @pytest.mark.asyncio - async def test_convert_local_skips_images_rendering_to_same_bytes(self): + async def test_convert_to_markdown_skips_images_rendering_to_same_bytes(self): """Distinct bboxes that still render identically are caught by the hash.""" parser = PDFParser() page = _FakePage("Page one") @@ -381,7 +381,7 @@ async def test_convert_local_skips_images_rendering_to_same_bytes(self): patch.object(parser, "_detect_headings_by_font", return_value=[]), patch.object(parser, "_extract_image_from_page", return_value=b"png") as extract, ): - _markdown, meta = await parser._convert_local( + _markdown, meta = await parser._convert_to_markdown( "dummy.pdf", storage=storage, resource_name="dummy" ) @@ -392,7 +392,7 @@ async def test_convert_local_skips_images_rendering_to_same_bytes(self): assert meta["images_deduplicated"] == 1 @pytest.mark.asyncio - async def test_convert_local_keeps_distinct_images_and_repeats_across_pages(self): + async def test_convert_to_markdown_keeps_distinct_images_and_repeats_across_pages(self): """Dedup is per-page: a logo on every page survives on every page.""" parser = PDFParser() logo = {"x0": 0.0, "top": 0.0, "x1": 50.0, "bottom": 50.0} @@ -423,7 +423,7 @@ async def test_convert_local_keeps_distinct_images_and_repeats_across_pages(self ], ), ): - _markdown, meta = await parser._convert_local( + _markdown, meta = await parser._convert_to_markdown( "dummy.pdf", storage=storage, resource_name="dummy" ) @@ -431,7 +431,7 @@ async def test_convert_local_keeps_distinct_images_and_repeats_across_pages(self assert meta["images_deduplicated"] == 0 @pytest.mark.asyncio - async def test_convert_local_closes_page_after_each_page(self): + async def test_convert_to_markdown_closes_page_after_each_page(self): parser = PDFParser() pages = [_FakePage("Page one"), _FakePage("Page two")] fake_pdf = SimpleNamespace(pages=pages) @@ -442,7 +442,9 @@ async def test_convert_local_closes_page_after_each_page(self): patch.object(parser, "_extract_bookmarks", return_value=[]), patch.object(parser, "_detect_headings_by_font", return_value=[]), ): - await parser._convert_local("dummy.pdf", storage=MagicMock(), resource_name="dummy") + await parser._convert_to_markdown( + "dummy.pdf", storage=MagicMock(), resource_name="dummy" + ) assert [page.close_count for page in pages] == [1, 1] assert [page.flush_count for page in pages] == [1, 1]