From 8a9f590e0872c6f855fe950fafdbfc837e0b3b16 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 14:58:41 +0800 Subject: [PATCH 01/14] refactor(filesystem): simplify PIFS core surface Align the CLI, public Python API, SQLite schemas, and summary projection with the locally approved PR #302 simplification spec. Add fail-first behavior coverage and real migration compatibility fixtures. --- examples/pifs_demo.py | 795 +----------- pageindex/filesystem/__init__.py | 50 +- pageindex/filesystem/agent.py | 25 +- pageindex/filesystem/cli.py | 36 +- pageindex/filesystem/commands.py | 117 +- pageindex/filesystem/core.py | 602 ++------- pageindex/filesystem/semantic_index.py | 62 +- pageindex/filesystem/semantic_projection.py | 627 ++++----- pageindex/filesystem/store.py | 345 +---- pageindex/filesystem/types.py | 46 - tests/test_filesystem_store.py | 7 +- tests/test_pifs_add_command.py | 484 ------- tests/test_pifs_agent_stream.py | 49 +- tests/test_pifs_cli.py | 81 +- tests/test_pifs_core_alignment.py | 356 ------ tests/test_pifs_like_escape.py | 22 +- tests/test_pifs_register_side_effects.py | 77 -- tests/test_pifs_scope_paths.py | 711 ----------- tests/test_pifs_simplification_acceptance.py | 1192 ++++++++++++++++++ tests/test_pifs_string_metadata_cleanup.py | 189 --- tests/test_semantic_index.py | 422 ++----- 21 files changed, 1881 insertions(+), 4414 deletions(-) delete mode 100644 tests/test_pifs_add_command.py delete mode 100644 tests/test_pifs_core_alignment.py delete mode 100644 tests/test_pifs_register_side_effects.py delete mode 100644 tests/test_pifs_scope_paths.py create mode 100644 tests/test_pifs_simplification_acceptance.py delete mode 100644 tests/test_pifs_string_metadata_cleanup.py diff --git a/examples/pifs_demo.py b/examples/pifs_demo.py index 4f2887986..9a7e22a78 100644 --- a/examples/pifs_demo.py +++ b/examples/pifs_demo.py @@ -1,781 +1,46 @@ -""" -PageIndex FileSystem (PIFS) agent demo. - -This mirrors examples/agentic_vectorless_rag_demo.py, but exposes a corpus -through the PageIndex FileSystem shell instead of direct PageIndex document -tools. The agent receives one read-only bash-like PIFS tool and must retrieve -evidence through tree, browse, stat, cat --structure, cat --page, and -single-file grep. - -The demo registers supported files under examples/documents. When a matching -examples/documents/results/*_structure.json file exists, it is loaded into the -PIFS workspace's PageIndexClient cache. Files without a cache exercise the -normal PageIndexClient.index() path during register(). - -Requirements: - pip install openai-agents +"""A minimal PageIndex FileSystem CLI walkthrough. -Example: - python examples/pifs_demo.py --stream-mode all --verbose +Configure the Summary Embedding Profile in your PIFS config and provide its API +key through ``PIFS_EMBEDDING_API_KEY`` before running this example. """ from __future__ import annotations import argparse -import hashlib -import json -import os -import re -import shlex -import shutil -import sys -import time +import subprocess from pathlib import Path -from typing import Any - -import PyPDF2 - -sys.path.insert(0, str(Path(__file__).parent.parent)) - -# Keep the local demo quiet in offline environments. -os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "true") -from pageindex import PageIndexClient -from pageindex.filesystem import MetadataGenerator, PageIndexFileSystem, PIFSCommandExecutor -from pageindex.filesystem.agent import run_pifs_agent -from pageindex.filesystem.core import DEFAULT_EMBEDDING_DIMENSIONS - -EXAMPLES_DIR = Path(__file__).parent -DOCUMENTS_DIR = EXAMPLES_DIR / "documents" -WORKSPACE = EXAMPLES_DIR / "pifs_workspace" -DEFAULT_MODEL = os.environ.get("PIFS_DEMO_MODEL", "gpt-5.4") -DEFAULT_METADATA_PROVIDER = os.environ.get("PIFS_DEMO_METADATA_PROVIDER") or os.environ.get( - "PIFS_METADATA_PROVIDER", "openai" -) -DEFAULT_EMBEDDING_PROVIDER = os.environ.get("PIFS_DEMO_EMBEDDING_PROVIDER") or os.environ.get( - "PIFS_EMBEDDING_PROVIDER", "openai" -) -DEFAULT_QUESTION = ( - "Use the PIFS workspace to find the Federal Reserve annual report. " - "Which section covers supervision and regulation, and what page range " - "should I inspect? Cite the document and evidence you used." -) - -PIFS_DEMO_AGENT_PROMPT = """ -You are a PageIndex FileSystem retrieval agent for a local demo workspace. - -Use only the bash tool. It is a read-only PIFS virtual shell, not a real OS -shell. The workspace contains registered example PDFs. - -Retrieval strategy: -- Start with tree to understand the workspace. -- Use concrete PIFS paths from tree/browse output, such as /documents/report.pdf, - or stable file_ref/document ids. Do not invent temporary ref_N aliases. -- Folder paths such as /documents are positional command targets; do not put - folder paths inside --where. -- After choosing a folder, use browse with a required quoted query to find - likely files, for example: - browse /documents "Federal Reserve supervision regulation" -- If the folder is uncertain, use recursive browse from a structural parent, - for example: - browse -R /documents "Federal Reserve supervision regulation" -- Use recursive browse only after inspecting plausible folders or rephrasing. -- browse returns document candidates only; it is not final evidence. -- After browse returns candidates, verify evidence with grep, cat - --structure, or cat --page before answering. -- Use browse --where with JSON metadata DSL when metadata pruning is needed. -- Use grep only for one selected file. -- Do not use find, recursive grep, folder grep, pipes, browse spaces, or stat - schema/field modes. -- Run one evidence command at a time. Do not chain large commands like - cat --structure, grep, and cat --page in one bash call. -- For PDFs, use cat --structure to inspect the PageIndex tree, then - cat --page for evidence, for example: - cat /documents/2023-annual-report.pdf --page 31-35 -- Do not use cat --page as the first inspection command for a selected PDF. - Run cat --structure for that same target first, then choose pages. -- Do not guess cat --page ranges from grep line numbers. -- For page-range questions, use cat --structure to identify the full section - range. Then run cat --page on the smallest useful evidence range, usually the - section start page or first 1-2 pages, before the final answer. Do not print - a broad multi-page section unless the user asks to read the whole section. -- Answer only from PIFS tool output and cite file refs or document ids. -""" +def run(workspace: Path, document: Path, query: str) -> int: + target = f"/documents/{document.name}" + commands = [ + ["add", str(document), "/documents"], + ["tree", "/documents", "-L", "1"], + ["browse", "/documents", query], + ["stat", target], + ["cat", target, "--structure"], + ["cat", target, "--page", "1"], + ["grep", query, target], + ] + for command in commands: + print(f"\n$ pifs {' '.join(command)}") + result = subprocess.run( + ["pifs", "--workspace", str(workspace), *command], + check=False, + ) + if result.returncode: + return result.returncode + return 0 def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Run a PIFS document retrieval agent demo.") - parser.add_argument("--workspace", type=Path, default=WORKSPACE) - parser.add_argument("--documents-dir", type=Path, default=DOCUMENTS_DIR) - parser.add_argument( - "--document", - action="append", - default=[], - help="Specific document filename or path to register. May be repeated.", - ) - parser.add_argument( - "--max-docs", - type=int, - default=0, - help="Limit number of cached example documents to register. 0 means all.", - ) - parser.add_argument("--reset", action="store_true", help="Delete and rebuild the demo workspace.") - parser.add_argument( - "--prepare-only", - action="store_true", - help="Register documents and print PIFS smoke commands without running the agent.", - ) - parser.add_argument("--question", default=DEFAULT_QUESTION) - parser.add_argument("--model", default=DEFAULT_MODEL) - parser.add_argument( - "--metadata-provider", - default=DEFAULT_METADATA_PROVIDER, - help="Provider used for register-time metadata generation.", - ) - parser.add_argument( - "--metadata-model", - default=os.environ.get("PIFS_METADATA_MODEL", "gpt-5-nano"), - help="Model used for register-time metadata generation.", - ) - parser.add_argument("--stream-mode", default="all", choices=["off", "tools", "model", "all"]) - parser.add_argument("--verbose", action="store_true") - parser.add_argument("--max-turns", type=int, default=12) - parser.add_argument("--max-seconds", type=float, default=90) - parser.add_argument("--reasoning-effort", default=None) - parser.add_argument("--reasoning-summary", default="auto") - parser.add_argument( - "--embedding-provider", - default=DEFAULT_EMBEDDING_PROVIDER, - help="Provider used for register-time summary projection embeddings.", - ) - parser.add_argument( - "--embedding-model", - default=os.environ.get("PIFS_DEMO_EMBEDDING_MODEL", "text-embedding-3-small"), - help="Embedding model used for register-time summary projection.", - ) - parser.add_argument( - "--embedding-dimensions", - type=int, - default=DEFAULT_EMBEDDING_DIMENSIONS, - ) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("document", type=Path, help="PDF or Markdown document") + parser.add_argument("--workspace", type=Path, default=Path("pifs-workspace")) + parser.add_argument("--query", default="key findings") return parser.parse_args() -def require_runtime_environment(*, metadata_provider: str, embedding_provider: str) -> None: - metadata_provider = metadata_provider.lower() - embedding_provider = embedding_provider.lower() - missing: list[str] = [] - if not os.environ.get("OPENAI_API_KEY"): - missing.append("OPENAI_API_KEY for the OpenAI Agents SDK demo agent") - if metadata_provider == "openai" and not ( - os.environ.get("PIFS_METADATA_API_KEY") or os.environ.get("OPENAI_API_KEY") - ): - missing.append("PIFS_METADATA_API_KEY or OPENAI_API_KEY for metadata generation") - if embedding_provider == "openai" and not ( - os.environ.get("PIFS_EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY") - ): - missing.append("PIFS_EMBEDDING_API_KEY or OPENAI_API_KEY for summary embeddings") - if missing: - raise RuntimeError( - "Missing required environment variable(s): " - + "; ".join(missing) - + ". Source your .env or export the required key before running." - ) - - -SUPPORTED_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown", ".txt", ".text"} - - -def discover_documents(documents_dir: Path) -> list[Path]: - return sorted( - path - for path in documents_dir.iterdir() - if path.is_file() and path.suffix.lower() in SUPPORTED_DOCUMENT_SUFFIXES - ) - - -def resolve_requested_documents(documents_dir: Path, requested: list[str]) -> list[Path]: - if not requested: - return discover_documents(documents_dir) - paths: list[Path] = [] - for item in requested: - path = Path(item).expanduser() - if not path.is_absolute(): - path = documents_dir / path - if not path.exists(): - raise FileNotFoundError(f"document not found: {path}") - paths.append(path) - return paths - - -def structure_path_for(document_path: Path, documents_dir: Path) -> Path | None: - path = documents_dir / "results" / f"{document_path.stem}_structure.json" - return path if path.exists() else None - - -def deterministic_doc_id(document_path: Path) -> str: - digest = hashlib.sha1(str(document_path.resolve()).encode("utf-8")).hexdigest()[:16] - return f"pifs_demo_{digest}" - - -def read_pdf_pages(document_path: Path) -> list[dict[str, Any]]: - pages: list[dict[str, Any]] = [] - with document_path.open("rb") as handle: - reader = PyPDF2.PdfReader(handle) - for page_num, page in enumerate(reader.pages, 1): - pages.append({"page": page_num, "content": page.extract_text() or ""}) - return pages - - -def load_structure_json(structure_path: Path) -> dict[str, Any]: - with structure_path.open("r", encoding="utf-8") as handle: - payload = json.load(handle) - if not isinstance(payload, dict) or not isinstance(payload.get("structure"), list): - raise ValueError(f"invalid PageIndex structure cache: {structure_path}") - return payload - - -def seed_pageindex_cache( - filesystem: PageIndexFileSystem, - document_path: Path, - *, - documents_dir: Path, -) -> str | None: - structure_path = structure_path_for(document_path, documents_dir) - if structure_path is None: - return None - - filesystem.pageindex_client_workspace.mkdir(parents=True, exist_ok=True) - meta_path = filesystem.pageindex_client_workspace / "_meta.json" - if not meta_path.exists(): - meta_path.write_text("{}", encoding="utf-8") - client = PageIndexClient(workspace=str(filesystem.pageindex_client_workspace)) - canonical_path = str(document_path.resolve()) - for doc_id, doc in client.documents.items(): - if Path(str(doc.get("path") or "")).resolve(strict=False) == Path(canonical_path): - return doc_id - - payload = load_structure_json(structure_path) - doc_id = deterministic_doc_id(document_path) - suffix = document_path.suffix.lower() - if suffix == ".pdf": - pages = read_pdf_pages(document_path) - client.documents[doc_id] = { - "id": doc_id, - "type": "pdf", - "path": canonical_path, - "doc_name": payload.get("doc_name") or document_path.name, - "doc_description": payload.get("doc_description") or "", - "page_count": len(pages), - "structure": payload["structure"], - "pages": pages, - } - elif suffix in {".md", ".markdown"}: - text = document_path.read_text(encoding="utf-8") - client.documents[doc_id] = { - "id": doc_id, - "type": "md", - "path": canonical_path, - "doc_name": payload.get("doc_name") or document_path.name, - "doc_description": payload.get("doc_description") or "", - "line_count": len(text.splitlines()), - "structure": payload["structure"], - } - else: - return None - client._save_doc(doc_id) - return doc_id - - -def content_type_for(path: Path) -> str: - suffix = path.suffix.lower() - if suffix == ".pdf": - return "application/pdf" - if suffix in {".md", ".markdown"}: - return "text/markdown" - return "text/plain" - - -def external_id_for(path: Path) -> str: - slug = "".join(ch.lower() if ch.isalnum() else "_" for ch in path.stem).strip("_") - slug = "_".join(part for part in slug.split("_") if part) - return f"example_{slug}" - - -def log_progress(message: str, *, indent: int = 0) -> None: - prefix = " " * indent - print(f"{prefix}{message}", flush=True) - - -def register_demo_metadata_schema(filesystem: PageIndexFileSystem) -> None: - filesystem.metadata.register_schema( - { - "fields": { - "source_collection": { - "type": "string", - "description": "Local example corpus collection.", - }, - "file_format": { - "type": "string", - "description": "Source file extension without the leading dot.", - }, - } - }, - source="demo", - ) - - -def backfill_registered_metadata_values(filesystem: PageIndexFileSystem, file_ref: str) -> None: - entry = filesystem.store.get_file(file_ref) - indexed_metadata = dict(entry.metadata or {}) - with filesystem.store.connect() as conn: - filesystem.store.replace_metadata_values(conn, file_ref, indexed_metadata) - - -def configure_summary_projection_backend( - filesystem: PageIndexFileSystem, - *, - embedding_provider: str, - embedding_model: str, - embedding_dimensions: int, -) -> None: - if not (filesystem.summary_projection_index_dir / "summary.sqlite").exists(): - return - filesystem.configure_semantic_projection_retrieval( - filesystem.summary_projection_index_dir, - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - ) - - -def has_ready_register_outputs(filesystem: PageIndexFileSystem, external_id: str) -> bool: - try: - file_ref = filesystem.store.resolve_file_ref(external_id) - entry = filesystem.store.get_file(file_ref) - except KeyError: - return False - status = entry.metadata_status or {} - fields = status.get("fields") or {} - required = ("summary", "doc_type", "domain", "topic") - if any(fields.get(field, {}).get("status") != "generated" for field in required): - return False - summary_projection = (status.get("projection_indexes") or {}).get("summary") or {} - return summary_projection.get("status") == "ready" - - -def register_documents( - filesystem: PageIndexFileSystem, - documents: list[Path], - *, - documents_dir: Path, -) -> list[dict[str, Any]]: - registered: list[dict[str, Any]] = [] - total = len(documents) - for index, document_path in enumerate(documents, 1): - document_path = document_path.resolve() - external_id = external_id_for(document_path) - log_progress(f"[{index}/{total}] {document_path.name}") - log_progress("PageIndex tree cache: checking examples/documents/results", indent=1) - cache_started = time.perf_counter() - cached_doc_id = seed_pageindex_cache( - filesystem, - document_path, - documents_dir=documents_dir, - ) - cache_seconds = time.perf_counter() - cache_started - if cached_doc_id: - log_progress( - f"PageIndex tree cache: ready doc_id={cached_doc_id} ({cache_seconds:.2f}s)", - indent=1, - ) - else: - log_progress( - f"PageIndex tree cache: no cached structure; register() will index if supported ({cache_seconds:.2f}s)", - indent=1, - ) - if has_ready_register_outputs(filesystem, external_id): - file_ref = filesystem.store.resolve_file_ref(external_id) - backfill_registered_metadata_values(filesystem, file_ref) - log_progress( - f"PIFS register: cached file_ref={file_ref}; metadata and summary projection already ready", - indent=1, - ) - registered.append( - { - "file_ref": file_ref, - "external_id": external_id, - "path": str(document_path), - "status": "cached", - "pageindex_doc_id": cached_doc_id, - } - ) - continue - - log_progress( - "PIFS register: running register() -> metadata generation -> summary embedding -> sqlite upsert", - indent=1, - ) - register_started = time.perf_counter() - file_ref = filesystem.register( - storage_uri=document_path.as_uri(), - folder_path="/documents", - external_id=external_id, - title=document_path.name, - content_type=content_type_for(document_path), - source_type="examples-documents", - metadata={ - "title": document_path.name, - "source_collection": "examples/documents", - "file_format": document_path.suffix.lower().lstrip("."), - }, - ) - register_seconds = time.perf_counter() - register_started - entry = filesystem.store.get_file(file_ref) - field_status = { - field: state.get("status") - for field, state in (entry.metadata_status.get("fields") or {}).items() - } - summary_projection = ( - entry.metadata_status.get("projection_indexes", {}).get("summary", {}) - ) - log_progress( - f"PIFS register: done file_ref={file_ref} ({register_seconds:.2f}s)", - indent=1, - ) - log_progress( - f"metadata: {entry.metadata_status.get('status', 'unknown')} fields={field_status}", - indent=1, - ) - log_progress( - "summary projection: " - f"{summary_projection.get('status', 'not_requested')} " - f"index={summary_projection.get('index_path', '')}", - indent=1, - ) - registered.append( - { - "file_ref": file_ref, - "external_id": external_id, - "path": str(document_path), - "status": entry.metadata_status.get("status", "unknown"), - "pageindex_tree_status": entry.pageindex_tree_status, - "pageindex_doc_id": entry.pageindex_doc_id, - } - ) - return registered - - -def print_section(title: str) -> None: - print("\n" + "#" * 78, flush=True) - print(f"# {title}", flush=True) - print("#" * 78, flush=True) - - -def print_step(title: str, detail: str = "") -> None: - print(f"\n>>> {title}", flush=True) - if detail: - print(f" {detail}", flush=True) - - -def sanitize_preview_text(text: str) -> str: - cleaned = str(text).replace("\r", "\n").replace("\f", "\n") - cleaned = "".join( - ch if ch == "\n" or ch == "\t" or ord(ch) >= 32 else " " - for ch in cleaned - ) - return "\n".join( - re.sub(r"[ \t]{2,}", " ", line).strip() - for line in cleaned.splitlines() - ) - - -def compact_lines(text: str, *, max_lines: int = 6, max_chars: int = 900) -> str: - lines = [line for line in sanitize_preview_text(text).splitlines() if line.strip()] - preview = "\n".join(lines[:max_lines]) - if len(preview) > max_chars: - preview = preview[:max_chars].rstrip() + "..." - omitted = len(lines) - min(len(lines), max_lines) - if omitted > 0: - preview += f"\n ... {omitted} more lines" - return preview - - -def find_structure_node(structure: Any, title_fragment: str) -> dict[str, Any] | None: - if isinstance(structure, list): - for item in structure: - found = find_structure_node(item, title_fragment) - if found: - return found - return None - if not isinstance(structure, dict): - return None - if title_fragment.lower() in str(structure.get("title", "")).lower(): - return structure - return find_structure_node(structure.get("nodes", []), title_fragment) - - -def page_range_for_node(node: dict[str, Any] | None) -> str: - if not node: - return "" - ranges: list[tuple[int, int]] = [] - - def collect(item: Any) -> None: - if not isinstance(item, dict): - return - start = item.get("start_index") - end = item.get("end_index") - if isinstance(start, int) and isinstance(end, int): - ranges.append((start, end)) - for child in item.get("nodes") or []: - collect(child) - - collect(node) - if not ranges: - return "" - start = min(item[0] for item in ranges) - end = max(item[1] for item in ranges) - return f"{start}-{end}" if start != end else str(start) - - -def opening_page_range_for_node(node: dict[str, Any] | None, *, max_pages: int = 2) -> str: - if not node: - return "" - ranges: list[tuple[int, int]] = [] - - def collect(item: Any) -> None: - if not isinstance(item, dict): - return - start = item.get("start_index") - end = item.get("end_index") - if isinstance(start, int) and isinstance(end, int): - ranges.append((start, end)) - for child in item.get("nodes") or []: - collect(child) - - collect(node) - if not ranges: - return "" - start = min(item[0] for item in ranges) - end = max(item[1] for item in ranges) - preview_end = min(end, start + max_pages - 1) - return f"{start}-{preview_end}" if start != preview_end else str(start) - - -def execute_json_command(executor: PIFSCommandExecutor, command: str) -> dict[str, Any]: - try: - return json.loads(executor.execute(command)) - except Exception as exc: - return {"ok": False, "error": str(exc), "data": None} - - -def show_capability( - *, - label: str, - command: str, - result: str, - raw: str = "", - verbose: bool = False, -) -> None: - print_step(label, command) - print(f" result: {result}", flush=True) - if verbose and raw: - print(" raw:", flush=True) - print(compact_lines(raw, max_lines=10, max_chars=1600), flush=True) - - -def show_registered_documents(registered: list[dict[str, Any]], *, verbose: bool = False) -> None: - print(f"\nRegistered {len(registered)} document(s):", flush=True) - for item in registered: - print( - " - " - f"{Path(str(item.get('path', ''))).name}: " - f"file_ref={item.get('file_ref')} | " - f"status={item.get('status')} | " - f"pageindex_doc_id={item.get('pageindex_doc_id')}", - flush=True, - ) - if verbose: - print("\nRaw registration records:", flush=True) - print(json.dumps(registered, ensure_ascii=False, indent=2), flush=True) - - -def run_smoke_commands( - filesystem: PageIndexFileSystem, - registered: list[dict[str, Any]], - *, - verbose: bool = False, -) -> None: - executor = PIFSCommandExecutor(filesystem) - - command = "tree / -L 2" - tree = execute_json_command(executor, command) - tree_data = tree.get("data") or {} - tree_root = tree_data.get("tree") or {} - show_capability( - label="Folder structure", - command=command, - result=( - f"{tree_data.get('total_folders', 0)} folders; " - f"root files={tree_root.get('file_count', 0)}" - ), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = "ls /documents" - listing = execute_json_command(executor, command) - listing_data = listing.get("data") or {} - show_capability( - label="List alias", - command=command, - result=f"ls delegates to tree depth={listing_data.get('depth')}", - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = 'browse /documents "Federal Reserve annual report supervision regulation section page range"' - browse = execute_json_command(executor, command) - browse_hits = ((browse.get("data") or {}).get("documents") or []) - if browse_hits: - summary_result = f"{len(browse_hits)} browse candidates; top={browse_hits[0].get('document_id')}" - else: - summary_result = "browse is available, but this tiny two-doc demo returned no candidates" - show_capability( - label="Relevance browse", - command=command, - result=summary_result, - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - first_target = f"/documents/{Path(str(registered[0]['path'])).name}" if registered else None - if not first_target: - return - - command = f"stat {first_target}" - stat = execute_json_command(executor, command) - stat_data = (stat.get("data") or {}).get("document") or {} - show_capability( - label="File stat", - command=command, - result=( - f"{stat_data.get('title')} | tree={stat_data.get('status')} | " - f"metadata_status={(stat_data.get('metadata_status') or {}).get('status')}" - ), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = f"cat {first_target} --structure" - structure_payload = execute_json_command(executor, command) - structure_data = structure_payload.get("data") or {} - structure = structure_data.get("structure") or [] - supervision_node = find_structure_node(structure, "Supervision and Regulation") - supervision_range = page_range_for_node(supervision_node) - show_capability( - label="PageIndex document structure", - command=command, - result=( - "found section 'Supervision and Regulation'" - + (f" with page span {supervision_range}" if supervision_range else "") - ), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - evidence_range = opening_page_range_for_node(supervision_node) or "1-2" - command = f"cat {first_target} --page {evidence_range}" - page = execute_json_command(executor, command) - page_text = str(((page.get("data") or {}).get("content") or {}).get("text") or "") - show_capability( - label="Page evidence", - command=command, - result=compact_lines(page_text, max_lines=3, max_chars=420), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = f'grep "Supervision and Regulation" {shlex.quote(first_target)}' - grep = execute_json_command(executor, command) - grep_hits = ((grep.get("data") or {}).get("matches") or []) - show_capability( - label="Lexical grep", - command=command, - result=f"{len(grep_hits)} real text matches", - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - -def main() -> None: - args = parse_args() - require_runtime_environment( - metadata_provider=args.metadata_provider, - embedding_provider=args.embedding_provider, - ) - workspace = args.workspace.expanduser() - documents_dir = args.documents_dir.expanduser() - if args.reset and workspace.exists(): - shutil.rmtree(workspace) - workspace.mkdir(parents=True, exist_ok=True) - - documents = resolve_requested_documents(documents_dir, args.document) - if args.max_docs > 0: - documents = documents[: args.max_docs] - if not documents: - raise RuntimeError(f"no cached example documents found under {documents_dir}") - - filesystem = PageIndexFileSystem( - workspace, - metadata_generator=MetadataGenerator( - provider=args.metadata_provider, - model=args.metadata_model, - ), - summary_projection_embedding_provider=args.embedding_provider, - summary_projection_embedding_model=args.embedding_model, - summary_projection_embedding_dimensions=args.embedding_dimensions, - ) - register_demo_metadata_schema(filesystem) - - print_section("STEP 1/3 Register Documents") - print(f"Workspace: {workspace}", flush=True) - print(f"Documents: {len(documents)}", flush=True) - registered = register_documents(filesystem, documents, documents_dir=documents_dir) - configure_summary_projection_backend( - filesystem, - embedding_provider=args.embedding_provider, - embedding_model=args.embedding_model, - embedding_dimensions=args.embedding_dimensions, - ) - show_registered_documents(registered, verbose=args.verbose) - - print_section("STEP 2/3 Explore PIFS Tool Surface") - run_smoke_commands(filesystem, registered, verbose=args.verbose) - - if args.prepare_only: - return - - print_section("STEP 3/3 Ask An Agent Using Only PIFS") - print(f"Question: {args.question}", flush=True) - answer = run_pifs_agent( - filesystem, - args.question, - model=args.model, - root="/", - system_prompt=PIFS_DEMO_AGENT_PROMPT, - max_turns=args.max_turns, - max_seconds=args.max_seconds, - verbose=args.verbose, - stream_mode=args.stream_mode, - reasoning_effort=args.reasoning_effort, - reasoning_summary=args.reasoning_summary, - ) - if answer: - print("\nFinal answer:", flush=True) - print(answer, flush=True) - - if __name__ == "__main__": - main() + args = parse_args() + raise SystemExit(run(args.workspace, args.document, args.query)) diff --git a/pageindex/filesystem/__init__.py b/pageindex/filesystem/__init__.py index 24b4c9746..e92b682d2 100644 --- a/pageindex/filesystem/__init__.py +++ b/pageindex/filesystem/__init__.py @@ -1,51 +1,3 @@ -from importlib import import_module -from typing import TYPE_CHECKING - -from .commands import PIFSCommandExecutor from .core import PageIndexFileSystem -from .types import OpenResult, SearchResult - -if TYPE_CHECKING: - from .semantic_projection import SemanticProjectionSearchBackend - from .semantic_projection import SummaryProjectionIndexer - from .semantic_index import ( - RebuildableSemanticIndex, - SemanticIndexRecord, - SemanticSearchResult, - SQLiteVecSemanticIndex, - ) - -_LAZY_EXPORTS = { - "SemanticProjectionSearchBackend": (".semantic_projection", "SemanticProjectionSearchBackend"), - "RebuildableSemanticIndex": (".semantic_index", "RebuildableSemanticIndex"), - "SemanticIndexRecord": (".semantic_index", "SemanticIndexRecord"), - "SemanticSearchResult": (".semantic_index", "SemanticSearchResult"), - "SQLiteVecSemanticIndex": (".semantic_index", "SQLiteVecSemanticIndex"), - "SummaryProjectionIndexer": (".semantic_projection", "SummaryProjectionIndexer"), -} - -__all__ = [ - "OpenResult", - "SemanticProjectionSearchBackend", - "PIFSCommandExecutor", - "PageIndexFileSystem", - "RebuildableSemanticIndex", - "SearchResult", - "SemanticIndexRecord", - "SemanticSearchResult", - "SummaryProjectionIndexer", - "SQLiteVecSemanticIndex", -] - - -def __getattr__(name: str): - if name in _LAZY_EXPORTS: - module_name, attribute_name = _LAZY_EXPORTS[name] - value = getattr(import_module(module_name, __name__), attribute_name) - globals()[name] = value - return value - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - -def __dir__() -> list[str]: - return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS)) +__all__ = ["PageIndexFileSystem"] diff --git a/pageindex/filesystem/agent.py b/pageindex/filesystem/agent.py index f782ecb2b..cd67efb17 100644 --- a/pageindex/filesystem/agent.py +++ b/pageindex/filesystem/agent.py @@ -35,9 +35,9 @@ If the user asks what tools or capabilities you have, describe only the PIFS virtual shell capabilities available inside this workspace: tree, browse, -stat, cat, grep, and ls as an alias for tree -L 1. Do not mention host runtime -tools, SDK internals, or orchestration helpers that are not part of the PIFS -shell. +stat, cat, and grep. Describe PIFS Listing as tree -L 1. Do not mention +host runtime tools, SDK internals, or orchestration helpers that are not part of +the PIFS shell. If the user asks a workspace-related topic question without naming a specific file, treat it as a retrieval task. Start with tree to understand folder @@ -52,7 +52,7 @@ BASH_TOOL_DESCRIPTION = """ Run a command in the PageIndex FileSystem virtual shell. This is not a real operating-system shell. The tool is read-only and always returns JSON. -Use only: tree, browse, stat, cat, grep, and ls as an exact alias for tree -L 1. +Use only: tree, browse, stat, cat, and grep. Start broad workspace questions with tree. Then choose a physical folder, inspect metadata-derived virtual axes such as @year or @ticker with tree, and copy returned scope paths into later commands. Run browse "" for document @@ -61,16 +61,15 @@ candidates only, not final evidence. Use stat only for identity, status, or metadata. Use cat --structure before any cat --page N[-M] for that file. Use grep only as a single-document lexical fallback. -Do not use find, recursive grep, folder grep, pipes, stat schema/field modes, -browse spaces, implicit cat reads, or general knowledge when workspace evidence -is not found. +Other shell commands, pipes, stat modes, folder-wide lexical searches, and +implicit cat reads are unsupported. """ AGENT_TOOL_POLICY = """ Tool policy: - The bash tool is a PageIndex virtual shell, not an operating-system shell. - The default agent tool surface is read-only. -- Use only tree, browse, stat, cat, grep, and ls as an alias for tree -L 1. +- Use only tree, browse, stat, cat, and grep. PIFS Listing is tree -L 1. - Folder paths such as /documents are positional command targets; never put folder paths in --where. - Start with tree to understand workspace and folder structure before document retrieval. - After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect metadata-derived virtual value nodes, and copy returned scope paths instead of handwriting encoded values. @@ -78,15 +77,15 @@ - Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query. - If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder. - Only after that persistence protocol may you say the workspace lacks evidence. -- browse uses summary retrieval only; do not use browse --space. +- browse always uses summary retrieval. - Use metadata scope paths for exact filters. Keep --where as a fallback for OR, IN, or contains only when tree paths cannot express the predicate. - browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims. - For financial statement, ratio, or calculation questions, prefer citing a primary statement, reconciliation table, or official table when available; use summaries, MD&A, or footnotes as supporting evidence. -- Use grep for a selected single file only. -- Do not use find, recursive grep, folder grep, pipes, schema browsing, batch metadata commands, browse spaces, implicit cat reads, or general-knowledge fallback. +- Use grep for a selected single file only. +- Other shell commands, pipes, schema browsing, batch metadata commands, folder-wide lexical searches, implicit cat reads, and general-knowledge fallback are unsupported. - Command errors are JSON; recover by trying an available command. - Use cat or grep to gather evidence before making source-backed claims. -- Do not reconstruct a file path from a title. Use exact paths returned by PIFS commands, or use file_ref/document_id when available; quote paths that contain spaces. +- Do not reconstruct a file path from a title. Use the exact path returned by PIFS commands and quote paths that contain spaces. - For broad topic, method, or "what solution" questions that are likely about the workspace, browse for candidate documents before asking the user to choose a document. - Use stat only for identity, metadata, or status questions. Do not run stat merely to understand what a document says. - Prefer target-first cat syntax with stable targets: cat --structure, cat --page 31-59. @@ -100,7 +99,7 @@ - Avoid fetching a broad page span unless page-level citation or verification is required. - Do not call cat --page ; if you need a page span, use cat --page -. - For metadata or summary-field questions, run stat for relevant files before answering. -- Distinguish default/register metadata from caller-provided custom metadata when the evidence supports it. +- Distinguish generated metadata from caller-provided custom metadata when the evidence supports it. - Do a final consistency check before answering: the headline number, calculation, unit, sign, and yes/no polarity must agree with each other. """ diff --git a/pageindex/filesystem/cli.py b/pageindex/filesystem/cli.py index 89ac1a390..2ebbed2f5 100644 --- a/pageindex/filesystem/cli.py +++ b/pageindex/filesystem/cli.py @@ -26,6 +26,13 @@ ANSI_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|.)") PIFS_CONFIG_FILE_ENV = "PIFS_CONFIG_FILE" PIFS_WORKSPACE_ENV = "PIFS_WORKSPACE" +PERSISTED_CONFIG_KEYS = { + "workspace", + "embedding_base_url", + "embedding_model", + "embedding_dimensions", + "embedding_timeout", +} def _config_path() -> Path: @@ -45,14 +52,23 @@ def _read_config() -> dict[str, str]: payload = json.load(handle) if not isinstance(payload, dict): raise ValueError(f"invalid PIFS config file: {path}") - return {str(key): str(value) for key, value in payload.items() if value is not None} + return { + str(key): str(value) + for key, value in payload.items() + if key in PERSISTED_CONFIG_KEYS and value is not None + } def _write_config(config: dict[str, str]) -> Path: path = _config_path() path.parent.mkdir(parents=True, exist_ok=True) + persisted = { + key: value + for key, value in config.items() + if key in PERSISTED_CONFIG_KEYS and value is not None + } with path.open("w", encoding="utf-8") as handle: - json.dump(config, handle, indent=2, sort_keys=True) + json.dump(persisted, handle, indent=2, sort_keys=True) handle.write("\n") return path @@ -182,8 +198,7 @@ def _filesystem_embedding_config() -> dict[str, object]: config_values = _read_config() config: dict[str, object] = {} base_url = config_values.get("embedding_base_url") - api_key = config_values.get("embedding_api_key") - provider = config_values.get("embedding_provider") + api_key = os.environ.get("PIFS_EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY") model = config_values.get("embedding_model") dimensions = _optional_int( "embedding_dimensions", @@ -193,8 +208,6 @@ def _filesystem_embedding_config() -> dict[str, object]: "embedding_timeout", config_values.get("embedding_timeout"), ) - if provider and provider.strip(): - config["summary_projection_embedding_provider"] = provider.strip() if model and model.strip(): config["summary_projection_embedding_model"] = model.strip() if dimensions is not None: @@ -344,7 +357,6 @@ def _run_add(argv: list[str], *, workspace: str) -> int: filesystem = _filesystem_from_workspace(workspace) info = filesystem.add_file(args.physical_path, args.virtual_target) print(f"added: {info.get('path')}") - print(f"file_ref: {info['file_ref']}") return 0 @@ -371,7 +383,15 @@ def _run_setmeta(argv: list[str], *, workspace: str) -> int: filesystem = _filesystem_from_workspace(workspace) info = filesystem.set_metadata(args.target, metadata, clear=args.clear) - print(json.dumps(info, ensure_ascii=False, sort_keys=True)) + document = { + "path": args.target if args.target.startswith("/") else info.get("path"), + "document_id": info.get("external_id"), + "title": info.get("title"), + "status": info.get("pageindex_tree_status"), + "metadata": info.get("metadata", {}), + "metadata_status": info.get("metadata_status", {}), + } + print(json.dumps(document, ensure_ascii=False, sort_keys=True)) return 0 diff --git a/pageindex/filesystem/commands.py b/pageindex/filesystem/commands.py index 5b31d5c18..828048429 100644 --- a/pageindex/filesystem/commands.py +++ b/pageindex/filesystem/commands.py @@ -13,7 +13,7 @@ class PIFSCommandError(ValueError): class PIFSCommandExecutor: - COMMAND_NAMES = {"ls", "tree", "browse", "stat", "cat", "grep"} + COMMAND_NAMES = {"tree", "browse", "stat", "cat", "grep"} FORBIDDEN_SUBSTRINGS = (";", "`", "$(", "||", "&&", "\n", "\r") FORBIDDEN_TOKENS = {"|", ">", "<", ">>", "<<", "&"} BROWSE_PAGE_SIZE = 10 @@ -33,9 +33,8 @@ def describe_available_command_surfaces(self) -> str: [ "Available PIFS BashLike commands:", "- tree [-L depth] [--page N]: folder and metadata-scope orientation", - "- ls : exact alias for tree -L 1", '- browse "" [--page N] [--where JSON] [-R]: summary-ranked document discovery', - "- stat : single document identity or scope metadata", + "- stat : single document identity, status, and metadata", "- cat --structure | --page N[-M]: structure-first document reads", "- grep : single-document lexical evidence fallback", ] @@ -68,14 +67,6 @@ def _execute(self, command: str) -> tuple[dict[str, Any], list[str]]: raise PIFSCommandError(f"Unsupported command: {name}") return getattr(self, f"_cmd_{name}")(args) - def _cmd_ls(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: - if len(args) > 1: - raise PIFSCommandError("ls accepts exactly one optional folder target") - if args and args[0].startswith("-"): - raise PIFSCommandError("ls is only an alias for tree -L 1") - path = args[0] if args else "/" - return self._cmd_tree([path, "-L", "1"]) - def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: path = "/" depth = 10 @@ -105,8 +96,11 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: root["folders"] = [ { "path": self._join_scope_path( - scope.path, - self.filesystem.encode_scope_segment(row["value"]), + scope.path.rsplit("/", 1)[0] or "/", + ( + f"@{self.filesystem.encode_scope_segment(scope.metadata_axis)}=" + f"{self.filesystem.encode_scope_segment(row['value'])}" + ), ), "name": str(row["value"]), "type": "metadata_value", @@ -129,7 +123,9 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: "next_page": page + 1 if has_more else None, }, } - next_steps = [f'browse {shlex.quote(scope.path)}/ ""'] + next_steps = [ + f'browse {shlex.quote(scope.path)}= ""' + ] return data, next_steps page_size = self.TREE_VALUE_PAGE_SIZE needed = page * page_size + 1 @@ -138,7 +134,10 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: max_depth=depth, limit=max(self.MAX_TREE_FOLDERS, needed), ) - files = self.filesystem.scope_files(scope, limit=needed) + files = [ + self._tree_file(row) + for row in self.filesystem.scope_files(scope, limit=needed) + ] axes = self.filesystem.scope_metadata_axes(scope) tree, has_more = self._folder_tree( scope, @@ -198,22 +197,15 @@ def _cmd_browse(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: scope = self.filesystem.resolve_query_scope(path) if scope.metadata_axis is not None: raise PIFSCommandError( - "Metadata axis paths require @field/value; run tree /@field to inspect values." + "Metadata axis paths require @field=value; run tree /@field to inspect values." ) merged_filter = self.filesystem.merge_scope_filter(scope, where) effective_recursive = recursive or bool(scope.metadata_filter) - if not self.filesystem.has_semantic_channel("summary"): - self.filesystem.configure_existing_projection_retrieval() - if not self.filesystem.has_semantic_channel("summary"): - raise PIFSCommandError("browse summary retrieval is not available") payload = self.filesystem.browse_semantic_files( scope.path, query, - retrieval_query=query, recursive=effective_recursive, - space="summary", page=page, - page_size=self.BROWSE_PAGE_SIZE, metadata_filter=where, ) documents = [self._document_hit(row) for row in payload.get("data", [])] @@ -365,41 +357,65 @@ def _folder_tree( def _document_hit(self, row: dict[str, Any]) -> dict[str, Any]: return { "path": row.get("path"), + "document_id": row.get("external_id"), "title": row.get("title"), + "status": row.get("pageindex_tree_status"), + "rank": row.get("rank"), + "similarity": row.get("similarity"), "summary": row.get("summary", ""), "metadata": row.get("metadata", {}), + "folder_path": row.get("folder_path"), + "folder_paths": row.get("folder_paths", []), + } + + @staticmethod + def _tree_file(row: dict[str, Any]) -> dict[str, Any]: + return { + "path": row["path"], + "document_id": row.get("external_id"), + "title": row["title"], + "status": row["pageindex_tree_status"], + "type": "file", + "metadata": row.get("metadata", {}), } def _document_stat(self, target: str) -> dict[str, Any]: - info = dict(self.filesystem._stat(target)) + file_ref = self.filesystem._resolve_target(target) + entry = self.filesystem.store.get_file(file_ref) + folder_paths = [ + folder["path"] + for folder in self.filesystem.store.folder_memberships(file_ref) + ] + folder_path = self.filesystem._preferred_folder_path( + folder_paths, + entry.folder_path, + entry.folder_path, + ) + path = ( + target + if str(target).startswith("/") + else self.filesystem._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + ) return { - "path": target if str(target).startswith("/") else info.get("path"), - "file_ref": info.get("file_ref"), - "document_id": info.get("external_id") or info.get("document_id"), - "status": info.get("pageindex_tree_status") or info.get("status"), - "page_count": info.get("pageNum"), - "folders": info.get("folders", []), - "metadata": info.get("metadata", {}), - "metadata_status": info.get("metadata_status", {}), - "pageindex_doc_id": info.get("pageindex_doc_id"), - "content_type": info.get("content_type"), - "title": info.get("title") or info.get("name"), + "path": path, + "document_id": entry.external_id, + "title": entry.title, + "status": entry.pageindex_tree_status, + "content_type": entry.content_type, + "metadata": entry.metadata, + "metadata_status": entry.metadata_status, + "folder_paths": folder_paths, } def _document_from_structural_payload(self, payload: dict[str, Any], target: str) -> dict[str, Any]: - document = { - "file_ref": payload.get("file_ref"), - "document_id": payload.get("external_id"), - "status": payload.get("status"), - "pageindex_doc_id": payload.get("pageindex_doc_id"), - "available": bool(payload.get("available", True)), - } + document = self._document_stat(target) + document["available"] = bool(payload.get("available", True)) if payload.get("message"): document["message"] = payload.get("message") - try: - document.update({k: v for k, v in self._document_stat(target).items() if v is not None}) - except (KeyError, ValueError): - pass return document def _page_next_steps(self, target: str, payload: dict[str, Any]) -> list[str]: @@ -530,11 +546,10 @@ def _scoped_folder_path(self, scope: Any, folder_path: str) -> str: for field, value in getattr(scope, "metadata_filter", {}).items(): path = self._join_scope_path( path, - f"@{self.filesystem.encode_scope_segment(field)}", - ) - path = self._join_scope_path( - path, - self.filesystem.encode_scope_segment(value), + ( + f"@{self.filesystem.encode_scope_segment(field)}=" + f"{self.filesystem.encode_scope_segment(value)}" + ), ) return path diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 9f5767ed5..c4ea76ebe 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -13,10 +13,9 @@ SQLiteFileSystemStore, fingerprint, make_file_ref, - metadata_text, normalize_path, ) -from .types import OpenResult, PIFSQueryScope, SearchResult +from .types import PIFSQueryScope if TYPE_CHECKING: from ..client import PageIndexClient @@ -30,10 +29,6 @@ } DEFAULT_EMBEDDING_DIMENSIONS = 1024 -SEMANTIC_RETRIEVAL_CHANNELS = ("summary",) -SEMANTIC_PROJECTION_INDEX_NAMES = { - "summary": "summary", -} PAGEINDEX_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown"} PAGEINDEX_DOCUMENT_CONTENT_TYPES = { "application/pdf", @@ -66,7 +61,6 @@ def __init__( workspace: Union[str, Path], *, summary_projection_index_dir: Union[str, Path, None] = None, - summary_projection_embedding_provider: str = "openai", summary_projection_embedding_model: str = "text-embedding-3-small", summary_projection_embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, summary_projection_embedding_timeout: float = 60, @@ -76,14 +70,12 @@ def __init__( self.workspace = Path(workspace).expanduser() self.store = SQLiteFileSystemStore(self.workspace) self.metadata = MetadataQueryEngine(self.store) - self.semantic_retrieval_backend: Any | None = None - self.summary_projection_indexer: Any | None = None + self.summary_projection: Any | None = None self.summary_projection_index_dir = ( Path(summary_projection_index_dir).expanduser() if summary_projection_index_dir is not None else self.workspace / "artifacts" / "projection_indexes" ) - self.summary_projection_embedding_provider = summary_projection_embedding_provider self.summary_projection_embedding_model = summary_projection_embedding_model self.summary_projection_embedding_dimensions = summary_projection_embedding_dimensions self.summary_projection_embedding_timeout = summary_projection_embedding_timeout @@ -117,10 +109,6 @@ def register_file( ] )[0] - def register(self, **kwargs: Any) -> str: - self._ensure_register_completion_defaults() - return self.register_file(**kwargs) - def add_file( self, physical_path: Union[str, Path], @@ -236,102 +224,35 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: raise return [record["file_ref"] for record in records] - def _ensure_register_completion_defaults(self) -> None: - if self.summary_projection_indexer is None: - from .semantic_projection import SummaryProjectionIndexer + def _ensure_add_completion_defaults(self) -> None: + self._open_summary_projection(create=True) - self.summary_projection_indexer = SummaryProjectionIndexer.from_provider( - self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - if self.semantic_retrieval_backend is None: - self.configure_semantic_projection_retrieval( - self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) + def _ensure_add_semantic_retrieval_ready(self) -> None: + projection = self._open_summary_projection(create=False) + if not projection.available: + raise RuntimeError("pifs add failed to make the Summary Projection available") + + def _summary_embedding_profile(self) -> Any: + from .semantic_projection import SummaryEmbeddingProfile + + return SummaryEmbeddingProfile( + base_url=self.summary_projection_embedding_base_url, + model=self.summary_projection_embedding_model, + dimensions=self.summary_projection_embedding_dimensions, + timeout=self.summary_projection_embedding_timeout, + api_key=self.summary_projection_embedding_api_key, + ) - def _ensure_add_completion_defaults(self) -> None: - if self.summary_projection_indexer is None: - from .semantic_projection import SummaryProjectionIndexer + def _open_summary_projection(self, *, create: bool) -> Any: + if self.summary_projection is None: + from .semantic_projection import SummaryProjection - self.summary_projection_indexer = SummaryProjectionIndexer.from_provider( + self.summary_projection = SummaryProjection( self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, + profile=self._summary_embedding_profile(), + create=create, ) - - def _ensure_add_semantic_retrieval_ready(self) -> None: - indexer = self.summary_projection_indexer - if indexer is None: - raise RuntimeError("pifs add requires a summary projection indexer") - from .semantic_projection import SemanticProjectionSearchBackend - - index_dir = Path(getattr(indexer, "index_dir", self.summary_projection_index_dir)) - embedder = getattr(indexer, "embedder", None) - if embedder is None: - self.configure_semantic_projection_retrieval( - index_dir, - embedding_provider=str( - getattr( - indexer, - "embedding_provider", - self.summary_projection_embedding_provider, - ) - ), - embedding_model=str( - getattr(indexer, "embedding_model", self.summary_projection_embedding_model) - ), - embedding_dimensions=int( - getattr( - indexer, - "embedding_dimensions", - self.summary_projection_embedding_dimensions, - ) - ), - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - else: - embedding_cache = getattr(indexer, "embedding_cache", None) - self.semantic_retrieval_backend = SemanticProjectionSearchBackend( - index_dir, - embedder=embedder, - embedding_provider=str( - getattr( - indexer, - "embedding_provider", - self.summary_projection_embedding_provider, - ) - ), - embedding_model=str( - getattr(indexer, "embedding_model", self.summary_projection_embedding_model) - ), - embedding_dimensions=int( - getattr( - indexer, - "embedding_dimensions", - self.summary_projection_embedding_dimensions, - ) - ), - embedding_cache_path=getattr(embedding_cache, "db_path", None), - ) - if "summary" not in self.semantic_retrieval_channels(): - raise RuntimeError("pifs add failed to configure summary semantic retrieval") + return self.summary_projection def _add_created_folder_paths(self, folder_path: str) -> list[str]: paths = self._folder_ancestor_paths(folder_path) @@ -348,75 +269,6 @@ def _folder_ancestor_paths(folder_path: str) -> list[str]: paths.append("/" + "/".join(segments[:index])) return paths - def configure_existing_projection_retrieval(self) -> bool: - """Attach semantic retrieval to already-built projection indexes. - - Register-time generation owns building the index files. Opening an - existing workspace should still expose semantic retrieval when the - configured embedding dimensions match the existing index. - """ - if self.semantic_retrieval_backend is not None: - return bool(self.semantic_retrieval_channels()) - index_config = self._existing_projection_index_config() - if index_config is None: - return False - existing_dimension = int(index_config.get("dimension") or 0) - if existing_dimension != self.summary_projection_embedding_dimensions: - raise RuntimeError( - "summary projection index dimension mismatch: " - f"{index_config.get('db_path') or self.summary_projection_index_dir} " - f"was built with dimension {existing_dimension}, but configured " - "summary_projection_embedding_dimensions is " - f"{self.summary_projection_embedding_dimensions}. Rebuild the " - "projection index or use a matching embedding configuration." - ) - self.configure_semantic_projection_retrieval( - self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - return bool(self.semantic_retrieval_channels()) - - def _existing_projection_index_config(self) -> dict[str, Any] | None: - for channel in SEMANTIC_RETRIEVAL_CHANNELS: - index_name = SEMANTIC_PROJECTION_INDEX_NAMES.get(channel) - if not index_name: - continue - index_path = self.summary_projection_index_dir / f"{index_name}.sqlite" - if not index_path.exists(): - continue - from .semantic_index import SQLiteVecSemanticIndex - - try: - info = SQLiteVecSemanticIndex(index_path).info() - except Exception: - continue - if int(info.get("document_count") or 0) <= 0: - continue - metadata = dict(info.get("metadata") or {}) - if metadata.get("channel") and metadata.get("channel") != channel: - continue - return info - return None - - def browse( - self, - path: str = "/", - recursive: bool = False, - limit: int = 100, - max_depth: int | None = None, - ) -> dict[str, list[dict[str, Any]]]: - return self.store.list_folder( - path, - recursive=recursive, - limit=limit, - max_depth=max_depth, - ) - def resolve_query_scope(self, path: str) -> PIFSQueryScope: normalized = normalize_path(path) if self._folder_exists(normalized): @@ -444,10 +296,12 @@ def resolve_query_scope(self, path: str) -> PIFSQueryScope: raise KeyError(f"Unknown folder path: {normalized}") raise ValueError( "Metadata axes must come after the physical folder prefix; " - "inspect the physical folder first, then append @field/value buckets. " + "inspect the physical folder first, then append @field=value buckets. " "Use the path returned by tree for values containing '/'." ) - field = unquote(segment[1:]) + axis_segment = segment[1:] + encoded_field, separator, encoded_value = axis_segment.partition("=") + field = unquote(encoded_field) self.metadata.validate_field_name(field) if not self.store.metadata_field_exists(field): raise ValueError("Unknown metadata axis; run tree to inspect available @field axes.") @@ -456,20 +310,24 @@ def resolve_query_scope(self, path: str) -> PIFSQueryScope: "A metadata field can appear only once in a scope path; " "choose one value or use browse --where for advanced predicates." ) - if index + 1 >= len(remainder): + if not separator: + if index + 1 != len(remainder): + raise ValueError( + "Metadata axis inspection must be the final path segment; " + "choose a value with @field=value before appending another axis." + ) return PIFSQueryScope( path=normalized, folder_path=folder_path, metadata_filter=metadata_filter, metadata_axis=field, ) - value_segment = remainder[index + 1] - if value_segment.startswith("@"): + if not encoded_value: raise ValueError( - "Metadata axis paths require @field/value; run tree /@field to inspect values." + "Metadata value cannot be empty; run tree /@field to inspect values." ) - metadata_filter[field] = unquote(value_segment) - index += 2 + metadata_filter[field] = unquote(encoded_value) + index += 1 return PIFSQueryScope( path=normalized, @@ -485,7 +343,7 @@ def merge_scope_filter( parsed = self.metadata.parse_filter(metadata_filter) if scope.metadata_axis is not None: raise ValueError( - "Metadata axis paths require @field/value; run tree /@field to inspect values." + "Metadata axis paths require @field=value; run tree /@field to inspect values." ) if not parsed: return dict(scope.metadata_filter) or None @@ -555,15 +413,23 @@ def scope_files(self, scope: PIFSQueryScope, *, limit: int) -> list[dict[str, An files.append( { "path": self._scope_file_locator(scope, locator_leaf), - "name": locator_leaf, + "locator_leaf": locator_leaf, "type": "file", "file_ref": row["file_ref"], - "document_id": row["document_id"], + "external_id": row["external_id"], "title": leaf, + "pageindex_tree_status": row["pageindex_tree_status"], "metadata": row["metadata"], } ) - files = sorted(files, key=lambda item: (str(item["name"]).lower(), item["path"], item["file_ref"])) + files = sorted( + files, + key=lambda item: ( + str(item["title"]).lower(), + item["path"], + item["file_ref"], + ), + ) return files[:limit] def scope_file_locator(self, scope: PIFSQueryScope, file_ref: str, leaf: str) -> str: @@ -575,8 +441,7 @@ def scope_file_locator(self, scope: PIFSQueryScope, file_ref: str, leaf: str) -> def _scope_file_leaf_items(self, scope: PIFSQueryScope, *, limit: int) -> list[tuple[dict[str, Any], str]]: recursive = bool(scope.metadata_filter) - rows = self.store.search_files( - None, + rows = self.store.list_files( scope={"folder_path": scope.folder_path, "recursive": recursive}, metadata_filter=scope.metadata_filter or None, limit=limit, @@ -603,24 +468,23 @@ def _scope_leaf_counts(leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str counts[leaf] = counts.get(leaf, 0) + 1 return counts - @staticmethod - def _disambiguated_scope_leaf(leaf: str, file_ref: str, leaf_counts: dict[str, int]) -> str: - if leaf_counts.get(leaf, 0) <= 1: - return leaf - return f"{leaf}~{file_ref}" - @classmethod def _scope_locator_leaf_by_file_ref(cls, leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str, str]: leaf_counts = cls._scope_leaf_counts(leaf_items) + reserved = {leaf for leaf, count in leaf_counts.items() if count == 1} used: set[str] = set() + next_suffix: dict[str, int] = {} locator_leaf_by_file_ref: dict[str, str] = {} for row, leaf in sorted(leaf_items, key=lambda item: (item[1].lower(), item[1], item[0]["file_ref"])): - base = cls._disambiguated_scope_leaf(leaf, row["file_ref"], leaf_counts) - locator_leaf = base - suffix = 2 - while locator_leaf in used: - locator_leaf = f"{base}~{suffix}" - suffix += 1 + if leaf_counts[leaf] == 1: + locator_leaf = leaf + else: + suffix = next_suffix.get(leaf, 1) + locator_leaf = f"{leaf}~{suffix}" + while locator_leaf in used or locator_leaf in reserved: + suffix += 1 + locator_leaf = f"{leaf}~{suffix}" + next_suffix[leaf] = suffix + 1 used.add(locator_leaf) locator_leaf_by_file_ref[row["file_ref"]] = locator_leaf return locator_leaf_by_file_ref @@ -647,40 +511,26 @@ def browse_semantic_files( path: str, query: str, *, - retrieval_query: str | None = None, recursive: bool = False, - space: str = "summary", page: int = 1, - page_size: int = 10, metadata_filter: Optional[dict[str, Any] | str] = None, ) -> dict[str, Any]: + page_size = 10 path = normalize_path(path) query_scope = self.resolve_query_scope(path) self.store.folder_info(query_scope.folder_path) - query_text = self._query_text(retrieval_query or query).strip() + query_text = self._query_text(query).strip() if not query_text: raise ValueError("browse requires a query") if page < 1: raise ValueError("browse --page must be at least 1") - if page_size < 1: - raise ValueError("browse page_size must be at least 1") - if space not in SEMANTIC_RETRIEVAL_CHANNELS: - raise ValueError( - "Unsupported browse --space: " - f"{space}. Supported spaces: {', '.join(SEMANTIC_RETRIEVAL_CHANNELS)}" - ) - available_spaces = self.semantic_retrieval_channels() - if space not in available_spaces: - available = ", ".join(available_spaces) if available_spaces else "none" - raise ValueError( - f"browse --space {space} is not available; available spaces: {available}" - ) - search_channel = getattr(self.semantic_retrieval_backend, "search_channel", None) - if search_channel is None: - available = ", ".join(available_spaces) if available_spaces else "none" - raise ValueError( - f"browse --space {space} is not available; available spaces: {available}" - ) + if self.summary_projection is None: + index_path = self.summary_projection_index_dir / "summary.sqlite" + if not index_path.exists(): + raise ValueError("browse Summary Projection is not available") + projection = self._open_summary_projection(create=False) + if not projection.available: + raise ValueError("browse Summary Projection is not available") parsed_filter = self.merge_scope_filter(query_scope, metadata_filter) effective_recursive = recursive or bool(query_scope.metadata_filter) scope = {"folder_path": query_scope.folder_path, "recursive": effective_recursive} @@ -690,13 +540,11 @@ def browse_semantic_files( ) offset = (page - 1) * page_size needed = offset + page_size + 1 - semantic_filters = {"file_ref": scope_file_refs} candidates = ( - search_channel( - space, + projection.search( query_text, limit=needed, - filters=semantic_filters, + file_refs=scope_file_refs, ) if scope_file_refs else [] @@ -705,10 +553,7 @@ def browse_semantic_files( rows: list[dict[str, Any]] = [] seen: set[str] = set() for candidate in candidates: - try: - file_ref = self.store.resolve_file_ref(candidate.document_id) - except KeyError: - continue + file_ref = candidate.file_ref if file_ref in seen: continue if file_ref not in scope_file_ref_set: @@ -746,21 +591,16 @@ def browse_semantic_files( rows.append( { "rank": rank, - "similarity": self._semantic_candidate_similarity(candidate), - "score": self._semantic_candidate_score(candidate), + "similarity": candidate.similarity, "path": stable_path, "file_ref": file_ref, - "document_id": entry.external_id, "external_id": entry.external_id, "title": display_title, - "original_title": entry.title, + "pageindex_tree_status": entry.pageindex_tree_status, "folder_path": folder_path, "folder_paths": folder_paths, "summary": str((entry.metadata or {}).get("summary") or ""), - "snippet": str(getattr(candidate, "snippet", "") or entry.descriptor), "metadata": entry.metadata, - "metadata_status": entry.metadata_status, - "sources": list(getattr(candidate, "sources", []) or []), } ) if len(rows) >= needed: @@ -768,12 +608,10 @@ def browse_semantic_files( page_rows = rows[offset : offset + page_size] payload = { "mode": "files", - "retrieval": f"{space}_vector", + "retrieval": "summary", "query": query, "scope": query_scope.path, "recursive": effective_recursive, - "space": space, - "available_spaces": list(available_spaces), "page": page, "page_size": page_size, "has_more": len(rows) > offset + page_size, @@ -786,23 +624,6 @@ def browse_semantic_files( def folder_info(self, path: str = "/") -> dict[str, Any]: return self.store.folder_info(path) - def find_folders( - self, - path: str = "/", - metadata_filter: Optional[dict[str, Any] | str] = None, - limit: int = 100, - max_depth: int | None = None, - include_self: bool = False, - ) -> list[dict[str, Any]]: - parsed_filter = self.metadata.parse_filter(metadata_filter) - return self.store.find_folders( - path, - metadata_filter=parsed_filter, - limit=limit, - max_depth=max_depth, - include_self=include_self, - ) - def create_folder( self, path: str, @@ -856,134 +677,6 @@ def set_metadata( ) return self.store.file_info(file_ref) - def search( - self, - query: Union[str, list[str], None] = None, - scope: Optional[dict[str, Any]] = None, - metadata_filter: Optional[dict[str, Any] | str] = None, - limit: int = 10, - ) -> list[SearchResult]: - parsed_filter = self.metadata.parse_filter(metadata_filter) - rows = self.store.search_files( - query, - scope=scope, - metadata_filter=parsed_filter, - limit=limit, - ) - results = [] - scope_path = self._scope_folder_path(scope) - for row in rows: - folder_paths = [ - folder["path"] - for folder in self.store.folder_memberships(row["file_ref"]) - ] - folder_path = self._preferred_folder_path(folder_paths, scope_path, row["folder_path"]) - display_title = self.store.membership_display_name(row["file_ref"], folder_path) or row["title"] - results.append( - SearchResult( - file_ref=row["file_ref"], - external_id=row["external_id"], - title=display_title, - snippet=row["snippet"], - folder_path=folder_path, - folder_paths=folder_paths, - metadata=row["metadata"], - metadata_status=row["metadata_status"], - id=row["id"], - document_id=row["document_id"], - name=display_title, - description=row["description"], - status=row["status"], - pageNum=row["pageNum"], - createdAt=row["createdAt"], - folderId=row["folderId"], - ) - ) - return results - - def configure_semantic_projection_retrieval( - self, - index_dir: Union[str, Path], - *, - embedding_provider: str = "openai", - embedding_model: str = "text-embedding-3-small", - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_timeout: float = 60, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - fetch_multiplier: int = 100, - ) -> Any: - from .semantic_projection import SemanticProjectionSearchBackend - - self.semantic_retrieval_backend = SemanticProjectionSearchBackend.from_provider( - index_dir, - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - embedding_timeout=embedding_timeout, - embedding_api_key=embedding_api_key, - embedding_base_url=embedding_base_url, - fetch_multiplier=fetch_multiplier, - ) - return self.semantic_retrieval_backend - - @property - def has_semantic_retrieval_backend(self) -> bool: - return self.semantic_retrieval_backend is not None - - def semantic_retrieval_channels(self) -> tuple[str, ...]: - backend = self.semantic_retrieval_backend - if backend is None: - return () - available_channels = getattr(backend, "available_channels", None) - if callable(available_channels): - raw_channels = available_channels() - else: - raw_channels = getattr(backend, "semantic_tool_channels", ()) - available = set(raw_channels or ()) - return tuple(channel for channel in SEMANTIC_RETRIEVAL_CHANNELS if channel in available) - - def has_semantic_channel(self, channel: str) -> bool: - return channel in self.semantic_retrieval_channels() - - def retrieval_capabilities(self) -> dict[str, Any]: - semantic_channels = ["summary"] if self.has_semantic_channel("summary") else [] - semantic_commands = ["browse"] if semantic_channels else [] - return { - "lexical": { - "grep_recursive": False, - "grep_recursive_semantic_prefilter": False, - "find_maxdepth": False, - }, - "semantic": { - "backend_configured": self.semantic_retrieval_backend is not None, - "channels": semantic_channels, - "commands": semantic_commands, - }, - } - - def open(self, target: str, location: str = "all") -> OpenResult: - file_ref = self._resolve_target(target) - entry = self.store.get_file(file_ref) - if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: - raise ValueError( - "open() text artifact reads are not supported for PDF/Markdown PageIndex files; " - "use pageindex_structure() or pageindex_pages()." - ) - if str(location).strip().lower() in {"all", "full", "*"}: - return self._open_all(file_ref) - start, end = self._parse_line_range(location) - return self._open_lines(file_ref, start, end) - - def cat_text_artifact(self, target: str, location: str = "all") -> OpenResult: - file_ref = self._resolve_target(target) - entry = self.store.get_file(file_ref) - self._require_text_artifact_file(entry, "cat --all") - if str(location).strip().lower() in {"all", "full", "*"}: - return self._open_all(file_ref) - start, end = self._parse_line_range(location) - return self._open_lines(file_ref, start, end) - def pageindex_structure( self, target: str, @@ -1061,20 +754,6 @@ def pageindex_pages(self, target: str, pages: str) -> dict[str, Any]: "text": text, } - def _stat(self, target: str) -> dict[str, Any]: - file_ref = self._resolve_target(target) - return self.store.file_info(file_ref) - - def _require_text_artifact_file(self, entry: Any, command: str) -> None: - if self._file_format(entry) == "text": - return - raise ValueError( - f"{command} is only supported for txt/text files; " - f"got title={entry.title!r}, content_type={entry.content_type!r}. " - "Use cat --structure, " - "or cat --page for PDF/Markdown PageIndex files." - ) - def _require_pageindex_document_file(self, entry: Any, command: str) -> None: if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: return @@ -1117,7 +796,12 @@ def pageindex_client_workspace(self) -> Path: def _pageindex_client(self) -> PageIndexClient: from ..client import PageIndexClient - return PageIndexClient(workspace=str(self.pageindex_client_workspace)) + workspace = self.pageindex_client_workspace + workspace.mkdir(parents=True, exist_ok=True) + metadata_index = workspace / "_meta.json" + if not metadata_index.exists(): + metadata_index.write_text("{}\n", encoding="utf-8") + return PageIndexClient(workspace=str(workspace)) def _pageindex_client_doc_for_entry(self, entry: Any) -> tuple[PageIndexClient, str | None]: client = self._pageindex_client() @@ -1224,9 +908,6 @@ def _client_json(payload: str) -> Any: except json.JSONDecodeError: return {"error": f"Invalid PageIndexClient JSON response: {payload}"} - def _create_folder(self, path: str) -> str: - return self.create_folder(path) - @classmethod def _resolve_add_target( cls, @@ -1350,12 +1031,10 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: pageindex_tree_status=pageindex_tree_status, fallback_content=content, ) - fts_content = file.get("fts_content", artifact_content) source_type = file.get("source_type") metadata_status = self._metadata_status_state(metadata=metadata) self._attach_pageindex_tree_failure(metadata_status, pageindex_tree_failure) indexed_metadata = SQLiteFileSystemStore.indexed_metadata_values(metadata) - searchable_metadata = indexed_metadata text_artifact_path = file.get("text_artifact_path") owns_text_artifact = text_artifact_path is None if text_artifact_path is None: @@ -1365,13 +1044,12 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: if raw_artifact_path is None and file.get("write_raw_artifact", True): raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" owns_raw_artifact = True - descriptor = self._build_descriptor(title, metadata) return { "file_ref": file_ref, "external_id": external_id, "storage_uri": storage_uri, "title": title, - "descriptor": descriptor, + "descriptor": title, "content_type": content_type, "source_type": source_type, "fingerprint": fingerprint(artifact_content), @@ -1384,10 +1062,7 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: "metadata_status": metadata_status, "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), "indexed_metadata": indexed_metadata, - "metadata_text": metadata_text(searchable_metadata), "folder_path": folder_path, - "content": fts_content, - "skip_fts": bool(file.get("skip_fts", False)), "_pifs_owned_text_artifact": owns_text_artifact, "_pifs_owned_raw_artifact": owns_raw_artifact, } @@ -1405,7 +1080,7 @@ def _registration_text_artifact_content( return fallback_content if pageindex_tree_status != "built" or not pageindex_doc_id: return fallback_content - return self._pageindex_extracted_text(pageindex_doc_id) + return self._pageindex_extracted_text(pageindex_doc_id) or fallback_content def _pageindex_extracted_text(self, doc_id: str) -> str: client = self._pageindex_client() @@ -1470,7 +1145,6 @@ def _sync_owned_raw_artifact(self, record: dict[str, Any]) -> None: ) def _record_from_file_entry(self, entry: Any) -> dict[str, Any]: - content = self.store.read_text(entry.file_ref) metadata_status = self._metadata_status_state(metadata=entry.metadata) self._attach_pageindex_tree_failure( metadata_status, @@ -1494,12 +1168,7 @@ def _record_from_file_entry(self, entry: Any) -> dict[str, Any]: "metadata_status": metadata_status, "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), "indexed_metadata": SQLiteFileSystemStore.indexed_metadata_values(entry.metadata), - "metadata_text": metadata_text( - SQLiteFileSystemStore.indexed_metadata_values(entry.metadata) - ), "folder_path": entry.folder_path, - "content": content, - "skip_fts": False, } def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: @@ -1510,11 +1179,11 @@ def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: summary = str(record.get("metadata", {}).get("summary") or "").strip() if not summary: return False - if self.summary_projection_indexer is None: + if self.summary_projection is None: self._refresh_record_metadata_status(record) return True try: - result = self.summary_projection_indexer.upsert_summary(record) + result = self.summary_projection.upsert_summary(record) except Exception as exc: summary_index["status"] = "failed" summary_index["error"] = str(exc) @@ -1561,22 +1230,15 @@ def _existing_file_refs(self, records: list[dict[str, Any]]) -> set[str]: return existing def _cleanup_add_summary_projection(self, records: list[dict[str, Any]]) -> None: - indexer = self.summary_projection_indexer - if indexer is None: + projection = self.summary_projection + if projection is None: return - delete_summary = getattr(indexer, "delete_summary", None) for record in records: file_ref = str(record.get("file_ref") or "") if not file_ref: continue try: - if callable(delete_summary): - delete_summary(file_ref) - continue - index = getattr(indexer, "index", None) - delete_file_refs = getattr(index, "delete_file_refs", None) - if callable(delete_file_refs): - delete_file_refs([file_ref]) + projection.delete_summary(file_ref) except Exception: continue @@ -1658,35 +1320,6 @@ def _refresh_record_metadata_status( record["metadata_json"] = json.dumps(record["metadata"], ensure_ascii=False) record["metadata_status_json"] = json.dumps(metadata_status, ensure_ascii=False) record["indexed_metadata"] = SQLiteFileSystemStore.indexed_metadata_values(record["metadata"]) - record["metadata_text"] = metadata_text(record["indexed_metadata"]) - - def _open_lines(self, file_ref: str, start: int, end: int) -> OpenResult: - entry = self.store.get_file(file_ref) - lines = self.store.read_text(file_ref).splitlines() - start = max(1, start) - end = min(max(start, end), len(lines)) - text = "\n".join(lines[start - 1:end]) - return OpenResult( - file_ref=file_ref, - start_line=start, - end_line=end, - text=text, - external_id=entry.external_id, - folder_path=entry.folder_path, - ) - - def _open_all(self, file_ref: str) -> OpenResult: - entry = self.store.get_file(file_ref) - text = self.store.read_text(file_ref) - line_count = len(text.splitlines()) - return OpenResult( - file_ref=file_ref, - start_line=1, - end_line=line_count, - text=text, - external_id=entry.external_id, - folder_path=entry.folder_path, - ) @classmethod def _structural_unavailable( @@ -1769,7 +1402,7 @@ def _resolve_scope_file_locator(self, target: str) -> str: matches = [ item for item in self.scope_files(scope, limit=self.scope_file_count(scope) + 1) - if item["name"] == leaf + if item["locator_leaf"] == leaf ] if not matches: raise KeyError(f"Unknown file target: {target}") @@ -1787,31 +1420,6 @@ def _scope_file_required_message(path: str) -> str: f'browse {path} "" to select a file leaf.' ) - @staticmethod - def _semantic_candidate_score(candidate: Any) -> float | None: - try: - return float(getattr(candidate, "score")) - except (AttributeError, TypeError, ValueError): - return None - - @classmethod - def _semantic_candidate_similarity(cls, candidate: Any) -> float: - distances: list[float] = [] - for source in getattr(candidate, "sources", []) or []: - if not isinstance(source, dict) or source.get("distance") is None: - continue - try: - distances.append(float(source["distance"])) - except (TypeError, ValueError): - continue - if distances: - distance = max(min(distances), 0.0) - return round(max(0.0, min(1.0, 1.0 / (1.0 + distance))), 4) - score = cls._semantic_candidate_score(candidate) - if score is None: - return 0.0 - return round(max(0.0, min(1.0, score)), 4) - @staticmethod def _metadata_filter_payload(metadata_filter: Any) -> str: if isinstance(metadata_filter, str): @@ -1859,11 +1467,6 @@ def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: PageIndexFileSystem.encode_scope_segment(str(leaf).strip("/")), ) - @staticmethod - def _build_descriptor(title: str, metadata: dict[str, Any]) -> str: - source = metadata.get("source_type") or metadata.get("repo") or metadata.get("channel") - return f"{title} ({source})" if source else title - @staticmethod def _validate_register_metadata(metadata: dict[str, Any]) -> None: if "summary" in metadata: @@ -1907,13 +1510,6 @@ def _refresh_summary_projection_status( if summary_index.get("status", "not_indexed") == "not_indexed": summary_index["status"] = "pending_index" - @staticmethod - def _scope_folder_path(scope: Optional[dict[str, Any]]) -> Optional[str]: - if not scope: - return None - path = scope.get("folder_path") or scope.get("path") - return normalize_path(path) if path else None - def _folder_exists(self, path: str) -> bool: try: self.store.folder_info(path) @@ -1947,15 +1543,3 @@ def _preferred_folder_path( if non_root: return sorted(non_root, key=lambda item: (len(item), item))[0] return fallback - - @staticmethod - def _parse_line_range(location: str) -> tuple[int, int]: - value = str(location).strip() - if "-" in value: - left, right = value.split("-", 1) - start, end = int(left), int(right) - else: - start = end = int(value) - if start < 1 or end < start: - raise ValueError(f"Invalid line range: {location}") - return start, end diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py index cc01d820a..e48428874 100644 --- a/pageindex/filesystem/semantic_index.py +++ b/pageindex/filesystem/semantic_index.py @@ -5,7 +5,7 @@ import sqlite3 from dataclasses import dataclass from pathlib import Path -from typing import Any, Protocol +from typing import Any import sqlite_vec @@ -14,6 +14,9 @@ class SemanticIndexError(RuntimeError): pass +SCHEMA_VERSION = 2 + + @dataclass(frozen=True) class SemanticIndexRecord: file_ref: str @@ -36,27 +39,6 @@ class SemanticSearchResult: metadata: dict[str, Any] -class RebuildableSemanticIndex(Protocol): - def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: - ... - - def upsert_many(self, records: list[SemanticIndexRecord]) -> int: - ... - - def search( - self, - vector: list[float], - *, - limit: int = 10, - filters: dict[str, Any] | None = None, - fetch_multiplier: int = 20, - ) -> list[SemanticSearchResult]: - ... - - def info(self) -> dict[str, Any]: - ... - - class SQLiteVecSemanticIndex: """Rebuildable local semantic index backed by sqlite-vec. @@ -93,8 +75,6 @@ def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> No created_at TEXT DEFAULT CURRENT_TIMESTAMP, updated_at TEXT DEFAULT CURRENT_TIMESTAMP ); - CREATE INDEX idx_semantic_index_docs_file_ref - ON semantic_index_docs(file_ref); CREATE INDEX idx_semantic_index_docs_external_id ON semantic_index_docs(external_id); CREATE INDEX idx_semantic_index_docs_source_type @@ -115,8 +95,42 @@ def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> No "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", sorted(config.items()), ) + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") conn.commit() + def validate(self, expected_identity: dict[str, Any]) -> dict[str, Any]: + with self.connect() as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + tables = { + str(row[1]) + for row in conn.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} + and not str(row[1]).startswith("sqlite_") + and not str(row[1]).startswith("semantic_index_vec_") + } + required = { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", + } + if version != SCHEMA_VERSION or tables != required: + raise SemanticIndexError( + "Incompatible PIFS Summary Projection schema; migrate this workspace " + "with pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + info = self.info() + if info["metadata"] != expected_identity: + raise SemanticIndexError( + "Incompatible PIFS Summary Embedding Profile; migrate the workspace or " + "use the base URL, model, and dimensions that created this projection." + ) + if int(info["dimension"]) != int(expected_identity["dimensions"]): + raise SemanticIndexError( + "Incompatible PIFS Summary Projection dimensions; migrate the workspace " + "or use the matching embedding profile." + ) + return info + def upsert_many(self, records: list[SemanticIndexRecord]) -> int: if not records: return 0 diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index 5eea7f5c9..5c22c3a1f 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -1,231 +1,128 @@ from __future__ import annotations -import json import sqlite3 import struct import time from dataclasses import dataclass from pathlib import Path from typing import Any +from urllib.parse import urlsplit, urlunsplit from .core import DEFAULT_EMBEDDING_DIMENSIONS from .semantic_index import ( + SCHEMA_VERSION, SQLiteVecSemanticIndex, - SemanticIndexError, SemanticIndexRecord, - SemanticSearchResult, ) -SUMMARY_CHANNEL = "summary" +DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" SUMMARY_INDEX_NAME = "summary" -@dataclass(frozen=True) -class SemanticProjectionCandidate: - document_id: str - score: float - sources: list[dict[str, Any]] - source_type: str - title: str - metadata: dict[str, Any] - snippet: str - - -class SemanticProjectionSearchBackend: - """Semantic channel retrieval over rebuildable projection indexes. - - The SQLite catalog remains the source of truth. This backend only reads - external sqlite-vec projection indexes and returns candidate document ids - for the catalog to resolve and filter. - """ - - def __init__( - self, - index_dir: str | Path, - *, - embedder: Any, - embedding_provider: str, - embedding_model: str, - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_cache_path: str | Path | None = None, - fetch_multiplier: int = 100, - ) -> None: - self.index_dir = Path(index_dir).expanduser() - self.embedder = embedder - self.embedding_provider = embedding_provider - self.embedding_model = embedding_model - self.embedding_dimensions = embedding_dimensions - self.cache_model = embedding_cache_model_key(embedding_model, embedding_dimensions) - self.embedding_cache = EmbeddingCache( - Path(embedding_cache_path).expanduser() - if embedding_cache_path is not None - else self.index_dir / "embedding_cache.sqlite" - ) - self.fetch_multiplier = fetch_multiplier - self.summary_index = SQLiteVecSemanticIndex( - self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" - ) - - @classmethod - def from_provider( - cls, - index_dir: str | Path, - *, - embedding_provider: str = "openai", - embedding_model: str = "text-embedding-3-small", - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_timeout: float = 60, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - **kwargs: Any, - ) -> "SemanticProjectionSearchBackend": - return cls( - index_dir, - embedder=make_embedder( - embedding_provider, - embedding_model, - dimensions=embedding_dimensions, - timeout=embedding_timeout, - api_key=embedding_api_key, - base_url=embedding_base_url, - ), - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - **kwargs, +def normalize_base_url(value: str | None) -> str: + raw = str(value or DEFAULT_OPENAI_BASE_URL).strip() + parsed = urlsplit(raw) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("embedding base URL must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError( + "embedding base URL must not contain credentials, query parameters, or a fragment" ) - - def search_channel( - self, - channel: str, - query: str, - *, - limit: int = 10, - filters: dict[str, Any] | None = None, - ) -> list[SemanticProjectionCandidate]: - if channel != SUMMARY_CHANNEL: - raise ValueError(f"unsupported semantic channel: {channel}") - if channel not in self.available_channels(): - return [] - query = normalize_text(query) - if not query: - return [] - vector = self.embedding_cache.embed_texts( - [query], - provider=self.embedding_provider, - model=self.cache_model, - embedder=self.embedder, - batch_size=1, - )[0] - results = self.summary_index.search( - vector, - limit=limit, - filters=filters, - fetch_multiplier=self.fetch_multiplier, + return urlunsplit( + ( + parsed.scheme.lower(), + parsed.netloc.lower(), + parsed.path.rstrip("/"), + "", + "", ) - return rank_single_semantic_channel(SUMMARY_CHANNEL, results) + ) - def available_channels(self) -> tuple[str, ...]: - return (SUMMARY_CHANNEL,) if self._summary_document_count() > 0 else () - def info(self) -> dict[str, Any]: +@dataclass(frozen=True) +class SummaryEmbeddingProfile: + base_url: str | None = DEFAULT_OPENAI_BASE_URL + model: str = "text-embedding-3-small" + dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS + timeout: float = 60 + api_key: str | None = None + + def __post_init__(self) -> None: + model = str(self.model or "").strip() + if not model: + raise ValueError("embedding model must not be empty") + if int(self.dimensions) <= 0: + raise ValueError("embedding dimensions must be positive") + if float(self.timeout) <= 0: + raise ValueError("embedding timeout must be positive") + object.__setattr__(self, "base_url", normalize_base_url(self.base_url)) + object.__setattr__(self, "model", model) + object.__setattr__(self, "dimensions", int(self.dimensions)) + object.__setattr__(self, "timeout", float(self.timeout)) + + @property + def identity(self) -> dict[str, Any]: return { - "index_dir": str(self.index_dir), - "embedding_provider": self.embedding_provider, - "embedding_model": self.embedding_model, - "embedding_dimensions": self.embedding_dimensions, - "strategy": "semantic_channel_vector", - "available_channels": list(self.available_channels()), - "channels": {SUMMARY_CHANNEL: self._safe_summary_info()}, + "base_url": self.base_url, + "model": self.model, + "dimensions": self.dimensions, } - def _summary_document_count(self) -> int: - info = self._safe_summary_info() - if not info.get("available"): - return 0 - return int(info.get("document_count") or 0) - - def _safe_summary_info(self) -> dict[str, Any]: - index = self.summary_index - if not index.db_path.exists(): - return { - "db_path": str(index.db_path), - "available": False, - "document_count": 0, - "error": "index file is missing", - } - try: - info = index.info() - except (OSError, sqlite3.Error, SemanticIndexError) as exc: - return { - "db_path": str(index.db_path), - "available": False, - "document_count": 0, - "error": str(exc), - } - return {**info, "available": int(info.get("document_count") or 0) > 0} + +@dataclass(frozen=True) +class SummaryProjectionCandidate: + file_ref: str + distance: float + source_type: str + title: str + metadata: dict[str, Any] + + @property + def similarity(self) -> float: + return 1.0 / (1.0 + max(0.0, self.distance)) -class SummaryProjectionIndexer: - """Synchronous register-time summary projection indexer.""" +class SummaryProjection: + """Own the complete PIFS Summary Projection lifecycle.""" def __init__( self, index_dir: str | Path, *, - embedder: Any, - embedding_provider: str, - embedding_model: str, - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_cache_path: str | Path | None = None, + profile: SummaryEmbeddingProfile, + embedder: Any | None = None, + create: bool = False, + fetch_multiplier: int = 100, ) -> None: self.index_dir = Path(index_dir).expanduser() - self.index_dir.mkdir(parents=True, exist_ok=True) - self.embedder = embedder - self.embedding_provider = embedding_provider - self.embedding_model = embedding_model - self.embedding_dimensions = embedding_dimensions - self.cache_model = embedding_cache_model_key(embedding_model, embedding_dimensions) - self.embedding_cache = EmbeddingCache( - Path(embedding_cache_path).expanduser() - if embedding_cache_path is not None - else self.index_dir / "embedding_cache.sqlite" - ) + self.profile = profile + self._embedder_instance = embedder + self.fetch_multiplier = fetch_multiplier self.index = SQLiteVecSemanticIndex( self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" ) - self._ensure_index() - - @classmethod - def from_provider( - cls, - index_dir: str | Path, - *, - embedding_provider: str = "openai", - embedding_model: str = "text-embedding-3-small", - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_timeout: float = 60, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - **kwargs: Any, - ) -> "SummaryProjectionIndexer": - cls._validate_existing_index_dimension(index_dir, embedding_dimensions) - return cls( - index_dir, - embedder=make_embedder( - embedding_provider, - embedding_model, - dimensions=embedding_dimensions, - timeout=embedding_timeout, - api_key=embedding_api_key, - base_url=embedding_base_url, - ), - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - **kwargs, - ) + summary_exists = self.index.db_path.exists() and self.index.db_path.stat().st_size > 0 + cache_path = self.index_dir / "embedding_cache.sqlite" + cache_exists = cache_path.exists() and cache_path.stat().st_size > 0 + if summary_exists: + self.index.validate(self.profile.identity) + self.embedding_cache = EmbeddingCache(cache_path, create=False) + elif cache_exists: + self.embedding_cache = EmbeddingCache(cache_path, create=False) + raise RuntimeError( + "PIFS Summary Projection is missing; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + elif create: + self.index_dir.mkdir(parents=True, exist_ok=True) + self.index.reset( + dimension=self.profile.dimensions, + metadata=self.profile.identity, + ) + self.embedding_cache = EmbeddingCache(cache_path, create=True) + else: + raise RuntimeError("PIFS Summary Projection is not available") def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: summary = str((record.get("metadata") or {}).get("summary") or "").strip() @@ -233,12 +130,10 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: return {"status": "skipped", "reason": "missing_summary"} vector = self.embedding_cache.embed_texts( [summary], - provider=self.embedding_provider, - model=self.cache_model, - embedder=self.embedder, + profile=self.profile, + embedder=self._embedder(), batch_size=1, )[0] - metadata = dict(record.get("metadata") or {}) count = self.index.upsert_many( [ SemanticIndexRecord( @@ -248,7 +143,7 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: external_id=record.get("external_id"), source_type=str(record.get("source_type") or ""), title=str(record.get("title") or ""), - metadata=metadata, + metadata=dict(record.get("metadata") or {}), ) ] ) @@ -256,135 +151,134 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: "status": "ready", "indexed_rows": count, "index_path": str(self.index.db_path), - "embedding_provider": self.embedding_provider, - "embedding_model": self.embedding_model, - "embedding_dimensions": self.embedding_dimensions, + **self.profile.identity, } def delete_summary(self, file_ref: str) -> int: return self.index.delete_file_refs([file_ref]) - def _ensure_index(self) -> None: - if not self.index.db_path.exists(): - self.index.reset( - dimension=self.embedding_dimensions, - metadata=self._index_metadata(), + def search( + self, + query: str, + *, + limit: int = 10, + file_refs: list[str] | None = None, + ) -> list[SummaryProjectionCandidate]: + query = normalize_text(query) + if not query or not self.available: + return [] + vector = self.embedding_cache.embed_texts( + [query], + profile=self.profile, + embedder=self._embedder(), + batch_size=1, + )[0] + filters = {"file_ref": file_refs} if file_refs is not None else None + return [ + SummaryProjectionCandidate( + file_ref=result.file_ref, + distance=result.distance, + source_type=result.source_type, + title=result.title, + metadata=result.metadata, ) - return - try: - existing_dimension = self.index.dimension() - except Exception as exc: - raise RuntimeError( - "could not validate existing summary projection index config; " - f"refusing to reset {self.index.db_path}. Move the existing index " - "aside or rebuild it intentionally before changing embedding config." - ) from exc - if existing_dimension != self.embedding_dimensions: - raise self._dimension_mismatch_error( - self.index.db_path, - existing_dimension, - self.embedding_dimensions, + for result in self.index.search( + vector, + limit=limit, + filters=filters, + fetch_multiplier=self.fetch_multiplier, ) + ] - def _index_metadata(self) -> dict[str, Any]: + @property + def available(self) -> bool: + return int(self.index.info().get("document_count") or 0) > 0 + + def info(self) -> dict[str, Any]: return { - "channel": "summary", - "embedding_provider": self.embedding_provider, - "embedding_model": self.embedding_model, - "embedding_dimensions": self.embedding_dimensions, + **self.index.info(), + "embedding_identity": self.profile.identity, + "available": self.available, } - @classmethod - def _validate_existing_index_dimension( - cls, - index_dir: str | Path, - embedding_dimensions: int, - ) -> None: - index_path = ( - Path(index_dir).expanduser() / f"{SUMMARY_INDEX_NAME}.sqlite" - ) - if not index_path.exists(): - return - index = SQLiteVecSemanticIndex(index_path) - try: - existing_dimension = index.dimension() - except Exception as exc: - raise RuntimeError( - "could not validate existing summary projection index config; " - f"refusing to reset {index_path}. Move the existing index " - "aside or rebuild it intentionally before changing embedding config." - ) from exc - if existing_dimension != embedding_dimensions: - raise cls._dimension_mismatch_error( - index_path, - existing_dimension, - embedding_dimensions, - ) - - @staticmethod - def _dimension_mismatch_error( - index_path: Path, - existing_dimension: int, - embedding_dimensions: int, - ) -> RuntimeError: - return RuntimeError( - "summary projection index dimension mismatch: " - f"{index_path} was built with dimension {existing_dimension}, " - f"but configured embedding_dimensions is {embedding_dimensions}. " - "Use the matching embedding config, or rebuild the projection index " - "at a new path after preserving the existing data." - ) + def _embedder(self) -> Any: + if self._embedder_instance is None: + self._embedder_instance = EmbeddingClient(self.profile) + return self._embedder_instance class EmbeddingCache: - def __init__(self, db_path: Path): + COLUMNS = { + "base_url", + "model", + "dimensions", + "text_hash", + "vector_blob", + "created_at", + } + + def __init__(self, db_path: Path, *, create: bool) -> None: self.db_path = db_path - self.db_path.parent.mkdir(parents=True, exist_ok=True) - with self.connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS embedding_cache ( - provider TEXT NOT NULL, - model TEXT NOT NULL, - text_hash TEXT NOT NULL, - dimension INTEGER NOT NULL, - vector_blob BLOB, - vector_json TEXT, - created_at TEXT DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(provider, model, text_hash) + exists = self.db_path.exists() and self.db_path.stat().st_size > 0 + if exists: + self._validate() + elif create: + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as connection: + connection.executescript( + f""" + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(base_url, model, dimensions, text_hash) + ); + PRAGMA user_version = {SCHEMA_VERSION}; + """ ) - """ + else: + raise RuntimeError( + "PIFS embedding cache is missing; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." ) - conn.commit() def connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn + connection = sqlite3.connect(self.db_path) + connection.row_factory = sqlite3.Row + return connection def embed_texts( self, texts: list[str], *, - provider: str, - model: str, + profile: SummaryEmbeddingProfile, embedder: Any, batch_size: int, ) -> list[list[float]]: hashes = [SQLiteVecSemanticIndex.text_hash(text) for text in texts] cached: dict[str, list[float]] = {} - with self.connect() as conn: + with self.connect() as connection: for text_hash in sorted(set(hashes)): - row = conn.execute( + row = connection.execute( """ - SELECT vector_blob, vector_json + SELECT vector_blob FROM embedding_cache - WHERE provider = ? AND model = ? AND text_hash = ? + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? """, - (provider, model, text_hash), + ( + profile.base_url, + profile.model, + profile.dimensions, + text_hash, + ), ).fetchone() if row is not None: - cached[text_hash] = decode_vector(row["vector_blob"], row["vector_json"]) + cached[text_hash] = decode_vector( + bytes(row["vector_blob"]), profile.dimensions + ) missing_positions = [ index for index, text_hash in enumerate(hashes) if text_hash not in cached ] @@ -397,117 +291,90 @@ def embed_texts( "embedding response length mismatch: " f"requested {len(positions)}, received {len(vectors)}" ) - with self.connect() as conn: - conn.executemany( + for vector in vectors: + if len(vector) != profile.dimensions: + raise ValueError( + "embedding dimension mismatch: " + f"expected {profile.dimensions}, received {len(vector)}" + ) + with self.connect() as connection: + connection.executemany( """ INSERT OR REPLACE INTO embedding_cache( - provider, model, text_hash, dimension, vector_blob, vector_json - ) - VALUES (?, ?, ?, ?, ?, '') + base_url, model, dimensions, text_hash, vector_blob + ) VALUES (?, ?, ?, ?, ?) """, [ ( - provider, - model, + profile.base_url, + profile.model, + profile.dimensions, hashes[index], - len(vector), encode_vector(vector), ) for index, vector in zip(positions, vectors) ], ) - conn.commit() for index, vector in zip(positions, vectors): cached[hashes[index]] = vector return [cached[text_hash] for text_hash in hashes] + def _validate(self) -> None: + with self.connect() as connection: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + tables = { + str(row[0]) + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + columns = { + str(row[1]) + for row in connection.execute("PRAGMA table_info(embedding_cache)") + } + if version != SCHEMA_VERSION or tables != {"embedding_cache"} or columns != self.COLUMNS: + raise RuntimeError( + "Incompatible PIFS embedding cache schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + class EmbeddingClient: - def __init__( - self, - *, - provider: str, - model: str, - dimensions: int, - timeout: float, - api_key: str | None = None, - base_url: str | None = None, - ): - self.provider = provider.lower() - self.model = model - self.dimensions = dimensions - if self.provider != "openai": - raise ValueError(f"unknown embedding provider: {provider}") + def __init__(self, profile: SummaryEmbeddingProfile) -> None: from openai import OpenAI - if not api_key: + if not profile.api_key: raise ValueError("embedding_api_key is required for PIFS embeddings") + self.profile = profile self.client = OpenAI( - api_key=api_key, - base_url=base_url or None, - timeout=timeout, + api_key=profile.api_key, + base_url=profile.base_url, + timeout=profile.timeout, ) def embed(self, texts: list[str]) -> list[list[float]]: - kwargs: dict[str, Any] = {"model": self.model, "input": texts} - if self.dimensions > 0: - kwargs["dimensions"] = self.dimensions - response = self.client.embeddings.create(**kwargs) - return [list(item.embedding) for item in sorted(response.data, key=lambda item: item.index)] - - -def make_embedder( - provider: str, - model: str, - *, - dimensions: int, - timeout: float, - api_key: str | None = None, - base_url: str | None = None, -) -> Any: - return EmbeddingClient( - provider=provider, - model=model, - dimensions=dimensions, - timeout=timeout, - api_key=api_key, - base_url=base_url, - ) - - -def rank_single_semantic_channel( - channel: str, - results: list[SemanticSearchResult], -) -> list[SemanticProjectionCandidate]: - rows: list[SemanticProjectionCandidate] = [] - seen: set[str] = set() - for rank, result in enumerate(results, 1): - doc_id = str(result.external_id or result.file_ref) - if doc_id in seen: - continue - seen.add(doc_id) - rows.append( - SemanticProjectionCandidate( - document_id=doc_id, - score=1 / (60 + rank), - sources=[{"channel": channel, "rank": rank, "distance": result.distance}], - source_type=result.source_type, - title=result.title, - metadata=result.metadata, - snippet=f"{channel}_vector rank={rank}", - ) + response = self.client.embeddings.create( + model=self.profile.model, + input=texts, + dimensions=self.profile.dimensions, ) - return rows + return [ + list(item.embedding) + for item in sorted(response.data, key=lambda item: item.index) + ] + def normalize_text(text: str) -> str: return " ".join(str(text or "").split()) -def embedding_cache_model_key(model: str, dimensions: int) -> str: - return f"{model}:dimensions={dimensions}" if dimensions > 0 else model - - -def embed_with_retry(embedder: Any, texts: list[str], *, max_attempts: int = 8) -> list[list[float]]: +def embed_with_retry( + embedder: Any, + texts: list[str], + *, + max_attempts: int = 8, +) -> list[list[float]]: for attempt in range(1, max_attempts + 1): try: return embedder.embed(texts) @@ -528,9 +395,7 @@ def is_retryable_embedding_error(exc: Exception) -> bool: except (TypeError, ValueError): status = None if status is not None: - if status in {408, 409, 429}: - return True - if status >= 500: + if status in {408, 409, 429} or status >= 500: return True if 400 <= status < 500: return False @@ -542,13 +407,9 @@ def encode_vector(vector: list[float]) -> bytes: return struct.pack(f"<{len(vector)}f", *vector) -def decode_vector(blob: bytes | None, vector_json: str | None) -> list[float]: - if blob: - if len(blob) % 4 != 0: - raise ValueError("invalid cached vector blob length") - return list(struct.unpack(f"<{len(blob) // 4}f", blob)) - if vector_json: - value = json.loads(vector_json) - if isinstance(value, list): - return [float(item) for item in value] - raise ValueError("cached embedding row does not contain a vector") +def decode_vector(blob: bytes, dimensions: int) -> list[float]: + if len(blob) != dimensions * 4: + raise ValueError( + f"cached embedding has {len(blob) // 4} dimensions, expected {dimensions}" + ) + return list(struct.unpack(f"<{dimensions}f", blob)) diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py index a714c3979..b7c80484a 100644 --- a/pageindex/filesystem/store.py +++ b/pageindex/filesystem/store.py @@ -9,7 +9,14 @@ from .types import FileEntry, MetadataField -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 +CATALOG_TABLES = { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", +} class SQLiteFileSystemStore: @@ -20,9 +27,9 @@ def __init__(self, workspace: str | Path): self.text_dir = self.workspace / "artifacts" / "text" self.raw_dir = self.workspace / "artifacts" / "raw" self.pageindex_client_dir = self.workspace / "artifacts" / "pageindex_client" + self.initialize_schema() for path in (self.text_dir, self.raw_dir, self.pageindex_client_dir): path.mkdir(parents=True, exist_ok=True) - self.initialize_schema() def connect(self) -> sqlite3.Connection: conn = sqlite3.connect(self.db_path) @@ -31,6 +38,25 @@ def connect(self) -> sqlite3.Connection: return conn def initialize_schema(self) -> None: + if self.db_path.exists() and self.db_path.stat().st_size > 0: + with self.connect() as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + tables = { + str(row[0]) + for row in conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + if version != SCHEMA_VERSION or tables != CATALOG_TABLES: + raise RuntimeError( + "Incompatible PIFS catalog schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it " + f"(expected version {SCHEMA_VERSION} and tables " + f"{sorted(CATALOG_TABLES)}, found version {version} and tables " + f"{sorted(tables)})." + ) + return with self.connect() as conn: self._create_current_schema(conn) self.ensure_folder(conn, "/") @@ -101,15 +127,10 @@ def _create_current_schema(self, conn: sqlite3.Connection) -> None: FOREIGN KEY(field_id) REFERENCES metadata_fields(field_id) ON DELETE CASCADE ); - CREATE VIRTUAL TABLE IF NOT EXISTS file_fts - USING fts5(file_ref UNINDEXED, title, body, metadata_text); - CREATE INDEX IF NOT EXISTS idx_files_external_id ON files(external_id); CREATE INDEX IF NOT EXISTS idx_files_source_type ON files(source_type); - CREATE INDEX IF NOT EXISTS idx_folders_path ON folders(path); CREATE INDEX IF NOT EXISTS idx_folders_parent_id ON folders(parent_id); CREATE INDEX IF NOT EXISTS idx_file_folders_folder ON file_folders(folder_id); - CREATE INDEX IF NOT EXISTS idx_metadata_fields_name ON metadata_fields(name); CREATE INDEX IF NOT EXISTS idx_metadata_values_field_text ON metadata_values(field_id, value_text); """ ) @@ -138,8 +159,6 @@ def insert_files(self, records: list[dict[str, Any]]) -> None: file_rows = [] membership_rows = [] file_ref_rows = [] - fts_file_ref_rows = [] - fts_rows = [] metadata_rows = [] pending_folder_titles: dict[tuple[str, str], str] = {} metadata_field_ids = { @@ -179,16 +198,6 @@ def insert_files(self, records: list[dict[str, Any]]) -> None: ) ) file_ref_rows.append((record["file_ref"],)) - if not record.get("skip_fts", False): - fts_file_ref_rows.append((record["file_ref"],)) - fts_rows.append( - ( - record["file_ref"], - record["title"], - record["content"], - record["metadata_text"], - ) - ) metadata_rows.extend( self._metadata_insert_values( record["file_ref"], @@ -214,15 +223,6 @@ def insert_files(self, records: list[dict[str, Any]]) -> None: """, metadata_rows, ) - if fts_file_ref_rows: - conn.executemany("DELETE FROM file_fts WHERE file_ref = ?", fts_file_ref_rows) - conn.executemany( - """ - INSERT INTO file_fts(file_ref, title, body, metadata_text) - VALUES (?, ?, ?, ?) - """, - fts_rows, - ) @staticmethod def _file_insert_sql() -> str: @@ -482,21 +482,6 @@ def _registered_field_id(conn: sqlite3.Connection, name: str) -> str | None: ).fetchone() return None if row is None else row["field_id"] - def replace_fts(self, conn: sqlite3.Connection, record: dict[str, Any]) -> None: - conn.execute("DELETE FROM file_fts WHERE file_ref = ?", (record["file_ref"],)) - conn.execute( - """ - INSERT INTO file_fts(file_ref, title, body, metadata_text) - VALUES (?, ?, ?, ?) - """, - ( - record["file_ref"], - record["title"], - record["content"], - record["metadata_text"], - ), - ) - def upsert_metadata_fields( self, fields: Iterable[MetadataField], @@ -787,48 +772,13 @@ def find_folders( rows = conn.execute(sql, params).fetchall() return [self._folder_row_to_dict(row) for row in rows] - def search_files( + def list_files( self, - query: str | list[str] | None, *, scope: Optional[dict[str, Any]] = None, metadata_filter: Optional[dict[str, Any]] = None, - limit: int = 10, + limit: int = 100, ) -> list[dict[str, Any]]: - query_text = self._query_text(query) - match_queries = self._fts_match_queries(query_text) if query_text else [None] - if query_text and not match_queries: - match_queries = [None] - results: list[dict[str, Any]] = [] - seen: set[str] = set() - for match_query in match_queries: - if query_text and match_query is None: - rows = self._search_literal_once( - query_text, - scope, - metadata_filter, - max(limit * 25, limit), - ) - else: - rows = self._search_once(match_query, scope, metadata_filter, max(limit * 25, limit)) - for row in rows: - if row["file_ref"] in seen: - continue - seen.add(row["file_ref"]) - results.append(self._search_row_to_dict(row)) - if len(results) >= limit: - return results - if results: - return results - return results - - def _search_literal_once( - self, - query_text: str, - scope: Optional[dict[str, Any]], - metadata_filter: Optional[dict[str, Any]], - limit: int, - ) -> list[sqlite3.Row]: selects = [ "f.file_ref", "f.external_id", @@ -860,21 +810,9 @@ def _search_literal_once( LIMIT 1 ) AS folder_path """, - "f.descriptor AS snippet", - "0 AS rank", - ] - where = [ - "f.deleted_at IS NULL", - """ - ( - lower(file_fts.title) LIKE lower(?) ESCAPE '\\' - OR lower(file_fts.body) LIKE lower(?) ESCAPE '\\' - OR lower(file_fts.metadata_text) LIKE lower(?) ESCAPE '\\' - ) - """, ] - literal_like = self._contains_like(query_text) - params: list[Any] = [literal_like, literal_like, literal_like] + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] scope_sql, scope_params = self._scope_sql(scope) if scope_sql: where.append(scope_sql) @@ -885,14 +823,14 @@ def _search_literal_once( sql = f""" SELECT {", ".join(selects)} FROM files f - JOIN file_fts ON file_fts.file_ref = f.file_ref WHERE {" AND ".join(where)} - ORDER BY f.created_at DESC, f.title + ORDER BY lower(f.title), f.title, f.file_ref LIMIT ? """ params.append(limit) with self.connect() as conn: - return conn.execute(sql, params).fetchall() + rows = conn.execute(sql, params).fetchall() + return [self._file_summary(row) for row in rows] def file_refs_for_scope( self, @@ -1027,78 +965,6 @@ def list_metadata_values( for row in rows ] - def _search_once( - self, - match_query: str | None, - scope: Optional[dict[str, Any]], - metadata_filter: Optional[dict[str, Any]], - limit: int, - ) -> list[sqlite3.Row]: - joins = [] - selects = [ - "f.file_ref", - "f.external_id", - "f.title", - "f.descriptor", - "f.pageindex_tree_status", - "f.metadata_json", - "f.metadata_status_json", - "f.created_at", - """ - ( - SELECT display_folder.folder_id - FROM file_folders display_ff - JOIN folders display_folder - ON display_folder.folder_id = display_ff.folder_id - WHERE display_ff.file_ref = f.file_ref - ORDER BY display_folder.path - LIMIT 1 - ) AS folder_id - """, - """ - ( - SELECT display_folder.path - FROM file_folders display_ff - JOIN folders display_folder - ON display_folder.folder_id = display_ff.folder_id - WHERE display_ff.file_ref = f.file_ref - ORDER BY display_folder.path - LIMIT 1 - ) AS folder_path - """, - ] - where = ["f.deleted_at IS NULL"] - params: list[Any] = [] - if match_query: - joins.append("JOIN file_fts ON file_fts.file_ref = f.file_ref") - selects.append("snippet(file_fts, 2, '', '', '...', 16) AS snippet") - selects.append("bm25(file_fts) AS rank") - where.append("file_fts MATCH ?") - params.append(match_query) - order_by = "rank" - else: - selects.append("f.descriptor AS snippet") - selects.append("0 AS rank") - order_by = "f.created_at DESC, f.title" - scope_sql, scope_params = self._scope_sql(scope) - if scope_sql: - where.append(scope_sql) - params.extend(scope_params) - metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) - where.extend(metadata_sql) - params.extend(metadata_params) - sql = f""" - SELECT {", ".join(selects)} - FROM files f - {" ".join(joins)} - WHERE {" AND ".join(where)} - ORDER BY {order_by} - LIMIT ? - """ - params.append(limit) - with self.connect() as conn: - return conn.execute(sql, params).fetchall() - def _metadata_filter_sql(self, metadata_filter: Optional[dict[str, Any]]) -> tuple[list[str], list[Any]]: if not metadata_filter: return [], [] @@ -1205,7 +1071,6 @@ def update_file_metadata_status( if row is None: raise KeyError(f"Unknown file_ref: {file_ref}") indexed_metadata = self.indexed_metadata_values(metadata) - metadata_text_value = metadata_text(indexed_metadata) conn.execute( """ UPDATE files @@ -1225,19 +1090,10 @@ def update_file_metadata_status( file_ref, indexed_metadata, ) - conn.execute( - """ - UPDATE file_fts - SET metadata_text = ? - WHERE file_ref = ? - """, - (metadata_text_value, file_ref), - ) def delete_file(self, target: str) -> None: with self.connect() as conn: file_ref = self._resolve_file_ref(conn, target) - conn.execute("DELETE FROM file_fts WHERE file_ref = ?", (file_ref,)) conn.execute("DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,)) conn.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) @@ -1573,9 +1429,7 @@ def folder_memberships(self, file_ref: str) -> list[dict[str, Any]]: return [ { "folder_id": row["folder_id"], - "id": row["folder_id"], "parent_id": row["parent_id"], - "parent_folder_id": row["parent_id"], "name": row["name"], "path": row["path"], "kind": row["kind"], @@ -1939,9 +1793,7 @@ def _folder_depth_sql(path_expr: str) -> str: def _folder_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: return { "folder_id": row["folder_id"], - "id": row["folder_id"], "parent_id": row["parent_id"], - "parent_folder_id": row["parent_id"], "name": row["name"], "description": cls._row_value(row, "description", ""), "path": row["path"], @@ -1956,44 +1808,17 @@ def _folder_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: @classmethod def _file_summary(cls, row: sqlite3.Row) -> dict[str, Any]: - external_id = row["external_id"] display_title = cls._row_value(row, "display_title", row["title"]) return { "file_ref": row["file_ref"], - "id": external_id or row["file_ref"], - "document_id": external_id, - "external_id": external_id, - "name": display_title, + "external_id": row["external_id"], "title": display_title, - "original_title": row["title"], - "description": cls._row_value(row, "descriptor", row["title"]), - "status": cls._row_value(row, "pageindex_tree_status", "not_built"), - "pageNum": None, - "createdAt": cls._row_value(row, "created_at"), - "folderId": cls._row_value(row, "folder_id"), - "folder_path": row["folder_path"], - "metadata": json.loads(row["metadata_json"] or "{}"), - "metadata_status": json.loads( - cls._row_value(row, "metadata_status_json", "{}") or "{}" + "descriptor": cls._row_value(row, "descriptor", row["title"]), + "pageindex_tree_status": cls._row_value( + row, "pageindex_tree_status", "not_built" ), - } - - @classmethod - def _search_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: - external_id = row["external_id"] - return { - "file_ref": row["file_ref"], - "id": external_id or row["file_ref"], - "document_id": external_id, - "external_id": external_id, - "name": row["title"], - "title": row["title"], - "description": cls._row_value(row, "descriptor", row["title"]), - "status": cls._row_value(row, "pageindex_tree_status", "not_built"), - "pageNum": None, - "createdAt": cls._row_value(row, "created_at"), - "folderId": cls._row_value(row, "folder_id"), - "snippet": row["snippet"] or row["title"], + "created_at": cls._row_value(row, "created_at"), + "folder_id": cls._row_value(row, "folder_id"), "folder_path": row["folder_path"], "metadata": json.loads(row["metadata_json"] or "{}"), "metadata_status": json.loads( @@ -2031,15 +1856,8 @@ def _file_entry(row: sqlite3.Row) -> FileEntry: def _file_entry_to_dict(cls, entry: FileEntry) -> dict[str, Any]: return { "file_ref": entry.file_ref, - "id": entry.external_id or entry.file_ref, - "document_id": entry.external_id, "external_id": entry.external_id, - "name": entry.title, - "path": cls._virtual_file_path(entry.folder_path, entry.title), "title": entry.title, - "description": entry.descriptor, - "status": entry.pageindex_tree_status, - "pageNum": None, "descriptor": entry.descriptor, "content_type": entry.content_type, "source_type": entry.source_type, @@ -2051,75 +1869,6 @@ def _file_entry_to_dict(cls, entry: FileEntry) -> dict[str, Any]: "folder_path": entry.folder_path, } - @staticmethod - def _virtual_file_path(folder_path: str, title: str) -> str: - folder_path = normalize_path(folder_path) - return f"/{title}" if folder_path == "/" else f"{folder_path}/{title}" - - @staticmethod - def _query_text(query: str | list[str] | None) -> str: - if query is None: - return "" - if isinstance(query, list): - return " ".join(str(item) for item in query) - return str(query) - - @classmethod - def _fts_match_queries(cls, query: str) -> list[str]: - terms = cls._fts_terms(query) - if not terms: - return [] - queries = [" ".join(terms)] - if len(terms) > 1: - queries.append(" OR ".join(terms)) - return queries - - @staticmethod - def _fts_terms(query: str) -> list[str]: - stopwords = { - "a", - "an", - "and", - "are", - "as", - "at", - "be", - "by", - "did", - "do", - "does", - "for", - "from", - "how", - "in", - "is", - "it", - "of", - "on", - "or", - "that", - "the", - "to", - "was", - "were", - "what", - "when", - "where", - "which", - "who", - "why", - "with", - } - terms = re.findall(r"[A-Za-z0-9_]+", query.lower()) - unique_terms = [] - seen = set() - for term in terms: - if term in stopwords or term in seen: - continue - seen.add(term) - unique_terms.append(term) - return unique_terms - @staticmethod def _metadata_value_items(value: Any) -> list[str]: if value is None: @@ -2179,15 +1928,3 @@ def make_file_ref(seed: str) -> str: def fingerprint(content: str) -> str: return hashlib.sha256(content.encode("utf-8")).hexdigest() - - -def metadata_text(metadata: dict[str, Any]) -> str: - values = [] - for value in metadata.values(): - if isinstance(value, list): - values.extend(str(item) for item in value) - elif isinstance(value, dict): - values.append(json.dumps(value, ensure_ascii=False, sort_keys=True)) - elif value is not None: - values.append(str(value)) - return " ".join(values) diff --git a/pageindex/filesystem/types.py b/pageindex/filesystem/types.py index 82c04bb54..6f933e5f6 100644 --- a/pageindex/filesystem/types.py +++ b/pageindex/filesystem/types.py @@ -4,45 +4,6 @@ from typing import Any, Optional -@dataclass(frozen=True) -class SearchResult: - file_ref: str - external_id: Optional[str] - title: str - snippet: str - folder_path: str - folder_paths: list[str] - metadata: dict[str, Any] - id: Optional[str] = None - document_id: Optional[str] = None - name: str = "" - description: str = "" - status: str = "" - pageNum: Optional[int] = None - createdAt: Optional[str] = None - folderId: Optional[str] = None - metadata_status: dict[str, Any] = field(default_factory=dict) - - -@dataclass(frozen=True) -class OpenResult: - file_ref: str - start_line: int - end_line: int - text: str - external_id: Optional[str] = None - folder_path: str = "" - - -@dataclass(frozen=True) -class FolderEntry: - folder_id: str - parent_id: Optional[str] - name: str - path: str - kind: str - - @dataclass(frozen=True) class FileEntry: file_ref: str @@ -75,10 +36,3 @@ class PIFSQueryScope: folder_path: str metadata_filter: dict[str, str] = field(default_factory=dict) metadata_axis: Optional[str] = None - - -@dataclass(frozen=True) -class CommandResult: - command: str - data: Any - text: str diff --git a/tests/test_filesystem_store.py b/tests/test_filesystem_store.py index affeb50cb..d9b726d42 100644 --- a/tests/test_filesystem_store.py +++ b/tests/test_filesystem_store.py @@ -35,9 +35,6 @@ def connect(self): "raw_artifact_path": None, "metadata": {}, "metadata_json": json.dumps({}), - "metadata_text": "", - "content": "", - "skip_fts": True, } ] ) @@ -106,7 +103,7 @@ def test_file_upsert_preserves_soft_delete_tombstone(tmp_path): assert row["storage_uri"].endswith("/doc_report.md") assert row["deleted_at"] == "2001-02-03 04:05:06" - assert filesystem.search(None) == [] + assert filesystem.store.list_files() == [] def test_listing_uses_one_consistent_folder_membership_row(tmp_path): @@ -127,7 +124,7 @@ def test_listing_uses_one_consistent_folder_membership_row(tmp_path): metadata={"display_name": "Alpha"}, ) - listing = filesystem.browse("/", recursive=True, limit=10) + listing = filesystem.store.list_folder("/", recursive=True, limit=10) row = next(item for item in listing["files"] if item["external_id"] == "doc_shared") assert row["folder_path"] == "/a" diff --git a/tests/test_pifs_add_command.py b/tests/test_pifs_add_command.py deleted file mode 100644 index 80670eb28..000000000 --- a/tests/test_pifs_add_command.py +++ /dev/null @@ -1,484 +0,0 @@ -import json -from pathlib import Path - -import pytest - - -class StaticEmbedder: - def embed(self, texts): - return [[1.0, 0.0, 0.0] for _ in texts] - - -def make_summary_indexer(workspace: Path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - return SummaryProjectionIndexer( - workspace / "artifacts" / "projection_indexes", - embedder=StaticEmbedder(), - embedding_provider="test", - embedding_model="static", - embedding_dimensions=3, - ) - - -def make_filesystem(workspace: Path): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem( - workspace=workspace, - summary_projection_embedding_provider="test", - summary_projection_embedding_model="static", - summary_projection_embedding_dimensions=3, - ) - filesystem.summary_projection_indexer = make_summary_indexer(workspace) - return filesystem - - -@pytest.fixture(autouse=True) -def fake_pageindex_index(monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - path = Path(file_path) - doc_id = f"doc_{path.stem}" - text = path.read_text(encoding="utf-8") - doc = { - "id": doc_id, - "type": "md", - "path": str(path.resolve()), - "doc_name": path.name, - "doc_description": f"Summary for {path.name}: {text[:60]}", - "line_count": len(text.splitlines()), - "structure": [ - { - "title": path.stem, - "node_id": "0001", - "line_num": 1, - "text": text, - "nodes": [], - } - ], - "pages": [{"page": 1, "content": text}], - } - write_pageindex_client_doc(self.workspace, doc_id, doc) - self.documents[doc_id] = doc - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - - -def write_pageindex_client_doc(workspace: Path, doc_id: str, doc: dict) -> None: - workspace.mkdir(parents=True, exist_ok=True) - (workspace / f"{doc_id}.json").write_text( - json.dumps(doc, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - meta = { - doc_id: { - "type": doc.get("type", ""), - "doc_name": doc.get("doc_name", ""), - "doc_description": doc.get("doc_description", ""), - "path": doc.get("path", ""), - "line_count": doc.get("line_count"), - } - } - (workspace / "_meta.json").write_text( - json.dumps(meta, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - - -def test_add_text_folder_target_copies_artifact_indexes_summary_and_is_readable(tmp_path): - from pageindex.filesystem import PIFSCommandExecutor - - source = tmp_path / "filing.md" - source.write_text("alpha filing text for pifs add", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - info = filesystem.add_file(str(source), "/documents/reports") - - assert info["path"] == "/documents/reports/filing.md" - assert info["folder_path"] == "/documents/reports" - assert filesystem.folder_info("/documents/reports")["path"] == "/documents/reports" - entry = filesystem.store.get_file(info["file_ref"]) - assert entry.storage_uri != source.as_uri() - assert "/artifacts/uploads/" in entry.storage_uri - copied_path = Path(entry.storage_uri.removeprefix("file://")) - assert copied_path.read_text(encoding="utf-8") == "alpha filing text for pifs add" - assert copied_path.resolve() != source.resolve() - - executor = PIFSCommandExecutor(filesystem) - rendered = json.loads(executor.execute("grep alpha /documents/reports/filing.md")) - - assert rendered["data"]["matches"] == [ - {"line": 1, "text": "alpha filing text for pifs add"} - ] - assert info["metadata"]["summary"].startswith("Summary for filing.md") - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 1 - - -def test_add_rejects_same_folder_same_basename_without_overwrite(tmp_path): - from pageindex.filesystem import PIFSCommandExecutor - - source = tmp_path / "conflict.md" - source.write_text("first body", encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - filesystem.add_file(source, "/documents") - source.write_text("second body must not overwrite", encoding="utf-8") - - with pytest.raises(FileExistsError, match="already exists"): - filesystem.add_file(source, "/documents") - - executor = PIFSCommandExecutor(filesystem) - rendered = json.loads(executor.execute("grep first /documents/conflict.md")) - assert rendered["data"]["matches"] == [{"line": 1, "text": "first body"}] - - -def test_add_rejects_unsupported_type_before_registration(tmp_path): - source = tmp_path / "payload.json" - source.write_text('{"unsupported": true}', encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - with pytest.raises(ValueError, match="Unsupported file type"): - filesystem.add_file(source, "/documents") - - assert filesystem.browse("/", recursive=True)["files"] == [] - assert not list((tmp_path / "workspace" / "artifacts" / "uploads").glob("**/*")) - - -def test_add_configures_semantic_retrieval_in_same_filesystem_instance(tmp_path): - source = tmp_path / "semantic.md" - source.write_text("alpha semantic recall text", encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - assert filesystem.semantic_retrieval_channels() == () - - filesystem.add_file(source, "/documents") - - assert filesystem.semantic_retrieval_channels() == ("summary",) - results = filesystem.browse_semantic_files( - "/documents", - "semantic recall", - recursive=True, - page_size=5, - ) - assert [item["path"] for item in results["data"]] == ["/documents/semantic.md"] - - -def test_add_markdown_builds_pageindex_tree_from_copied_artifact(tmp_path, monkeypatch): - from pageindex import PageIndexClient - from pageindex.filesystem import PIFSCommandExecutor - - indexed_paths = [] - - def fake_index(self, file_path, mode="auto"): - indexed_paths.append(Path(file_path)) - doc_id = "doc_added_md" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "notes.md", - "doc_description": "summary", - "line_count": 3, - "structure": [ - { - "title": "Notes", - "node_id": "0001", - "line_num": 1, - "text": "# Notes\n\ncopied markdown body", - "nodes": [], - } - ], - } - write_pageindex_client_doc(self.workspace, doc_id, doc) - self.documents[doc_id] = doc - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "notes.md" - source.write_text("# Notes\n\ncopied markdown body", encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - info = filesystem.add_file(source, "/documents") - executor = PIFSCommandExecutor(filesystem) - structure = json.loads(executor.execute("cat /documents/notes.md --structure")) - entry = filesystem.store.get_file(info["file_ref"]) - - assert structure["data"]["document"]["available"] is True - assert structure["data"]["structure"][0]["title"] == "Notes" - assert indexed_paths == [Path(entry.storage_uri.removeprefix("file://"))] - assert indexed_paths[0].resolve() != source.resolve() - - -def test_add_failure_does_not_leave_visible_catalog_or_artifacts(tmp_path, monkeypatch): - source = tmp_path / "atomic.md" - source.write_text("atomic body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - filesystem.add_file(source, "/documents") - - assert filesystem.browse("/", recursive=True)["files"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_markdown_insert_failure_removes_pageindex_cache(tmp_path, monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = "doc_failed_add_md" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "failed.md", - "doc_description": "summary", - "line_count": 3, - "structure": [ - { - "title": "Failed", - "node_id": "0001", - "line_num": 1, - "text": "# Failed\n\nbody", - "nodes": [], - } - ], - } - write_pageindex_client_doc(self.workspace, doc_id, doc) - self.documents[doc_id] = doc - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "failed.md" - source.write_text("# Failed\n\nbody", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - filesystem.add_file(source, "/documents/reports") - - pageindex_workspace = workspace / "artifacts" / "pageindex_client" - assert not (pageindex_workspace / "doc_failed_add_md.json").exists() - meta_path = pageindex_workspace / "_meta.json" - if meta_path.exists(): - meta = json.loads(meta_path.read_text(encoding="utf-8")) - assert "doc_failed_add_md" not in meta - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert listing["folders"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_markdown_index_failure_removes_pageindex_cache_delta(tmp_path, monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = "doc_partial_before_raise" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "partial.md", - "doc_description": "summary", - "line_count": 3, - "structure": [{"title": "Partial", "node_id": "0001", "nodes": []}], - } - self.documents[doc_id] = doc - self._save_doc(doc_id) - raise RuntimeError("index failed after cache write") - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "partial.md" - source.write_text("# Partial\n\nbody", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - pageindex_workspace = workspace / "artifacts" / "pageindex_client" - - with pytest.raises(RuntimeError, match="requires PageIndex extraction"): - filesystem.add_file(source, "/documents/reports") - - assert not (pageindex_workspace / "doc_partial_before_raise.json").exists() - meta_path = pageindex_workspace / "_meta.json" - if meta_path.exists(): - meta = json.loads(meta_path.read_text(encoding="utf-8")) - assert "doc_partial_before_raise" not in meta - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert listing["folders"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_markdown_failure_preserves_unrelated_pageindex_cache(tmp_path, monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = "doc_failed_add_md" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "failed.md", - "doc_description": "summary", - "line_count": 3, - "structure": [{"title": "Failed", "node_id": "0001", "nodes": []}], - } - self.documents[doc_id] = doc - self._save_doc(doc_id) - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "failed.md" - source.write_text("# Failed\n\nbody", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - pageindex_workspace = workspace / "artifacts" / "pageindex_client" - write_pageindex_client_doc( - pageindex_workspace, - "doc_unrelated", - { - "id": "doc_unrelated", - "type": "md", - "path": str((tmp_path / "unrelated.md").resolve()), - "doc_name": "unrelated.md", - "doc_description": "summary", - "line_count": 1, - "structure": [{"title": "Unrelated", "node_id": "0001", "nodes": []}], - }, - ) - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - filesystem.add_file(source, "/documents") - - assert not (pageindex_workspace / "doc_failed_add_md.json").exists() - assert (pageindex_workspace / "doc_unrelated.json").exists() - meta = json.loads((pageindex_workspace / "_meta.json").read_text(encoding="utf-8")) - assert "doc_failed_add_md" not in meta - assert "doc_unrelated" in meta - - -def test_add_failure_after_summary_vector_rolls_back_catalog_and_vector( - tmp_path, monkeypatch -): - source = tmp_path / "post_vector.md" - source.write_text("post vector rollback body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_status_update(*args, **kwargs): - raise RuntimeError("metadata status update failed") - - monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) - - with pytest.raises(RuntimeError, match="metadata status update failed"): - filesystem.add_file(source, "/documents") - - assert filesystem.browse("/", recursive=True)["files"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_failure_removes_nested_folders_created_only_for_add(tmp_path, monkeypatch): - source = tmp_path / "nested.md" - source.write_text("nested rollback body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_status_update(*args, **kwargs): - raise RuntimeError("metadata status update failed") - - monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) - - with pytest.raises(RuntimeError, match="metadata status update failed"): - filesystem.add_file(source, "/documents/reports") - - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert listing["folders"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_failure_preserves_preexisting_parent_folder(tmp_path, monkeypatch): - source = tmp_path / "nested.md" - source.write_text("nested rollback body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - filesystem.create_folder("/documents") - - def fail_status_update(*args, **kwargs): - raise RuntimeError("metadata status update failed") - - monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) - - with pytest.raises(RuntimeError, match="metadata status update failed"): - filesystem.add_file(source, "/documents/reports") - - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert [folder["path"] for folder in listing["folders"]] == ["/documents"] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - - -def test_cli_add_uses_workspace_and_prints_added_file(monkeypatch, capsys, tmp_path): - from pageindex.filesystem import cli - - source = tmp_path / "cli.md" - source.write_text("cli body", encoding="utf-8") - calls = [] - - class FakeAddFileSystem: - def __init__(self, workspace, **_kwargs): - self.workspace = Path(workspace) - - def configure_existing_projection_retrieval(self): - return False - - def add_file(self, physical_path, virtual_target): - calls.append((self.workspace, physical_path, virtual_target)) - return { - "file_ref": "file_cli", - "path": "/documents/cli.md", - } - - monkeypatch.setattr(cli, "PageIndexFileSystem", FakeAddFileSystem) - - status = cli.main(["--workspace", str(tmp_path / "workspace"), "add", str(source), "/documents"]) - - assert status == 0 - assert calls == [(tmp_path / "workspace", str(source), "/documents")] - assert capsys.readouterr().out == ( - "added: /documents/cli.md\n" - "file_ref: file_cli\n" - ) diff --git a/tests/test_pifs_agent_stream.py b/tests/test_pifs_agent_stream.py index 5e1d0e684..d2c664586 100644 --- a/tests/test_pifs_agent_stream.py +++ b/tests/test_pifs_agent_stream.py @@ -1,10 +1,8 @@ -import ast import io import os import tempfile import threading import unittest -from pathlib import Path from unittest.mock import patch from types import SimpleNamespace @@ -30,21 +28,6 @@ from pageindex.filesystem import PageIndexFileSystem -def load_demo_agent_prompt() -> str: - demo_path = Path(__file__).resolve().parents[1] / "examples" / "pifs_demo.py" - module = ast.parse(demo_path.read_text(encoding="utf-8")) - for node in module.body: - if isinstance(node, ast.Assign): - names = [ - target.id - for target in node.targets - if isinstance(target, ast.Name) - ] - if "PIFS_DEMO_AGENT_PROMPT" in names and isinstance(node.value, ast.Constant): - return str(node.value.value) - raise AssertionError("PIFS_DEMO_AGENT_PROMPT not found") - - class StructuredAnswer(BaseModel): model_config = ConfigDict(extra="forbid") @@ -88,20 +71,25 @@ def test_tools_mode_does_not_print_model_text(self): observer = PIFSAgentStreamObserver("tools", stream_log=stream_log, output=output) observer.handle_event(self.raw_event("response.output_text.delta", "hidden from tools mode")) - observer.handle_event(self.raw_event("response.function_call_arguments.delta", '{"command":"ls /"}')) - observer.emit_tool_call("ls /") + observer.handle_event(self.raw_event("response.function_call_arguments.delta", '{"command":"tree / -L 1"}')) + observer.emit_tool_call("tree / -L 1") observer.emit_tool_result(ok=True, output='{"ok": true}', seconds=0.001) observer.finish() printed = output.getvalue() self.assertNotIn("hidden from tools mode", printed) self.assertIn("[llm -> pifs command]", printed) - self.assertIn("ls /", printed) + self.assertIn("tree / -L 1", printed) self.assertIn("[pifs -> llm result preview]", printed) self.assertIn('{"ok": true}', printed) - self.assertEqual(stream_log[0], {"kind": "tool_call", "command": "ls /"}) + self.assertEqual( + stream_log[0], {"kind": "tool_call", "command": "tree / -L 1"} + ) self.assertEqual(stream_log[1]["kind"], "tool_result") - self.assertEqual(stream_log[2], {"kind": "tool_args", "text": '{"command":"ls /"}'}) + self.assertEqual( + stream_log[2], + {"kind": "tool_args", "text": '{"command":"tree / -L 1"}'}, + ) def test_empty_tool_command_is_not_printed_or_logged(self): output = io.StringIO() @@ -258,7 +246,7 @@ def test_prompt_routes_topic_retrieval_through_browse_after_folder_exploration(s self.assertIn("verify the relevant facts with cat or grep", AGENT_TOOL_POLICY) self.assertIn("cat --structure", AGENT_TOOL_POLICY) self.assertIn("cat --page", AGENT_TOOL_POLICY) - self.assertIn("Use grep for a selected single file", AGENT_TOOL_POLICY) + self.assertIn("Use grep for a selected single file", AGENT_TOOL_POLICY) self.assertIn("Use grep only as a single-document lexical fallback", BASH_TOOL_DESCRIPTION) def test_prompt_allows_recursive_browse_for_low_signal_tree_labels(self): @@ -283,20 +271,9 @@ def test_default_agent_prompts_do_not_suggest_legacy_semantic_commands(self): ): self.assertNotIn(old_command, prompt_surface) - def test_demo_prompt_uses_browse_strategy_and_not_old_vector_commands(self): - demo_prompt = load_demo_agent_prompt() - - self.assertIn("Start with tree", demo_prompt) - self.assertIn('browse /documents "Federal Reserve supervision regulation"', demo_prompt) - self.assertIn('browse -R /documents "Federal Reserve supervision regulation"', demo_prompt) - self.assertIn("verify", demo_prompt) - self.assertIn("cat --structure", demo_prompt) - self.assertIn("Use grep only for one selected file", demo_prompt) - self.assertIn("Do not guess cat --page ranges from grep line numbers", demo_prompt) - self.assertNotIn("search-summary", demo_prompt) - def test_prompt_rejects_find_grep_as_exhaustive_search(self): - self.assertIn("Do not use find, recursive grep, folder grep, pipes", AGENT_TOOL_POLICY) + self.assertIn("folder-wide lexical searches", AGENT_TOOL_POLICY) + self.assertIn("Other shell commands, pipes", AGENT_TOOL_POLICY) self.assertIn("Only after that persistence protocol may you say the workspace lacks evidence", AGENT_TOOL_POLICY) def test_system_prompt_sets_workspace_identity_and_scope(self): diff --git a/tests/test_pifs_cli.py b/tests/test_pifs_cli.py index d6f5b0a5d..5453504b3 100644 --- a/tests/test_pifs_cli.py +++ b/tests/test_pifs_cli.py @@ -37,16 +37,12 @@ def test_pifs_config_file_overrides_default_location(monkeypatch, tmp_path): class FakeFileSystem: - def __init__(self, workspace): + def __init__(self, workspace, **kwargs): self.workspace = Path(workspace) - self.projection_retrieval_configured = False + self.kwargs = kwargs - def configure_existing_projection_retrieval(self): - self.projection_retrieval_configured = True - return True - -def test_cli_workspace_does_not_eagerly_configure_projection_retrieval(monkeypatch, tmp_path): +def test_cli_workspace_does_not_eagerly_open_projection(monkeypatch, tmp_path): from pageindex.filesystem import cli workspace = tmp_path / "workspace" @@ -56,7 +52,7 @@ def test_cli_workspace_does_not_eagerly_configure_projection_retrieval(monkeypat filesystem = cli._filesystem_from_workspace(str(workspace)) assert filesystem.workspace == workspace - assert filesystem.projection_retrieval_configured is False + assert filesystem.kwargs.get("summary_projection_embedding_model") is None def test_cli_workspace_without_projection_index_does_not_require_sqlite_vec( @@ -81,7 +77,6 @@ def block_sqlite_vec(name, globals=None, locals=None, fromlist=(), level=0): filesystem = cli._filesystem_from_workspace(str(workspace)) assert filesystem.workspace == workspace - assert filesystem.semantic_retrieval_channels() == () def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): @@ -94,12 +89,10 @@ def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): json.dumps( { "workspace": str(workspace), - "embedding_provider": "openai", "embedding_model": "gemini-embedding-2-preview", "embedding_dimensions": 3072, "embedding_timeout": 12.5, "embedding_base_url": "https://example.invalid/openai/", - "embedding_api_key": "test-gemini-key", } ), encoding="utf-8", @@ -118,59 +111,16 @@ def __init__(self, workspace, **kwargs): assert filesystem.workspace == workspace assert filesystem.kwargs == { - "summary_projection_embedding_provider": "openai", "summary_projection_embedding_model": "gemini-embedding-2-preview", "summary_projection_embedding_dimensions": 3072, "summary_projection_embedding_timeout": 12.5, "summary_projection_embedding_base_url": "https://example.invalid/openai/", - "summary_projection_embedding_api_key": "test-gemini-key", + "summary_projection_embedding_api_key": "ignored-env-key", } assert os.environ["PIFS_EMBEDDING_API_KEY"] == "ignored-env-key" assert os.environ["PIFS_EMBEDDING_BASE_URL"] == "https://ignored.invalid/" -def test_browse_surfaces_projection_dimension_mismatch_lazily(tmp_path): - from pageindex.filesystem import cli - from pageindex.filesystem.commands import PIFSCommandExecutor - from pageindex.filesystem.semantic_index import SemanticIndexRecord, SQLiteVecSemanticIndex - - workspace = tmp_path / "workspace" - index_dir = workspace / "artifacts" / "projection_indexes" - summary_index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - summary_index.reset( - dimension=3, - metadata={ - "channel": "summary", - "embedding_provider": "test", - "embedding_model": "fake", - "embedding_dimensions": 3, - }, - ) - summary_index.upsert_many( - [ - SemanticIndexRecord( - file_ref="file_a", - external_id="doc_a", - source_type="documents", - title="A", - text="summary", - vector=[1.0, 0.0, 0.0], - ) - ] - ) - - filesystem = cli._filesystem_from_workspace(str(workspace)) - - assert filesystem.semantic_retrieval_channels() == () - result = json.loads(PIFSCommandExecutor(filesystem).execute('browse / "summary"')) - - assert result["success"] is False - assert result["error"]["code"] == "invalid_command" - assert "summary projection index dimension mismatch" in result["error"]["message"] - assert "dimension 3" in result["error"]["message"] - assert "summary_projection_embedding_dimensions is 1024" in result["error"]["message"] - - def test_cli_passthrough_invokes_pifs_command_executor(monkeypatch, capsys, tmp_path): from pageindex.filesystem import cli @@ -190,13 +140,13 @@ def execute(self, command): monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) - status = cli.main(["--workspace", str(workspace), "ls", "/documents"]) + status = cli.main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) assert status == 0 - assert capsys.readouterr().out == "executed:ls /documents\n" + assert capsys.readouterr().out == "executed:tree /documents -L 1\n" assert len(executor_instances) == 1 assert executor_instances[0].filesystem.workspace == workspace - assert executor_instances[0].commands == ["ls /documents"] + assert executor_instances[0].commands == ["tree /documents -L 1"] def test_cli_passthrough_returns_nonzero_for_failed_json_envelope(monkeypatch, capsys, tmp_path): @@ -271,9 +221,12 @@ def set_metadata(self, target, metadata, *, clear=False): (workspace, "/documents/report.md", {"ticker": "AAPL"}, False) ] assert json.loads(capsys.readouterr().out) == { - "file_ref": "file_report", + "document_id": None, "metadata": {"ticker": "AAPL"}, + "metadata_status": {}, "path": "/documents/report.md", + "status": None, + "title": None, } @@ -297,8 +250,12 @@ def set_metadata(self, target, metadata, *, clear=False): assert status == 0 assert calls == [(workspace, "/documents/report.md", {}, True)] assert json.loads(capsys.readouterr().out) == { - "file_ref": "file_report", + "document_id": None, "metadata": {}, + "metadata_status": {}, + "path": "/documents/report.md", + "status": None, + "title": None, } @@ -326,10 +283,10 @@ def execute(self, command): assert cli.main(["set", "workspace", str(workspace)]) == 0 capsys.readouterr() - status = cli.main(["ls", "/documents"]) + status = cli.main(["tree", "/documents", "-L", "1"]) assert status == 0 - assert capsys.readouterr().out == "executed:ls /documents\n" + assert capsys.readouterr().out == "executed:tree /documents -L 1\n" assert executor_instances[0].filesystem.workspace == workspace diff --git a/tests/test_pifs_core_alignment.py b/tests/test_pifs_core_alignment.py deleted file mode 100644 index 6c22fa92c..000000000 --- a/tests/test_pifs_core_alignment.py +++ /dev/null @@ -1,356 +0,0 @@ -import json -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace - -from tests.pifs_markdown_fixture import register_markdown - - -class BrowseBackend: - def __init__(self, document_ids, channels=("summary",), file_refs_by_document_id=None): - self.document_ids = list(document_ids) - self.channels = channels - self.file_refs_by_document_id = dict(file_refs_by_document_id or {}) - self.calls = [] - - def available_channels(self): - return self.channels - - def search_channel(self, channel, query, *, limit=10, filters=None): - self.calls.append((channel, query, limit, filters)) - file_ref_filter = set() - if isinstance(filters, dict): - raw_file_refs = filters.get("file_ref") or [] - if isinstance(raw_file_refs, str): - file_ref_filter = {raw_file_refs} - else: - file_ref_filter = {str(item) for item in raw_file_refs} - document_ids = self.document_ids - if file_ref_filter and self.file_refs_by_document_id: - document_ids = [ - document_id - for document_id in document_ids - if self.file_refs_by_document_id.get(document_id) in file_ref_filter - ] - return [ - SimpleNamespace( - document_id=document_id, - snippet=f"{channel} candidate {rank}: {query}", - score=1.0 - rank * 0.01, - sources=[{"channel": channel, "rank": rank}], - ) - for rank, document_id in enumerate(document_ids[:limit], 1) - ] - - -def _payload(output): - return json.loads(output) - - -def _register_file(filesystem, root, external_id, folder_path, *, title=None, text=None): - return register_markdown( - filesystem, - root, - external_id, - folder_path, - title=title or f"{external_id}.md", - text=text, - metadata={"department": "finance"}, - ) - - -class PIFSCoreAlignmentTest(unittest.TestCase): - def test_aligned_command_surface_and_json_errors(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file(filesystem, root, "doc_a", "/documents") - executor = PIFSCommandExecutor(filesystem) - - self.assertEqual( - executor.allowed_commands(), - {"browse", "cat", "grep", "ls", "stat", "tree"}, - ) - - for command in ( - "find /documents", - "grep -R alpha /documents", - "grep alpha /documents", - "stat --schema", - "stat --field department doc_a", - "stat doc_a doc_a", - "cat doc_a", - "cat doc_a --all", - "tree /documents | grep doc", - 'browse /documents "alpha" --space entity', - 'browse /documents "alpha" --json', - ): - result = _payload(executor.execute(command)) - self.assertFalse(result["success"]) - self.assertEqual(result["error"]["code"], "invalid_command") - self.assertEqual(result["next_steps"], []) - - def test_ls_is_tree_depth_one_alias(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file(filesystem, root, "doc_root", "/documents") - _register_file(filesystem, root, "doc_child", "/documents/team") - executor = PIFSCommandExecutor(filesystem) - calls = [] - original = executor._cmd_tree - - def record_tree(args): - calls.append(args) - return original(args) - - executor._cmd_tree = record_tree - - ls_result = _payload(executor.execute("ls /documents")) - tree_result = _payload(executor.execute("tree /documents -L 1")) - - self.assertEqual(calls[0], ["/documents", "-L", "1"]) - self.assertEqual(ls_result, tree_result) - self.assertTrue(ls_result["success"]) - self.assertEqual( - list(ls_result["data"]), - ["tree", "total_folders", "depth", "truncated"], - ) - self.assertEqual(ls_result["data"]["depth"], 1) - - def test_tree_defaults_to_chat_aligned_depth_ten_without_clamping_explicit_depth(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file(filesystem, root, "doc_root", "/documents") - executor = PIFSCommandExecutor(filesystem) - - default_tree = _payload(executor.execute("tree /documents")) - explicit_tree = _payload(executor.execute("tree /documents -L 99")) - - self.assertTrue(default_tree["success"]) - self.assertEqual(default_tree["data"]["depth"], 10) - self.assertTrue(explicit_tree["success"]) - self.assertEqual(explicit_tree["data"]["depth"], 99) - - def test_browse_summary_scoped_and_paginated(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_direct": _register_file(filesystem, root, "doc_direct", "/documents"), - "doc_deep": _register_file(filesystem, root, "doc_deep", "/documents/reports"), - "doc_other": _register_file(filesystem, root, "doc_other", "/other"), - } - backend = BrowseBackend( - ["doc_other", "doc_deep", "doc_direct"], - file_refs_by_document_id=refs, - ) - filesystem.semantic_retrieval_backend = backend - executor = PIFSCommandExecutor(filesystem) - - missing_query = _payload(executor.execute("browse /documents")) - self.assertFalse(missing_query["success"]) - self.assertIn("requires a query", missing_query["error"]["message"]) - - removed_space = _payload( - executor.execute('browse /documents "alpha" --space entity') - ) - self.assertFalse(removed_space["success"]) - self.assertIn("--space is removed", removed_space["error"]["message"]) - - direct = _payload(executor.execute('browse /documents "alpha"')) - self.assertTrue(direct["success"]) - self.assertEqual( - set(direct["data"]["documents"][0]), - {"path", "title", "summary", "metadata"}, - ) - self.assertEqual( - [doc["path"] for doc in direct["data"]["documents"]], - ["/documents/doc_direct.md"], - ) - self.assertEqual( - direct["data"]["scope"], - { - "folder": "/documents", - "recursive": False, - "query": "alpha", - "where": None, - "retrieval": "summary", - }, - ) - self.assertEqual(backend.calls[-1][0], "summary") - - recursive = _payload(executor.execute('browse -R /documents "alpha"')) - self.assertEqual( - [doc["path"] for doc in recursive["data"]["documents"]], - ["/documents/reports/doc_deep.md", "/documents/doc_direct.md"], - ) - self.assertTrue(recursive["data"]["scope"]["recursive"]) - - def test_stat_and_grep_are_single_document_only(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file( - filesystem, - root, - "doc_a", - "/documents", - title="report.md", - text="alpha evidence\nbeta evidence", - ) - executor = PIFSCommandExecutor(filesystem) - - stat = _payload(executor.execute("stat /documents/report.md")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["document_id"], "doc_a") - self.assertTrue(stat["data"]["document"]["file_ref"].startswith("file_")) - - grep = _payload(executor.execute("grep alpha /documents/report.md")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["document_id"], "doc_a") - self.assertEqual( - grep["data"]["matches"], - [{"line": 1, "text": "alpha evidence"}], - ) - - folder_grep = _payload(executor.execute("grep alpha /documents")) - self.assertFalse(folder_grep["success"]) - self.assertIn("not a folder", folder_grep["error"]["message"]) - - def test_cat_structure_and_page_use_aligned_json_shapes(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - class FakePageClient: - documents = { - "pi_doc": { - "doc_description": "Summary for report.md", - "pages": [{"page": 1, "content": "cached text"}], - } - } - - def index(self, file_path, mode="auto"): - return "pi_doc" - - def _ensure_doc_loaded(self, doc_id): - return None - - def get_document_structure(self, doc_id): - assert doc_id == "pi_doc" - return json.dumps( - [{"title": "Risk", "node_id": "0001", "text": "hidden", "nodes": []}] - ) - - def get_page_content(self, doc_id, pages): - assert doc_id == "pi_doc" - assert pages == "1-2" - return json.dumps( - [ - {"page": 1, "content": "page one evidence"}, - {"page": 2, "content": "page two evidence"}, - ] - ) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - source = root / "report.md" - source.write_text("cached text", encoding="utf-8") - filesystem = PageIndexFileSystem(workspace=root / "workspace") - filesystem._pageindex_client = lambda: FakePageClient() - file_ref = filesystem.register_file( - storage_uri=source.as_uri(), - folder_path="/documents", - external_id="doc_pdf", - title="report.md", - content_type="text/markdown", - content="cached text", - ) - filesystem.store.update_pageindex_pointer( - file_ref, - pageindex_doc_id="pi_doc", - pageindex_tree_status="built", - ) - executor = PIFSCommandExecutor(filesystem) - - structure = _payload(executor.execute("cat doc_pdf --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["document_id"], "doc_pdf") - self.assertEqual( - structure["data"]["structure"], - [{"title": "Risk", "node_id": "0001", "nodes": []}], - ) - self.assertEqual(structure["data"]["pagination"], {"available": True}) - - page = _payload(executor.execute("cat doc_pdf --page 1-2")) - self.assertTrue(page["success"]) - self.assertEqual(page["data"]["requested_pages"], "1-2") - self.assertEqual( - [item["page"] for item in page["data"]["returned_pages"]], - [1, 2], - ) - self.assertEqual( - page["data"]["content"], - { - "text": "page one evidence\n\npage two evidence", - "available": True, - }, - ) - - def test_agent_policy_mentions_only_aligned_surface(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - from pageindex.filesystem.agent import build_pifs_agent_instructions - - with tempfile.TemporaryDirectory() as tmp: - filesystem = PageIndexFileSystem(workspace=Path(tmp) / "workspace") - executor = PIFSCommandExecutor(filesystem) - - instructions = build_pifs_agent_instructions(filesystem, executor=executor) - - for expected in ("tree", "browse", "stat", "cat", "grep"): - self.assertIn(expected, instructions) - for removed in ( - "find --", - "grep -R", - "stat --schema", - "stat --field", - "cat --all", - "cat --range", - ): - self.assertNotIn(removed, instructions) - self.assertIn( - "run cat --structure before the first cat --page", - instructions, - ) - self.assertIn("metadata-derived virtual axis", instructions) - self.assertIn("metadata-derived virtual value", instructions) - - def test_projection_surface_is_summary(self): - from pageindex.filesystem.core import ( - SEMANTIC_PROJECTION_INDEX_NAMES, - SEMANTIC_RETRIEVAL_CHANNELS, - ) - semantic_projection_source = Path("pageindex/filesystem/semantic_projection.py").read_text( - encoding="utf-8" - ) - - self.assertEqual(SEMANTIC_RETRIEVAL_CHANNELS, ("summary",)) - self.assertEqual(SEMANTIC_PROJECTION_INDEX_NAMES, {"summary": "summary"}) - self.assertIn('SUMMARY_INDEX_NAME = "summary"', semantic_projection_source) - self.assertNotIn("entity_vectors", semantic_projection_source) - self.assertNotIn("relation_vectors", semantic_projection_source) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pifs_like_escape.py b/tests/test_pifs_like_escape.py index 304bad646..262306c70 100644 --- a/tests/test_pifs_like_escape.py +++ b/tests/test_pifs_like_escape.py @@ -42,20 +42,16 @@ def test_descendant_folder_filter_treats_underscore_literally(tmp_path): external_id="wildcard_neighbor", ) - recursive = filesystem.browse("/proj_1", recursive=True, limit=10) - folder_id = filesystem.folder_info("/proj_1")["folder_id"] - scoped_results = filesystem.search( - scope={"folder_id": folder_id, "recursive": True}, + recursive = filesystem.store.list_folder("/proj_1", recursive=True, limit=10) + scoped_results = filesystem.store.list_files( + scope={"folder_path": "/proj_1", "recursive": True}, limit=10, ) - ranked_folders = { - folder["path"]: folder - for folder in filesystem.find_folders("/", max_depth=1, limit=10) - } + ranked_folders = {folder["path"]: folder for folder in filesystem.store.find_folders("/", max_depth=1, limit=10)} assert {folder["path"] for folder in recursive["folders"]} == {"/proj_1/docs"} assert {file["external_id"] for file in recursive["files"]} == {"literal_underscore"} - assert {result.external_id for result in scoped_results} == {"literal_underscore"} + assert {result["external_id"] for result in scoped_results} == {"literal_underscore"} assert ranked_folders["/proj_1"]["matched_files"] == 1 assert ranked_folders["/projA1"]["matched_files"] == 1 assert filesystem.store.count_files_in_folder("/proj_1", recursive=True) == 1 @@ -99,14 +95,14 @@ def test_metadata_contains_treats_percent_and_underscore_literally(tmp_path): metadata={"status": "buildXalpha"}, ) - percent_results = filesystem.search( + percent_results = filesystem.store.list_files( metadata_filter={"status": {"$contains": "100% done"}}, limit=10, ) - underscore_results = filesystem.search( + underscore_results = filesystem.store.list_files( metadata_filter={"status": {"$contains": "build_alpha"}}, limit=10, ) - assert {result.external_id for result in percent_results} == {"literal_percent"} - assert {result.external_id for result in underscore_results} == {"literal_underscore"} + assert {result["external_id"] for result in percent_results} == {"literal_percent"} + assert {result["external_id"] for result in underscore_results} == {"literal_underscore"} diff --git a/tests/test_pifs_register_side_effects.py b/tests/test_pifs_register_side_effects.py deleted file mode 100644 index 6d2b8a65b..000000000 --- a/tests/test_pifs_register_side_effects.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import Path - -import pytest - -from tests.pifs_markdown_fixture import register_markdown - - -class RecordingSummaryIndexer: - def __init__(self): - self.upserted = [] - self.deleted = [] - - def upsert_summary(self, record): - self.upserted.append(dict(record)) - return {"status": "ready"} - - def delete_summary(self, file_ref): - self.deleted.append(file_ref) - - -def test_register_insert_failure_cleans_owned_artifacts_and_skips_projection( - tmp_path: Path, monkeypatch -): - from pageindex.filesystem import PageIndexFileSystem - - workspace = tmp_path / "workspace" - indexer = RecordingSummaryIndexer() - filesystem = PageIndexFileSystem(workspace=workspace) - filesystem.summary_projection_indexer = indexer - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - register_markdown( - filesystem, - tmp_path, - "doc_insert_failure", - "/documents", - title="insert_failure.md", - text="Markdown content for registration.", - ) - - assert indexer.upserted == [] - assert list((workspace / "artifacts" / "raw").glob("*.json")) == [] - assert list((workspace / "artifacts" / "text").glob("*.txt")) == [] - - -def test_register_failure_after_catalog_insert_cleans_catalog_and_projection( - tmp_path: Path, monkeypatch -): - from pageindex.filesystem import PageIndexFileSystem - - workspace = tmp_path / "workspace" - indexer = RecordingSummaryIndexer() - filesystem = PageIndexFileSystem(workspace=workspace) - filesystem.summary_projection_indexer = indexer - - def fail_sync(record): - raise RuntimeError("raw sync failed") - - monkeypatch.setattr(filesystem, "_sync_owned_raw_artifact", fail_sync) - - with pytest.raises(RuntimeError, match="raw sync failed"): - register_markdown( - filesystem, - tmp_path, - "doc_sync_failure", - "/documents", - title="sync_failure.md", - text="Markdown content for registration.", - ) - - assert filesystem.search(None) == [] - assert indexer.deleted == [indexer.upserted[0]["file_ref"]] diff --git a/tests/test_pifs_scope_paths.py b/tests/test_pifs_scope_paths.py deleted file mode 100644 index 0e021908a..000000000 --- a/tests/test_pifs_scope_paths.py +++ /dev/null @@ -1,711 +0,0 @@ -import json -import tempfile -import unittest -from types import SimpleNamespace - -from tests.pifs_markdown_fixture import register_markdown - - -class BrowseBackend: - def __init__(self, document_ids, *, file_refs_by_document_id=None): - self.document_ids = list(document_ids) - self.file_refs_by_document_id = dict(file_refs_by_document_id or {}) - - def available_channels(self): - return ("summary",) - - def search_channel(self, channel, query, *, limit=10, filters=None): - file_ref_filter = set((filters or {}).get("file_ref") or []) - document_ids = self.document_ids - if file_ref_filter: - document_ids = [ - document_id - for document_id in document_ids - if self.file_refs_by_document_id.get(document_id) in file_ref_filter - ] - return [ - SimpleNamespace( - document_id=document_id, - snippet=f"{channel} candidate {rank}: {query}", - score=1.0 - rank * 0.01, - sources=[{"channel": channel, "rank": rank}], - ) - for rank, document_id in enumerate(document_ids[:limit], 1) - ] - - -def _payload(output): - return json.loads(output) - - -class PIFSScopePathTest(unittest.TestCase): - def test_tree_browse_and_stat_accept_metadata_scope_paths(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_aapl_2024": register_markdown( - filesystem, - root, - "doc_aapl_2024", - "/documents/sec-filings", - title="aapl-2024.md", - metadata={"year": 2024, "ticker": "AAPL", "doc_type": "10-K"}, - ), - "doc_aapl_2023": register_markdown( - filesystem, - root, - "doc_aapl_2023", - "/documents/sec-filings", - title="aapl-2023.md", - metadata={"year": 2023, "ticker": "AAPL", "doc_type": "10-K"}, - ), - "doc_msft_2024": register_markdown( - filesystem, - root, - "doc_msft_2024", - "/documents/sec-filings", - title="msft-2024.md", - metadata={"year": 2024, "ticker": "MSFT", "doc_type": "10-K"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_msft_2024", "doc_aapl_2024", "doc_aapl_2023"], - file_refs_by_document_id=refs, - ) - executor = PIFSCommandExecutor(filesystem) - - tree_root = _payload(executor.execute("tree /documents -L 1")) - self.assertTrue(tree_root["success"]) - - root_folders = tree_root["data"]["tree"]["folders"] - self.assertTrue(any(item["path"] == "/documents/sec-filings" for item in root_folders)) - self.assertEqual( - [item["name"] for item in root_folders if item["type"] == "metadata_axis"], - ["@doc_type", "@ticker", "@year"], - ) - - tree_year = _payload(executor.execute("tree /documents/@year")) - self.assertTrue(tree_year["success"]) - self.assertEqual( - [item["path"] for item in tree_year["data"]["tree"]["folders"]], - ["/documents/@year/2024", "/documents/@year/2023"], - ) - - browse = _payload( - executor.execute('browse /documents/@year/2024/@ticker/AAPL "risk factors"') - ) - self.assertTrue(browse["success"]) - self.assertEqual( - [item["path"] for item in browse["data"]["documents"]], - ["/documents/@year/2024/@ticker/AAPL/aapl-2024.md"], - ) - self.assertEqual(browse["data"]["scope"]["path"], "/documents/@year/2024/@ticker/AAPL") - self.assertEqual(browse["data"]["scope"]["folder_path"], "/documents") - self.assertEqual( - browse["data"]["scope"]["metadata_filter"], - {"year": "2024", "ticker": "AAPL"}, - ) - - locator = browse["data"]["documents"][0]["path"] - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - self.assertEqual(stat["data"]["document"]["title"], "aapl-2024.md") - - def test_metadata_scope_file_locators_round_trip_to_file_commands(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_3m_2018": register_markdown( - filesystem, - root, - "doc_3m_2018", - "/docs/filings", - title="3M_2018_10K.md", - text="cash flow evidence\nnet sales evidence", - metadata={"company": "3M"}, - ), - "doc_other": register_markdown( - filesystem, - root, - "doc_other", - "/docs/filings", - title="Other_2018_10K.md", - text="other cash flow", - metadata={"company": "Other"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_3m_2018", "doc_other"], - file_refs_by_document_id=refs, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - self.assertEqual( - [item["path"] for item in tree["data"]["tree"]["files"]], - ["/docs/@company/3M/3M_2018_10K.md"], - ) - - browse = _payload(executor.execute('browse /docs/@company/3M "cash flow"')) - self.assertTrue(browse["success"]) - locator = browse["data"]["documents"][0]["path"] - self.assertEqual(locator, "/docs/@company/3M/3M_2018_10K.md") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - self.assertEqual(stat["data"]["document"]["title"], "3M_2018_10K.md") - - structure = _payload(executor.execute(f"cat {locator} --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["path"], locator) - self.assertTrue(structure["data"]["structure"]) - - grep = _payload(executor.execute(f"grep cash {locator}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], locator) - self.assertEqual(grep["data"]["matches"][0]["line"], 1) - - scope_only = _payload(executor.execute("cat /docs/@company/3M --structure")) - self.assertFalse(scope_only["success"]) - self.assertIn("scope, not a file locator", scope_only["error"]["message"]) - self.assertIn("tree /docs/@company/3M", scope_only["error"]["message"]) - self.assertIn('browse /docs/@company/3M ""', scope_only["error"]["message"]) - - def test_tree_depth_does_not_bypass_file_pagination(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - for index in range(55): - register_markdown( - filesystem, - root, - f"doc_{index:02d}", - "/docs", - title=f"report-{index:02d}.md", - metadata={"company": "3M"}, - ) - executor = PIFSCommandExecutor(filesystem) - - first_page = _payload(executor.execute("tree /docs/@company/3M -L 100")) - self.assertTrue(first_page["success"]) - self.assertEqual(len(first_page["data"]["tree"]["files"]), 50) - self.assertEqual( - first_page["data"]["pagination"], - {"page": 1, "page_size": 50, "has_more": True, "next_page": 2}, - ) - - second_page = _payload(executor.execute("tree /docs/@company/3M -L 100 --page 2")) - self.assertTrue(second_page["success"]) - self.assertEqual(len(second_page["data"]["tree"]["files"]), 5) - self.assertEqual( - second_page["data"]["pagination"], - {"page": 2, "page_size": 50, "has_more": False, "next_page": None}, - ) - - def test_tree_file_pagination_sorts_full_scope_before_slicing(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - for index in range(55): - file_ref = register_markdown( - filesystem, - root, - f"doc_{index:02d}", - "/source", - title=f"source-{54 - index:02d}.md", - metadata={"company": "3M"}, - ) - filesystem.attach_file_to_folder( - file_ref, - "/docs", - metadata={"display_name": f"report-{index:02d}.md"}, - ) - executor = PIFSCommandExecutor(filesystem) - - first_page = _payload(executor.execute("tree /docs/@company/3M --page 1")) - second_page = _payload(executor.execute("tree /docs/@company/3M --page 2")) - - self.assertTrue(first_page["success"]) - self.assertTrue(second_page["success"]) - first_names = [item["name"] for item in first_page["data"]["tree"]["files"]] - second_names = [item["name"] for item in second_page["data"]["tree"]["files"]] - - self.assertEqual(first_names, [f"report-{index:02d}.md" for index in range(50)]) - self.assertEqual(second_names, [f"report-{index:02d}.md" for index in range(50, 55)]) - self.assertEqual(len(set(first_names + second_names)), 55) - - def test_duplicate_file_leaves_in_scope_return_disambiguated_locators(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_a": register_markdown( - filesystem, - root, - "doc_a", - "/docs/a", - title="report.md", - metadata={"company": "3M"}, - ), - "doc_b": register_markdown( - filesystem, - root, - "doc_b", - "/docs/b", - title="report.md", - metadata={"company": "3M"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_a", "doc_b"], - file_refs_by_document_id=refs, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - paths = [item["path"] for item in tree["data"]["tree"]["files"]] - self.assertEqual(len(paths), 2) - self.assertEqual(len(set(paths)), 2) - - for path in paths: - stat = _payload(executor.execute(f"stat {path}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], path) - - browse = _payload(executor.execute('browse /docs/@company/3M "report"')) - self.assertTrue(browse["success"]) - browse_paths = [item["path"] for item in browse["data"]["documents"]] - self.assertEqual(len(browse_paths), 2) - self.assertEqual(len(set(browse_paths)), 2) - - def test_disambiguated_file_leaf_does_not_collide_with_real_leaf(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - first_ref = register_markdown( - filesystem, - root, - "doc_first", - "/docs/a", - title="x.md", - text="first collision evidence", - metadata={"company": "3M"}, - ) - register_markdown( - filesystem, - root, - "doc_second", - "/docs/b", - title="x.md", - text="second collision evidence", - metadata={"company": "3M"}, - ) - register_markdown( - filesystem, - root, - "doc_real_suffix", - "/docs/c", - title=f"x.md~{first_ref}", - text="real suffix collision evidence", - metadata={"company": "3M"}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - paths = [item["path"] for item in tree["data"]["tree"]["files"]] - self.assertEqual(len(paths), 3) - self.assertEqual(len(set(paths)), 3) - - for path in paths: - stat = _payload(executor.execute(f"stat {path}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], path) - - grep = _payload(executor.execute(f"grep evidence {path}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], path) - - def test_metadata_scope_file_leaf_can_match_metadata_axis_name(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - register_markdown( - filesystem, - root, - "doc_axis_leaf", - "/docs", - title="@ticker", - text="axis leaf evidence", - metadata={"company": "3M", "ticker": "MMM"}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - locator = tree["data"]["tree"]["files"][0]["path"] - self.assertEqual(locator, "/docs/@company/3M/%40ticker") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - - structure = _payload(executor.execute(f"cat {locator} --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["path"], locator) - - grep = _payload(executor.execute(f"grep evidence {locator}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], locator) - self.assertEqual(grep["data"]["matches"][0]["line"], 1) - - def test_metadata_scope_file_leaf_with_slash_round_trips(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - file_ref = register_markdown( - filesystem, - root, - "doc_slash_leaf", - "/source", - title="source.md", - text="slash leaf evidence", - metadata={"company": "3M"}, - ) - filesystem.attach_file_to_folder( - file_ref, - "/docs", - metadata={"display_name": "2024/report.md"}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - locator = tree["data"]["tree"]["files"][0]["path"] - self.assertEqual(locator, "/docs/@company/3M/2024%2Freport.md") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - - structure = _payload(executor.execute(f"cat {locator} --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["path"], locator) - - grep = _payload(executor.execute(f"grep evidence {locator}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], locator) - self.assertEqual(grep["data"]["matches"][0]["line"], 1) - - def test_scope_paths_reject_duplicate_and_unknown_metadata_fields(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - register_markdown( - filesystem, - root, - "doc_aapl_2024", - "/documents", - metadata={"year": 2024, "ticker": "AAPL"}, - ) - executor = PIFSCommandExecutor(filesystem) - - duplicate = _payload(executor.execute("tree /documents/@ticker/AAPL/@ticker/MSFT")) - self.assertFalse(duplicate["success"]) - self.assertIn("can appear only once", duplicate["error"]["message"]) - - unknown = _payload(executor.execute("tree /documents/@sector")) - self.assertFalse(unknown["success"]) - self.assertIn("Unknown metadata axis", unknown["error"]["message"]) - - def test_tree_metadata_value_pagination_uses_page_flag(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - for index in range(55): - ticker = f"T{index:02d}" - register_markdown( - filesystem, - root, - f"doc_{ticker}", - "/documents", - title=f"{ticker}.md", - metadata={"ticker": ticker}, - ) - - executor = PIFSCommandExecutor(filesystem) - - first_page = _payload(executor.execute("tree /documents/@ticker")) - self.assertTrue(first_page["success"]) - self.assertEqual(len(first_page["data"]["tree"]["folders"]), 50) - self.assertEqual( - first_page["data"]["pagination"], - {"page": 1, "page_size": 50, "has_more": True, "next_page": 2}, - ) - self.assertEqual( - [item["value"] for item in first_page["data"]["tree"]["folders"][:3]], - ["T00", "T01", "T02"], - ) - - second_page = _payload(executor.execute("tree /documents/@ticker --page 2")) - self.assertTrue(second_page["success"]) - self.assertEqual( - [item["value"] for item in second_page["data"]["tree"]["folders"]], - ["T50", "T51", "T52", "T53", "T54"], - ) - self.assertEqual( - second_page["data"]["pagination"], - {"page": 2, "page_size": 50, "has_more": False, "next_page": None}, - ) - - def test_tree_child_paths_keep_active_metadata_scope(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - register_markdown( - filesystem, - root, - "doc_sec_2024", - "/documents/sec", - title="sec-2024.md", - metadata={"year": 2024, "ticker": "AAPL"}, - ) - register_markdown( - filesystem, - root, - "doc_sec_2023", - "/documents/sec", - title="sec-2023.md", - metadata={"year": 2023, "ticker": "AAPL"}, - ) - register_markdown( - filesystem, - root, - "doc_other_2024", - "/documents/other", - title="other-2024.md", - metadata={"year": 2024, "ticker": "MSFT"}, - ) - - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /documents/@year/2024 -L 1")) - self.assertTrue(tree["success"]) - sec_node = next( - item - for item in tree["data"]["tree"]["folders"] - if item["name"] == "sec" - ) - self.assertEqual(sec_node["path"], "/documents/sec/@year/2024") - self.assertEqual(sec_node["file_count"], 1) - - scope_stat = _payload(executor.execute("stat /documents/sec/@year/2024")) - self.assertFalse(scope_stat["success"]) - self.assertIn("scope, not a file locator", scope_stat["error"]["message"]) - - scoped_tree = _payload(executor.execute("tree /documents/sec/@year/2024")) - self.assertTrue(scoped_tree["success"]) - locator = scoped_tree["data"]["tree"]["files"][0]["path"] - self.assertEqual(locator, "/documents/sec/@year/2024/sec-2024.md") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - - def test_core_browse_semantic_files_accepts_metadata_scoped_path(self): - from pageindex.filesystem import PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_aapl": register_markdown( - filesystem, - root, - "doc_aapl", - "/documents/sec", - title="aapl.md", - metadata={"ticker": "AAPL"}, - ), - "doc_msft": register_markdown( - filesystem, - root, - "doc_msft_direct", - "/documents", - title="msft-direct.md", - metadata={"ticker": "MSFT"}, - ), - "doc_msft_descendant": register_markdown( - filesystem, - root, - "doc_msft_descendant", - "/documents/sec", - title="msft-descendant.md", - metadata={"ticker": "MSFT"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_msft_direct", "doc_msft_descendant", "doc_aapl"], - file_refs_by_document_id=refs, - ) - - payload = filesystem.browse_semantic_files( - "/documents/@ticker/AAPL", - "query", - recursive=False, - ) - - self.assertEqual( - [item["document_id"] for item in payload["data"]], - ["doc_aapl"], - ) - self.assertEqual(payload["scope"], "/documents/@ticker/AAPL") - - def test_core_browse_semantic_files_prefers_in_scope_physical_membership(self): - from pageindex.filesystem import PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - file_ref = register_markdown( - filesystem, - root, - "doc_shared", - "/adocs", - title="shared.md", - metadata={"ticker": "AAPL"}, - ) - filesystem.store.attach_file_to_folder(file_ref, "/zdocs") - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_shared"], - file_refs_by_document_id={"doc_shared": file_ref}, - ) - - payload = filesystem.browse_semantic_files( - "/zdocs/@ticker/AAPL", - "query", - recursive=False, - ) - - self.assertEqual([item["document_id"] for item in payload["data"]], ["doc_shared"]) - self.assertEqual(payload["data"][0]["folder_path"], "/zdocs") - self.assertTrue(payload["data"][0]["path"].startswith("/zdocs/")) - - def test_one_document_is_reachable_through_multiple_metadata_virtual_paths(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - file_ref = register_markdown( - filesystem, - root, - "doc_contract", - "/documents/contracts", - title="apple-renewal.md", - metadata={"vendor": "Apple", "year": 2024, "doc_type": "contract"}, - ) - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_contract"], - file_refs_by_document_id={"doc_contract": file_ref}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree_root = _payload(executor.execute("tree /documents -L 1")) - self.assertTrue(tree_root["success"]) - root_nodes = tree_root["data"]["tree"]["folders"] - self.assertIn( - "/documents/@vendor", - [item["path"] for item in root_nodes if item["type"] == "metadata_axis"], - ) - - tree_vendor = _payload(executor.execute("tree /documents/@vendor")) - self.assertTrue(tree_vendor["success"]) - self.assertEqual( - [ - item["type"] - for item in tree_vendor["data"]["tree"]["folders"] - if item["path"] == "/documents/@vendor/Apple" - ], - ["metadata_value"], - ) - - vendor = _payload( - executor.execute('browse /documents/@vendor/Apple "renewal contract"') - ) - year = _payload(executor.execute('browse /documents/@year/2024 "renewal contract"')) - - self.assertTrue(vendor["success"]) - self.assertTrue(year["success"]) - self.assertEqual( - [item["path"] for item in vendor["data"]["documents"]], - ["/documents/@vendor/Apple/apple-renewal.md"], - ) - self.assertEqual( - [item["path"] for item in year["data"]["documents"]], - ["/documents/@year/2024/apple-renewal.md"], - ) - self.assertEqual( - vendor["data"]["scope"]["metadata_filter"], - {"vendor": "Apple"}, - ) - self.assertEqual( - year["data"]["scope"]["metadata_filter"], - {"year": "2024"}, - ) diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py new file mode 100644 index 000000000..6220fe8a3 --- /dev/null +++ b/tests/test_pifs_simplification_acceptance.py @@ -0,0 +1,1192 @@ +import json +import hashlib +import sqlite3 +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +class FakeOpenAI: + def __init__(self, **_kwargs): + self.embeddings = self + + def create(self, *, model, input, dimensions): + assert model == "test-embedding" + assert dimensions == 3 + + def vector(text): + lowered = str(text).lower() + if "alpha" in lowered: + return [1.0, 0.0, 0.0] + if "beta" in lowered: + return [0.0, 1.0, 0.0] + return [0.0, 0.0, 1.0] + + return SimpleNamespace( + data=[ + SimpleNamespace(index=index, embedding=vector(text)) + for index, text in enumerate(input) + ] + ) + + +def install_network_fakes(monkeypatch): + import openai + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + path = Path(file_path) + text = ( + path.read_text(encoding="utf-8") + if path.suffix.lower() in {".md", ".markdown"} + else "pdf alpha evidence" + ) + document_id = f"pageindex_{path.stem}" + document = { + "id": document_id, + "type": "pdf" if path.suffix.lower() == ".pdf" else "md", + "path": str(path.resolve()), + "doc_name": path.name, + "doc_description": f"Summary for {path.name}: {text}", + "line_count": len(text.splitlines()), + "structure": [ + { + "title": path.stem, + "node_id": "0001", + "line_num": 1, + "text": text, + "nodes": [], + } + ], + "pages": [{"page": 1, "content": text}], + } + self.documents[document_id] = document + self.workspace.mkdir(parents=True, exist_ok=True) + (self.workspace / f"{document_id}.json").write_text( + json.dumps(document), encoding="utf-8" + ) + (self.workspace / "_meta.json").write_text( + json.dumps( + { + document_id: { + "type": document["type"], + "doc_name": path.name, + "doc_description": document["doc_description"], + "path": str(path.resolve()), + "line_count": document["line_count"], + } + } + ), + encoding="utf-8", + ) + return document_id + + monkeypatch.setattr(openai, "OpenAI", FakeOpenAI) + monkeypatch.setattr(PageIndexClient, "index", fake_index) + + +def write_embedding_config(tmp_path, monkeypatch): + config = tmp_path / "pifs.json" + config.write_text( + json.dumps( + { + "embedding_base_url": "https://EXAMPLE.invalid/v1/", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + "embedding_timeout": 12, + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("PIFS_CONFIG_FILE", str(config)) + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "runtime-secret") + return config + + +def logical_tables(connection): + return { + row[1] + for row in connection.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} and not row[1].startswith("sqlite_") + and not row[1].startswith("semantic_index_vec_") + } + + +def catalog_file_count(workspace): + with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: + return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] + + +def test_cli_rejects_removed_ls_command_with_structured_error(tmp_path, capsys): + from pageindex.filesystem.cli import main + + status = main(["--workspace", str(tmp_path / "workspace"), "ls", "/"]) + + payload = json.loads(capsys.readouterr().out) + assert status == 2 + assert payload == { + "success": False, + "error": {"code": "invalid_command", "message": "Unsupported command: ls"}, + "next_steps": [], + } + + +def test_fresh_cli_workspace_uses_the_five_table_catalog_schema_v2(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + version = connection.execute("PRAGMA user_version").fetchone()[0] + tables = { + row[0] + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + + assert version == 2 + assert tables == { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", + } + + +def test_cli_rejects_legacy_catalog_without_mutating_it(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + workspace.mkdir() + database = workspace / "filesystem.sqlite" + with sqlite3.connect(database) as connection: + connection.execute("CREATE TABLE legacy_sentinel(value TEXT NOT NULL)") + connection.execute("INSERT INTO legacy_sentinel(value) VALUES ('preserve me')") + connection.execute("PRAGMA user_version = 1") + before = hashlib.sha256(database.read_bytes()).hexdigest() + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert hashlib.sha256(database.read_bytes()).hexdigest() == before + + +def test_cli_rejects_partial_legacy_projection_without_creating_summary( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = workspace / "artifacts" / "projection_indexes" + projection_dir.mkdir(parents=True) + cache_path = projection_dir / "embedding_cache.sqlite" + with sqlite3.connect(cache_path) as connection: + connection.execute( + "CREATE TABLE embedding_cache(provider TEXT, model TEXT, text_hash TEXT)" + ) + connection.execute("PRAGMA user_version = 1") + before = hashlib.sha256(cache_path.read_bytes()).hexdigest() + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert hashlib.sha256(cache_path.read_bytes()).hexdigest() == before + assert not (projection_dir / "summary.sqlite").exists() + assert catalog_file_count(workspace) == 0 + + +def test_pageindex_is_the_only_public_python_entry_point_for_pifs(): + import pageindex + import pageindex.filesystem as filesystem_module + + assert pageindex.PageIndexFileSystem is filesystem_module.PageIndexFileSystem + for internal_name in ( + "PIFSCommandExecutor", + "OpenResult", + "SearchResult", + "SummaryProjectionIndexer", + "SemanticProjectionSearchBackend", + "SQLiteVecSemanticIndex", + "SemanticIndexRecord", + "SemanticSearchResult", + ): + assert not hasattr(filesystem_module, internal_name) + assert not hasattr(filesystem_module, "_LAZY_EXPORTS") + + +def test_cli_add_creates_migration_compatible_summary_and_cache_v2( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + projection_dir = workspace / "artifacts" / "projection_indexes" + with sqlite3.connect(projection_dir / "summary.sqlite") as summary: + summary_version = summary.execute("PRAGMA user_version").fetchone()[0] + summary_tables = logical_tables(summary) + metadata = json.loads( + summary.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as cache: + cache_version = cache.execute("PRAGMA user_version").fetchone()[0] + cache_tables = logical_tables(cache) + cache_columns = { + row[1] for row in cache.execute("PRAGMA table_info(embedding_cache)") + } + + assert summary_version == cache_version == 2 + assert summary_tables == { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", + } + assert cache_tables == {"embedding_cache"} + assert cache_columns == { + "base_url", + "model", + "dimensions", + "text_hash", + "vector_blob", + "created_at", + } + assert metadata == { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 3, + } + for database in ( + workspace / "filesystem.sqlite", + projection_dir / "summary.sqlite", + projection_dir / "embedding_cache.sqlite", + ): + assert b"runtime-secret" not in database.read_bytes() + + +def test_cli_add_reports_path_without_persistence_identity(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + + assert ( + main( + [ + "--workspace", + str(tmp_path / "workspace"), + "add", + str(source), + "/documents", + ] + ) + == 0 + ) + + output = capsys.readouterr().out + assert output == "added: /documents/notes.md\n" + + +def test_cli_browse_reopens_owned_file_with_canonical_result_fields( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + assert ( + main( + [ + "--workspace", + str(workspace), + "browse", + "/documents", + "alpha", + ] + ) + == 0 + ) + + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is True + document = payload["data"]["documents"][0] + assert set(document) == { + "path", + "document_id", + "title", + "status", + "rank", + "similarity", + "summary", + "metadata", + "folder_path", + "folder_paths", + } + assert document["path"] == "/documents/notes.md" + assert document["title"] == "notes.md" + assert document["status"] == "built" + assert document["rank"] == 1 + assert document["summary"].startswith("Summary for notes.md") + assert document["folder_path"] == "/documents" + + +def test_cli_stat_translates_persistence_identity_once(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "stat", "/documents/notes.md"]) == 0 + + document = json.loads(capsys.readouterr().out)["data"]["document"] + assert set(document) == { + "path", + "document_id", + "title", + "status", + "content_type", + "metadata", + "metadata_status", + "folder_paths", + } + assert document["path"] == "/documents/notes.md" + assert document["title"] == "notes.md" + assert document["status"] == "built" + assert document["folder_paths"] == ["/documents"] + + +def test_cli_cat_reads_structure_without_internal_identity(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + assert ( + main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--structure", + ] + ) + == 0 + ) + + data = json.loads(capsys.readouterr().out)["data"] + assert set(data["document"]) == { + "path", + "document_id", + "title", + "status", + "content_type", + "metadata", + "metadata_status", + "folder_paths", + "available", + } + assert data["structure"] == [ + {"title": "notes", "node_id": "0001", "line_num": 1, "nodes": []} + ] + + +def test_tree_virtual_value_uses_one_axis_equals_value_segment( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert ( + main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) + == 0 + ) + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents/@year"]) == 0 + values = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"] + assert values[0]["path"] == "/documents/@year=2024" + + assert ( + main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@year=2024", + "-L", + "1", + ] + ) + == 0 + ) + files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + assert files[0]["path"] == "/documents/@year=2024/notes.md" + + +def test_setmeta_translates_identity_at_the_cli_boundary( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + document = json.loads(capsys.readouterr().out) + + assert set(document) == { + "path", + "document_id", + "title", + "status", + "metadata", + "metadata_status", + } + assert document["path"] == "/documents/notes.md" + assert document["metadata"]["year"] == 2024 + + +def test_tree_file_records_use_only_command_identity_names( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) == 0 + file_record = json.loads(capsys.readouterr().out)["data"]["tree"]["files"][0] + + assert set(file_record) == { + "path", + "document_id", + "title", + "status", + "type", + "metadata", + } + + +def test_duplicate_virtual_leaves_use_actionable_paths_without_internal_ids( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for folder_name in ("a", "b"): + source_dir = tmp_path / folder_name + source_dir.mkdir() + source = source_dir / "notes.md" + source.write_text(f"alpha evidence {folder_name}", encoding="utf-8") + folder = f"/documents/{folder_name}" + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@year=2024", "-L", "1"] + ) == 0 + files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + paths = [row["path"] for row in files] + + assert len(paths) == len(set(paths)) == 2 + assert all("file_" not in path for path in paths) + for path in paths: + assert main(["--workspace", str(workspace), "stat", path]) == 0 + assert json.loads(capsys.readouterr().out)["data"]["document"]["path"] == path + + +@pytest.mark.parametrize( + ("filename", "content"), + [ + ("notes.md", b"markdown alpha evidence"), + ("report.pdf", b"%PDF-1.4 fake fixture"), + ], +) +def test_cli_add_owns_supported_documents_and_keeps_evidence_readable( + filename, content, tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / filename + source.write_bytes(content) + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + owned = list((workspace / "artifacts" / "uploads").glob(f"*/{filename}")) + assert len(owned) == 1 + assert owned[0].read_bytes() == content + assert main( + ["--workspace", str(workspace), "grep", "alpha", f"/documents/{filename}"] + ) == 0 + matches = json.loads(capsys.readouterr().out)["data"]["matches"] + assert matches[0]["text"].endswith("alpha evidence") + + +def test_cli_add_rejects_unsupported_type_before_registration( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.txt" + source.write_text("unsupported", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "Unsupported file type" in capsys.readouterr().err + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + + +def test_cli_add_rolls_back_when_pageindex_fails( + tmp_path, monkeypatch, capsys +): + from pageindex import PageIndexClient + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + + def fail_index(self, file_path, mode="auto"): + raise RuntimeError("pageindex unavailable") + + monkeypatch.setattr(PageIndexClient, "index", fail_index) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "pageindex unavailable" in capsys.readouterr().err + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + pageindex_cache = workspace / "artifacts" / "pageindex_client" + assert json.loads((pageindex_cache / "_meta.json").read_text(encoding="utf-8")) == {} + assert not [path for path in pageindex_cache.glob("*.json") if path.name != "_meta.json"] + + +def test_cli_add_rolls_back_when_summary_projection_fails( + tmp_path, monkeypatch, capsys +): + import openai + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + + class FailingOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + raise RuntimeError("embedding unavailable") + + monkeypatch.setattr(openai, "OpenAI", FailingOpenAI) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "summary projection" in capsys.readouterr().err.lower() + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + with sqlite3.connect( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM semantic_index_docs" + ).fetchone()[0] == 0 + + +def test_cli_add_rejects_duplicate_target_without_changing_owned_document( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + first_dir.mkdir() + second_dir.mkdir() + first = first_dir / "notes.md" + second = second_dir / "notes.md" + first.write_text("first alpha evidence", encoding="utf-8") + second.write_text("second beta evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + command = ["--workspace", str(workspace), "add"] + + assert main([*command, str(first), "/documents"]) == 0 + capsys.readouterr() + assert main([*command, str(second), "/documents"]) == 1 + + assert "already exists" in capsys.readouterr().err + assert catalog_file_count(workspace) == 1 + owned = list((workspace / "artifacts" / "uploads").glob("*/notes.md")) + assert len(owned) == 1 + assert owned[0].read_text(encoding="utf-8") == "first alpha evidence" + + +def test_browse_recursively_returns_one_global_ranked_page_of_ten( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for index in range(12): + source = tmp_path / f"doc-{index:02d}.md" + relevance = "alpha" if index in {1, 10} else "beta" + source.write_text(f"{relevance} evidence {index}", encoding="utf-8") + folder = "/documents/a" if index % 2 == 0 else "/documents/b" + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + + command = [ + "--workspace", + str(workspace), + "browse", + "/documents", + "alpha", + "--recursive", + ] + assert main(command) == 0 + first = json.loads(capsys.readouterr().out)["data"] + assert len(first["documents"]) == 10 + assert first["pagination"] == { + "page": 1, + "page_size": 10, + "has_more": True, + "next_page": 2, + } + assert {row["title"] for row in first["documents"][:2]} == { + "doc-01.md", + "doc-10.md", + } + assert {row["folder_path"] for row in first["documents"]} == { + "/documents/a", + "/documents/b", + } + + assert main([*command, "--page", "2"]) == 0 + second = json.loads(capsys.readouterr().out)["data"] + assert len(second["documents"]) == 2 + assert second["pagination"] == { + "page": 2, + "page_size": 10, + "has_more": False, + "next_page": None, + } + assert not { + row["path"] for row in first["documents"] + }.intersection(row["path"] for row in second["documents"]) + + +def test_browse_respects_physical_and_virtual_scope( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + records = [ + ("current.md", "alpha current", "/documents/current", 2024), + ("old.md", "alpha old", "/documents/archive", 2023), + ] + for filename, content, folder, year in records: + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/{filename}", + json.dumps({"year": year}), + ] + ) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "browse", + "/documents/current", + "alpha", + ] + ) == 0 + physical = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [row["title"] for row in physical] == ["current.md"] + + assert main( + [ + "--workspace", + str(workspace), + "browse", + "/documents/@year=2024", + "alpha", + ] + ) == 0 + virtual = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [row["title"] for row in virtual] == ["current.md"] + assert virtual[0]["path"] == "/documents/@year=2024/current.md" + + +def test_browse_requires_query_and_an_existing_summary_projection( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "browse", "/documents"]) == 2 + missing_query = json.loads(capsys.readouterr().out) + assert "requires a query" in missing_query["error"]["message"] + + assert main(["--workspace", str(workspace), "browse", "/", "alpha"]) == 2 + missing_projection = json.loads(capsys.readouterr().out) + assert "Summary Projection is not available" in missing_projection["error"]["message"] + + +def test_browse_rejects_incompatible_embedding_identity_without_mutation( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + config = write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + databases = [ + workspace / "filesystem.sqlite", + workspace / "artifacts" / "projection_indexes" / "summary.sqlite", + workspace / "artifacts" / "projection_indexes" / "embedding_cache.sqlite", + ] + before = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases} + config.write_text( + json.dumps( + { + "embedding_base_url": "https://example.invalid/v1", + "embedding_model": "different-model", + "embedding_dimensions": 3, + } + ), + encoding="utf-8", + ) + + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 2 + + payload = json.loads(capsys.readouterr().out) + assert "Incompatible PIFS Summary Embedding Profile" in payload["error"]["message"] + assert { + path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases + } == before + + +def test_agent_prompts_describe_only_the_retained_command_surface(): + from pageindex.filesystem.agent import ( + AGENT_SYSTEM_PROMPT, + AGENT_TOOL_POLICY, + BASH_TOOL_DESCRIPTION, + ) + + prompts = "\n".join( + [AGENT_SYSTEM_PROMPT, BASH_TOOL_DESCRIPTION, AGENT_TOOL_POLICY] + ) + for command in ("tree", "browse", "stat", "cat", "grep"): + assert command in prompts + for retired_contract in ( + "ls as an alias", + "file_ref", + "--space", + "Do not use find", + "recursive grep", + ): + assert retired_contract not in prompts + + +def test_cat_page_reads_are_bounded_and_grep_is_single_document( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("line one\nalpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--page", + "1", + ] + ) == 0 + page = json.loads(capsys.readouterr().out)["data"] + assert page["requested_pages"] == "1" + assert page["content"]["text"] == "line one\nalpha evidence" + + assert main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--page", + "1-6", + ] + ) == 2 + too_wide = json.loads(capsys.readouterr().out) + assert "at most 5 pages" in too_wide["error"]["message"] + + assert main( + ["--workspace", str(workspace), "grep", "alpha", "/documents"] + ) == 2 + folder_grep = json.loads(capsys.readouterr().out) + assert "resolved file locator" in folder_grep["error"]["message"] + + +@pytest.mark.parametrize( + "command", + [ + ["ls", "/"], + ["find", "/"], + ["browse", "/", "alpha", "--space", "entity"], + ["browse", "/", "alpha", "--limit", "2"], + ["stat", "/", "--schema"], + ["stat", "/", "--field", "year"], + ["stat", "/one", "/two"], + ["cat", "/document"], + ["cat", "/document", "--all"], + ["cat", "/document", "--range", "1-2"], + ["grep", "alpha", "/document", "--recursive"], + ], +) +def test_retired_command_forms_return_structured_errors(command, tmp_path, capsys): + from pageindex.filesystem.cli import main + + status = main(["--workspace", str(tmp_path / "workspace"), *command]) + + payload = json.loads(capsys.readouterr().out) + assert status == 2 + assert payload["success"] is False + assert payload["error"]["code"] == "invalid_command" + + +def test_set_workspace_persists_runtime_config_without_credentials_or_provider( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + config = tmp_path / "pifs.json" + config.write_text( + json.dumps( + { + "embedding_base_url": "https://example.invalid/v1", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + "embedding_timeout": 12, + "embedding_api_key": "must-not-survive", + "embedding_provider": "legacy-provider", + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("PIFS_CONFIG_FILE", str(config)) + workspace = tmp_path / "workspace" + + assert main(["set", "workspace", str(workspace)]) == 0 + capsys.readouterr() + + assert json.loads(config.read_text(encoding="utf-8")) == { + "embedding_base_url": "https://example.invalid/v1", + "embedding_dimensions": "3", + "embedding_model": "test-embedding", + "embedding_timeout": "12", + "workspace": str(workspace), + } + + +def test_external_migration_fixture_reopens_without_reembedding( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "legacy-workspace" + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + + projection_dir = workspace / "artifacts" / "projection_indexes" + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute("CREATE TABLE metadata_schema(name TEXT)") + connection.execute("CREATE TABLE semantic_folders(folder_id TEXT)") + connection.execute("CREATE TABLE file_fts(file_ref TEXT, body TEXT)") + connection.execute("PRAGMA user_version = 1") + with sqlite3.connect(projection_dir / "summary.sqlite") as connection: + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + ( + json.dumps( + { + "embedding_provider": "openai", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + } + ), + ), + ) + connection.execute( + "CREATE INDEX idx_semantic_index_docs_file_ref " + "ON semantic_index_docs(file_ref)" + ) + connection.execute("PRAGMA user_version = 1") + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as connection: + connection.executescript( + """ + ALTER TABLE embedding_cache RENAME TO embedding_cache_v2; + CREATE TABLE embedding_cache ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + text_hash TEXT NOT NULL, + dimension INTEGER NOT NULL, + vector_blob BLOB, + vector_json TEXT, + created_at TEXT, + PRIMARY KEY(provider, model, text_hash) + ); + INSERT INTO embedding_cache( + provider, model, text_hash, dimension, + vector_blob, vector_json, created_at + ) + SELECT + 'openai', model || ':dimensions=' || dimensions, + text_hash, dimensions, vector_blob, NULL, created_at + FROM embedding_cache_v2; + DROP TABLE embedding_cache_v2; + PRAGMA user_version = 1; + """ + ) + + script = Path( + "/Users/chengjie/Projects/pifs-data/scripts/migrate_pifs_workspace.py" + ) + migration_command = [ + sys.executable, + str(script), + "--workspace", + str(workspace), + "--embedding-base-url", + "https://EXAMPLE.invalid/v1/", + "--embedding-model", + "test-embedding", + ] + migrated = subprocess.run( + migration_command, + check=False, + capture_output=True, + text=True, + ) + assert migrated.returncode == 0, migrated.stderr + report = json.loads(migrated.stdout) + assert report["status"] == "complete" + assert report["filesystem"]["files"] == 1 + assert report["summary"] == {"documents": 1, "vectors": 1, "dimensions": 3} + assert report["embedding_cache"] == {"rows": 1} + + databases = [ + workspace / "filesystem.sqlite", + projection_dir / "summary.sqlite", + projection_dir / "embedding_cache.sqlite", + ] + hashes = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases} + repeated = subprocess.run( + migration_command, + check=False, + capture_output=True, + text=True, + ) + assert repeated.returncode == 0, repeated.stderr + assert json.loads(repeated.stdout)["status"] == "already_migrated" + assert { + path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases + } == hashes + + with sqlite3.connect(databases[0]) as catalog: + assert logical_tables(catalog) == { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", + } + with sqlite3.connect(databases[1]) as summary: + assert summary.execute("PRAGMA user_version").fetchone()[0] == 2 + metadata = json.loads( + summary.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + assert metadata == { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 3, + } + with sqlite3.connect(databases[2]) as cache: + assert logical_tables(cache) == {"embedding_cache"} + assert cache.execute("SELECT COUNT(*) FROM embedding_cache").fetchone()[0] == 1 + + backup_roots = list((workspace / ".pifs-migration-backups").iterdir()) + assert len(backup_roots) == 1 + assert json.loads( + (backup_roots[0] / "migration.json").read_text(encoding="utf-8") + )["status"] == "complete" + assert not list(workspace.rglob("*.migrating-*")) + assert not list(workspace.rglob("*.rollback-*")) + + assert main(["--workspace", str(workspace), "tree", "/documents/@year=2024"]) == 0 + tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert tree["files"][0]["path"] == "/documents/@year=2024/notes.md" + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 0 + documents = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [document["title"] for document in documents] == ["notes.md"] + + +def test_public_example_is_a_short_supported_cli_walkthrough(): + example = Path(__file__).parents[1] / "examples" / "pifs_demo.py" + source = example.read_text(encoding="utf-8") + + assert len(source.splitlines()) <= 60 + for command in ("add", "tree", "browse", "stat", "cat", "grep"): + assert f'"{command}"' in source + for retired_surface in ( + "embedding_provider", + "metadata_provider", + "file_ref", + "SummaryProjectionIndexer", + "shutil.rmtree", + '"ls"', + ): + assert retired_surface not in source diff --git a/tests/test_pifs_string_metadata_cleanup.py b/tests/test_pifs_string_metadata_cleanup.py deleted file mode 100644 index e8df83f02..000000000 --- a/tests/test_pifs_string_metadata_cleanup.py +++ /dev/null @@ -1,189 +0,0 @@ -import json -from pathlib import Path - -import pytest - - -class StaticEmbedder: - def embed(self, texts): - return [[1.0, 0.0, 0.0] for _ in texts] - - -def _patch_summary_indexer(monkeypatch): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - def fake_from_provider(index_dir, **kwargs): - return SummaryProjectionIndexer( - Path(index_dir), - embedder=StaticEmbedder(), - embedding_provider="test", - embedding_model="static", - embedding_dimensions=3, - ) - - monkeypatch.setattr(SummaryProjectionIndexer, "from_provider", fake_from_provider) - - -def _patch_pageindex_client(monkeypatch, *, description="PageIndex summary"): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = Path(file_path).stem + "_doc" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": Path(file_path).name, - "doc_description": description, - "line_count": 2, - "structure": [{"title": "Doc", "node_id": "0001", "nodes": []}], - "pages": [{"page": 1, "content": Path(file_path).read_text(encoding="utf-8")}], - } - self.documents[doc_id] = doc - self._save_doc(doc_id) - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - - -def _register_md(filesystem, tmp_path, monkeypatch, name, metadata=None): - _patch_summary_indexer(monkeypatch) - _patch_pageindex_client(monkeypatch, description=f"summary for {name}") - path = tmp_path / name - path.write_text(f"# {name}\n\nbody", encoding="utf-8") - return filesystem.register_file( - storage_uri=path.as_uri(), - folder_path="/documents", - title=name, - content_type="text/markdown", - metadata=metadata or {}, - ) - - -def test_metadata_schema_is_string_registry_only(tmp_path): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - filesystem.metadata.register_schema( - {"fields": {"ticker": {"description": "stock ticker"}}} - ) - - assert filesystem.metadata.export_schema() == { - "fields": {"ticker": {"description": "stock ticker", "source": "manual"}} - } - with filesystem.store.connect() as conn: - tables = { - row["name"] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table'" - ).fetchall() - } - metadata_value_columns = { - row["name"] for row in conn.execute("PRAGMA table_info(metadata_values)") - } - - assert "metadata_schema" not in tables - assert metadata_value_columns == {"file_ref", "field_id", "value_text", "created_at"} - - -def test_range_filters_are_removed(tmp_path): - from pageindex.filesystem import PageIndexFileSystem - from pageindex.filesystem.metadata import MetadataQueryError - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - filesystem.metadata.register_schema({"fields": {"score": {}}}) - - with pytest.raises(MetadataQueryError, match="Unsupported metadata operator: \\$gte"): - filesystem.metadata.parse_filter({"score": {"$gte": 3}}) - - -def test_numeric_metadata_filter_is_string_equality(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - filesystem.metadata.register_schema({"fields": {"score": {}}}) - _register_md(filesystem, tmp_path, monkeypatch, "int.md", {"score": 3}) - _register_md(filesystem, tmp_path, monkeypatch, "float.md", {"score": 3.0}) - _register_md(filesystem, tmp_path, monkeypatch, "text.md", {"score": "3"}) - - rows = filesystem.search(None, metadata_filter={"score": {"$eq": 3}}) - - assert {row.title for row in rows} == {"int.md", "text.md"} - - -def test_register_uses_pageindex_description_as_summary_only(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - file_ref = _register_md(filesystem, tmp_path, monkeypatch, "summary.md", {"ticker": "AAPL"}) - info = filesystem.store.file_info(file_ref) - - assert info["metadata"]["summary"] == "summary for summary.md" - assert "summary" not in filesystem.metadata.export_schema()["fields"] - with filesystem.store.connect() as conn: - fields = [row["name"] for row in conn.execute("SELECT name FROM metadata_fields")] - assert fields == ["ticker"] - - -def test_register_rejects_plain_text_without_pageindex(tmp_path): - from pageindex.filesystem import PageIndexFileSystem - - source = tmp_path / "plain.txt" - source.write_text("plain text", encoding="utf-8") - filesystem = PageIndexFileSystem(tmp_path / "workspace") - - with pytest.raises(ValueError, match="PDF/Markdown"): - filesystem.register_file( - storage_uri=source.as_uri(), - folder_path="/documents", - title="plain.txt", - content_type="text/plain", - content="plain text", - ) - - -def test_set_metadata_replaces_custom_metadata_without_touching_summary(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - file_ref = _register_md(filesystem, tmp_path, monkeypatch, "meta.md", {"ticker": "AAPL"}) - - info = filesystem.set_metadata(file_ref, {"ticker": "MSFT", "year": 2026}) - - assert info["metadata"] == { - "summary": "summary for meta.md", - "ticker": "MSFT", - "year": 2026, - } - assert {field.name for field in filesystem.store.list_metadata_fields()} == { - "ticker", - "year", - } - assert filesystem.set_metadata(file_ref, {}, clear=True)["metadata"] == { - "summary": "summary for meta.md" - } - - -def test_set_metadata_keeps_summary_out_of_ordinary_lexical_search(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - unique_summary_token = "summaryleaktoken" - filesystem = PageIndexFileSystem(tmp_path / "workspace") - _patch_summary_indexer(monkeypatch) - _patch_pageindex_client(monkeypatch, description=unique_summary_token) - path = tmp_path / "leak.md" - path.write_text("# Leak\n\nordinary body", encoding="utf-8") - file_ref = filesystem.register_file( - storage_uri=path.as_uri(), - folder_path="/documents", - title="leak.md", - content_type="text/markdown", - metadata={"ticker": "AAPL"}, - ) - - assert filesystem.search(unique_summary_token) == [] - - filesystem.set_metadata(file_ref, {"ticker": "MSFT"}) - - assert filesystem.search(unique_summary_token) == [] - assert {row.title for row in filesystem.search("MSFT")} == {"leak.md"} diff --git a/tests/test_semantic_index.py b/tests/test_semantic_index.py index 2b277dc08..0346f60f9 100644 --- a/tests/test_semantic_index.py +++ b/tests/test_semantic_index.py @@ -1,19 +1,19 @@ -import sys -from pathlib import Path +import hashlib import pytest -REPO_ROOT = Path(__file__).resolve().parents[1] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - from pageindex.filesystem.semantic_index import ( SemanticIndexRecord, SQLiteVecSemanticIndex, ) +from pageindex.filesystem.semantic_projection import ( + EmbeddingCache, + SummaryEmbeddingProfile, + SummaryProjection, +) -class FixedDimensionEmbedder: +class FixedEmbedder: def __init__(self, dimensions: int): self.dimensions = dimensions @@ -23,14 +23,13 @@ def embed(self, texts): def test_sqlite_vec_semantic_index_round_trip(tmp_path): index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") - index.reset(dimension=3, metadata={"field_mode": "summary"}) - + index.reset(dimension=3, metadata={"kind": "summary"}) index.upsert_many( [ SemanticIndexRecord( file_ref="file_a", external_id="doc_a", - source_type="github", + source_type="documents", title="Multipart upload limits", text="multipart upload limits", vector=[1.0, 0.0, 0.0], @@ -39,7 +38,7 @@ def test_sqlite_vec_semantic_index_round_trip(tmp_path): SemanticIndexRecord( file_ref="file_b", external_id="doc_b", - source_type="slack", + source_type="documents", title="GPU cache issue", text="gpu cache issue", vector=[0.0, 1.0, 0.0], @@ -48,23 +47,14 @@ def test_sqlite_vec_semantic_index_round_trip(tmp_path): ] ) - assert index.info()["document_count"] == 2 - - results = index.search([0.9, 0.1, 0.0], limit=2) - assert [item.external_id for item in results] == ["doc_a", "doc_b"] - - filtered = index.search( - [0.9, 0.1, 0.0], - limit=2, - filters={"source_type": "slack"}, - ) - assert [item.external_id for item in filtered] == ["doc_b"] + assert [ + item.external_id for item in index.search([0.9, 0.1, 0.0], limit=2) + ] == ["doc_a", "doc_b"] -def test_sqlite_vec_semantic_index_file_ref_filter_not_limited_by_global_rank(tmp_path): +def test_sqlite_vec_file_ref_filter_is_not_limited_by_global_rank(tmp_path): index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") - index.reset(dimension=2, metadata={"field_mode": "summary"}) - + index.reset(dimension=2, metadata={"kind": "summary"}) records = [ SemanticIndexRecord( file_ref=f"file_off_{item:02d}", @@ -97,237 +87,108 @@ def test_sqlite_vec_semantic_index_file_ref_filter_not_limited_by_global_rank(tm assert [item.file_ref for item in results] == ["file_in_scope"] -def test_summary_projection_indexes_unified_metadata_summary(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - indexer = SummaryProjectionIndexer( - tmp_path / "projection", - embedder=FixedDimensionEmbedder(3), - embedding_provider="test", - embedding_model="fake", - embedding_dimensions=3, +def test_summary_projection_owns_write_search_and_delete_lifecycle(tmp_path): + profile = SummaryEmbeddingProfile( + base_url="https://EXAMPLE.invalid/v1/", + model="fake", + dimensions=3, + api_key="runtime-only", ) - - result = indexer.upsert_summary( - { - "file_ref": "file_a", - "external_id": "doc_a", - "source_type": "documents", - "title": "A", - "metadata": { - "summary": "Unified metadata summary.", - "department": "ops", - }, - } - ) - - assert result["status"] == "ready" - hits = indexer.index.search([1.0, 0.0, 0.0], limit=1) - assert hits[0].external_id == "doc_a" - assert hits[0].metadata["summary"] == "Unified metadata summary." - assert hits[0].metadata["department"] == "ops" - - -def test_summary_projection_indexer_defaults_to_1024_dimensions(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - indexer = SummaryProjectionIndexer( + projection = SummaryProjection( tmp_path / "projection", - embedder=FixedDimensionEmbedder(1024), - embedding_provider="test", - embedding_model="fake", + profile=profile, + embedder=FixedEmbedder(3), + create=True, ) + record = { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": {"summary": "Unified summary", "department": "ops"}, + } + + assert projection.upsert_summary(record)["status"] == "ready" + assert projection.info()["embedding_identity"] == { + "base_url": "https://example.invalid/v1", + "model": "fake", + "dimensions": 3, + } + assert [candidate.file_ref for candidate in projection.search("summary")] == [ + "file_a" + ] + assert projection.delete_summary("file_a") == 1 + assert projection.available is False - info = indexer.index.info() - - assert info["dimension"] == 1024 - assert info["metadata"]["embedding_dimensions"] == 1024 - result = indexer.upsert_summary( - { - "file_ref": "file_a", - "external_id": "doc_a", - "source_type": "documents", - "title": "A", - "metadata": {"summary": "Default dimension summary."}, - } +def test_projection_profile_mismatch_preserves_existing_database(tmp_path): + index_dir = tmp_path / "projection" + profile = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="fake", + dimensions=3, + api_key="runtime-only", ) - - assert result["status"] == "ready" - assert result["embedding_dimensions"] == 1024 - - -def test_summary_projection_indexer_allows_explicit_256_dimensions(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - indexer = SummaryProjectionIndexer( - tmp_path / "projection", - embedder=FixedDimensionEmbedder(256), - embedding_provider="test", - embedding_model="fake", - embedding_dimensions=256, + projection = SummaryProjection( + index_dir, + profile=profile, + embedder=FixedEmbedder(3), + create=True, ) - - assert indexer.index.info()["dimension"] == 256 - assert indexer.upsert_summary( + projection.upsert_summary( { "file_ref": "file_a", "external_id": "doc_a", "source_type": "documents", "title": "A", - "metadata": {"summary": "Explicit 256 dimension summary."}, + "metadata": {"summary": "Preserve me"}, } - )["status"] == "ready" - - -def test_summary_projection_default_rejects_existing_256_index_for_writes(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - index_dir = tmp_path / "projection" - index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - index.reset( - dimension=256, - metadata={ - "channel": "summary", - "embedding_provider": "test", - "embedding_model": "fake", - "embedding_dimensions": 256, - }, - ) - - with pytest.raises(RuntimeError, match="configured embedding_dimensions is 1024"): - SummaryProjectionIndexer( - index_dir, - embedder=FixedDimensionEmbedder(1024), - embedding_provider="test", - embedding_model="fake", - ) - - assert SQLiteVecSemanticIndex(index.db_path).info()["dimension"] == 256 - - -def test_summary_projection_from_provider_rejects_dimension_mismatch_before_embedder( - tmp_path, monkeypatch -): - from pageindex.filesystem import semantic_projection - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - index_dir = tmp_path / "projection" - index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - index.reset( - dimension=256, - metadata={ - "channel": "summary", - "embedding_provider": "openai", - "embedding_model": "text-embedding-3-small", - "embedding_dimensions": 256, - }, ) - - def fail_make_embedder(*args, **kwargs): - raise AssertionError("embedder should not be constructed before dimension validation") - - monkeypatch.setattr(semantic_projection, "make_embedder", fail_make_embedder) - - with pytest.raises(RuntimeError, match="configured embedding_dimensions is 1024"): - SummaryProjectionIndexer.from_provider(index_dir) - - -def test_embedding_cache_key_separates_model_dimensions(tmp_path): - from pageindex.filesystem.semantic_projection import ( - EmbeddingCache, - embedding_cache_model_key, + summary_path = index_dir / "summary.sqlite" + before = hashlib.sha256(summary_path.read_bytes()).hexdigest() + mismatch = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="different", + dimensions=3, + api_key="runtime-only", ) - class CountingEmbedder: - def __init__(self, dimensions: int): - self.dimensions = dimensions - self.calls = 0 + with pytest.raises(Exception, match="Incompatible PIFS Summary Embedding Profile"): + SummaryProjection(index_dir, profile=mismatch, create=False) - def embed(self, texts): - self.calls += 1 - return [[float(self.dimensions), *([0.0] * (self.dimensions - 1))] for _ in texts] - - cache = EmbeddingCache(tmp_path / "cache.sqlite") - embedder_256 = CountingEmbedder(256) - embedder_1024 = CountingEmbedder(1024) - key_256 = embedding_cache_model_key("fake", 256) - key_1024 = embedding_cache_model_key("fake", 1024) - - assert key_256 != key_1024 - - vector_256 = cache.embed_texts( - ["same text"], - provider="test", - model=key_256, - embedder=embedder_256, - batch_size=1, - )[0] - vector_1024 = cache.embed_texts( - ["same text"], - provider="test", - model=key_1024, - embedder=embedder_1024, - batch_size=1, - )[0] - - assert len(vector_256) == 256 - assert len(vector_1024) == 1024 - assert embedder_256.calls == 1 - assert embedder_1024.calls == 1 + assert hashlib.sha256(summary_path.read_bytes()).hexdigest() == before def test_embedding_cache_rejects_response_length_mismatch(tmp_path): - from pageindex.filesystem.semantic_projection import EmbeddingCache - class ShortEmbedder: def embed(self, texts): return [[1.0, 0.0, 0.0]] - cache = EmbeddingCache(tmp_path / "cache.sqlite") + cache = EmbeddingCache(tmp_path / "cache.sqlite", create=True) + profile = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="fake", + dimensions=3, + api_key="runtime-only", + ) with pytest.raises(ValueError, match="embedding response length mismatch"): cache.embed_texts( ["first", "second"], - provider="test", - model="fake", + profile=profile, embedder=ShortEmbedder(), batch_size=2, ) -def test_embed_with_retry_does_not_retry_permanent_errors(monkeypatch): +def test_embed_with_retry_only_retries_transient_errors(monkeypatch): from pageindex.filesystem.semantic_projection import embed_with_retry sleeps = [] - class PermanentEmbeddingError(Exception): - status_code = 401 - - class FailingEmbedder: - calls = 0 - - def embed(self, texts): - self.calls += 1 - raise PermanentEmbeddingError("unauthorized") - - monkeypatch.setattr("pageindex.filesystem.semantic_projection.time.sleep", sleeps.append) - embedder = FailingEmbedder() - - with pytest.raises(PermanentEmbeddingError): - embed_with_retry(embedder, ["text"]) - - assert embedder.calls == 1 - assert sleeps == [] - - -def test_embed_with_retry_retries_transient_errors(monkeypatch): - from pageindex.filesystem.semantic_projection import embed_with_retry - - sleeps = [] - - class TransientEmbeddingError(Exception): - status_code = 500 + class EmbeddingError(Exception): + def __init__(self, status_code): + self.status_code = status_code class FlakyEmbedder: calls = 0 @@ -335,122 +196,25 @@ class FlakyEmbedder: def embed(self, texts): self.calls += 1 if self.calls == 1: - raise TransientEmbeddingError("server error") + raise EmbeddingError(500) return [[1.0, 0.0, 0.0]] - monkeypatch.setattr("pageindex.filesystem.semantic_projection.time.sleep", sleeps.append) + monkeypatch.setattr( + "pageindex.filesystem.semantic_projection.time.sleep", sleeps.append + ) embedder = FlakyEmbedder() - assert embed_with_retry(embedder, ["text"]) == [[1.0, 0.0, 0.0]] assert embedder.calls == 2 assert sleeps == [1.0] + class PermanentEmbedder: + calls = 0 -def test_summary_projection_dimension_mismatch_preserves_existing_index(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - index_dir = tmp_path / "projection" - index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - index.reset( - dimension=3, - metadata={ - "channel": "summary", - "embedding_provider": "test", - "embedding_model": "fake", - "embedding_dimensions": 3, - }, - ) - index.upsert_many( - [ - SemanticIndexRecord( - file_ref="file_a", - external_id="doc_a", - source_type="documents", - title="A", - text="summary", - vector=[1.0, 0.0, 0.0], - ) - ] - ) - - with pytest.raises(RuntimeError, match="summary projection index dimension mismatch"): - SummaryProjectionIndexer( - index_dir, - embedder=FixedDimensionEmbedder(4), - embedding_provider="test", - embedding_model="fake", - embedding_dimensions=4, - ) - - preserved = SQLiteVecSemanticIndex(index.db_path) - assert preserved.info()["dimension"] == 3 - assert preserved.info()["document_count"] == 1 - assert preserved.search([1.0, 0.0, 0.0], limit=1)[0].external_id == "doc_a" - - -def test_hash_embedding_provider_is_not_available(): - from pageindex.filesystem.semantic_projection import make_embedder - - with pytest.raises(ValueError, match="unknown embedding provider: hash"): - make_embedder("hash", "unused", dimensions=256, timeout=1) - - -def test_make_embedder_accepts_direct_runtime_config(monkeypatch): - from pageindex.filesystem.semantic_projection import make_embedder - - calls = [] - - class FakeOpenAI: - def __init__(self, **kwargs): - calls.append(kwargs) - - fake_openai = type(sys)("openai") - fake_openai.OpenAI = FakeOpenAI - monkeypatch.setitem(sys.modules, "openai", fake_openai) - monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") - monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") - monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") - monkeypatch.setenv("OPENAI_BASE_URL", "https://ignored-openai.invalid/") - - make_embedder( - "openai", - "gemini-embedding-2-preview", - dimensions=3072, - timeout=12.5, - api_key="direct-key", - base_url="https://example.invalid/openai/", - ) - - assert calls == [ - { - "api_key": "direct-key", - "base_url": "https://example.invalid/openai/", - "timeout": 12.5, - } - ] - - -def test_make_embedder_requires_direct_api_key(monkeypatch): - from pageindex.filesystem.semantic_projection import make_embedder - - calls = [] - - class FakeOpenAI: - def __init__(self, **kwargs): - calls.append(kwargs) - - fake_openai = type(sys)("openai") - fake_openai.OpenAI = FakeOpenAI - monkeypatch.setitem(sys.modules, "openai", fake_openai) - monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") - monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") - - with pytest.raises(ValueError, match="embedding_api_key is required"): - make_embedder( - "openai", - "gemini-embedding-2-preview", - dimensions=3072, - timeout=12.5, - ) + def embed(self, texts): + self.calls += 1 + raise EmbeddingError(401) - assert calls == [] + permanent = PermanentEmbedder() + with pytest.raises(EmbeddingError): + embed_with_retry(permanent, ["text"]) + assert permanent.calls == 1 From bd03e1593b7647afd6d2ecd384e6112eb8d71ddc Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 15:31:01 +0800 Subject: [PATCH 02/14] fix(filesystem): enforce exact PIFS schema validation Fail closed on pseudo-v2 catalog and projection databases, validate projection topology before any workspace mutation, and keep migration subprocess coverage outside the PageIndex test suite. --- pageindex/filesystem/core.py | 17 +- pageindex/filesystem/semantic_index.py | 319 +++++++++++----- pageindex/filesystem/semantic_projection.py | 179 ++++++--- pageindex/filesystem/store.py | 108 +++++- tests/test_pifs_simplification_acceptance.py | 374 +++++++++++-------- 5 files changed, 682 insertions(+), 315 deletions(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index c4ea76ebe..401186c6f 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -68,14 +68,25 @@ def __init__( summary_projection_embedding_base_url: str | None = None, ): self.workspace = Path(workspace).expanduser() - self.store = SQLiteFileSystemStore(self.workspace) - self.metadata = MetadataQueryEngine(self.store) - self.summary_projection: Any | None = None self.summary_projection_index_dir = ( Path(summary_projection_index_dir).expanduser() if summary_projection_index_dir is not None else self.workspace / "artifacts" / "projection_indexes" ) + summary_path = self.summary_projection_index_dir / "summary.sqlite" + cache_path = self.summary_projection_index_dir / "embedding_cache.sqlite" + if summary_path.exists() != cache_path.exists(): + raise RuntimeError( + "PIFS Summary Projection topology is incomplete; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + if summary_path.exists(): + from .semantic_projection import validate_projection_topology + + validate_projection_topology(self.summary_projection_index_dir) + self.store = SQLiteFileSystemStore(self.workspace) + self.metadata = MetadataQueryEngine(self.store) + self.summary_projection: Any | None = None self.summary_projection_embedding_model = summary_projection_embedding_model self.summary_projection_embedding_dimensions = summary_projection_embedding_dimensions self.summary_projection_embedding_timeout = summary_projection_embedding_timeout diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py index e48428874..7a9fc39ce 100644 --- a/pageindex/filesystem/semantic_index.py +++ b/pageindex/filesystem/semantic_index.py @@ -2,6 +2,7 @@ import hashlib import json +import re import sqlite3 from dataclasses import dataclass from pathlib import Path @@ -15,6 +16,13 @@ class SemanticIndexError(RuntimeError): SCHEMA_VERSION = 2 +SUMMARY_TABLES = { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", +} +SUMMARY_CONFIG_KEYS = {"adapter", "adapter_version", "dimension", "metadata"} +VEC_DIMENSION_RE = re.compile(r"\bembedding\s+float\[(\d+)\]", re.IGNORECASE) @dataclass(frozen=True) @@ -48,77 +56,16 @@ class SQLiteVecSemanticIndex: def __init__(self, db_path: str | Path): self.db_path = Path(db_path).expanduser() - self.db_path.parent.mkdir(parents=True, exist_ok=True) def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: if dimension <= 0: raise SemanticIndexError("semantic index dimension must be positive") + self.db_path.parent.mkdir(parents=True, exist_ok=True) with self.connect() as conn: - conn.executescript( - """ - DROP TABLE IF EXISTS semantic_index_vec; - DROP TABLE IF EXISTS semantic_index_docs; - DROP TABLE IF EXISTS semantic_index_config; - CREATE TABLE semantic_index_config ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - CREATE TABLE semantic_index_docs ( - rowid INTEGER PRIMARY KEY, - file_ref TEXT NOT NULL UNIQUE, - external_id TEXT, - source_type TEXT NOT NULL DEFAULT '', - title TEXT NOT NULL DEFAULT '', - text_hash TEXT NOT NULL, - text_chars INTEGER NOT NULL DEFAULT 0, - metadata_json TEXT NOT NULL DEFAULT '{}', - created_at TEXT DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX idx_semantic_index_docs_external_id - ON semantic_index_docs(external_id); - CREATE INDEX idx_semantic_index_docs_source_type - ON semantic_index_docs(source_type); - """ - ) - conn.execute( - "CREATE VIRTUAL TABLE semantic_index_vec USING " - f"vec0(source_type TEXT partition key, embedding float[{dimension}])" - ) - config = { - "dimension": str(dimension), - "adapter": "sqlite-vec", - "adapter_version": sqlite_vec.__version__, - "metadata": json.dumps(metadata or {}, ensure_ascii=False, sort_keys=True), - } - conn.executemany( - "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", - sorted(config.items()), - ) - conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") - conn.commit() + self._create_schema(conn, dimension=dimension, metadata=metadata or {}) def validate(self, expected_identity: dict[str, Any]) -> dict[str, Any]: - with self.connect() as conn: - version = int(conn.execute("PRAGMA user_version").fetchone()[0]) - tables = { - str(row[1]) - for row in conn.execute("PRAGMA table_list") - if row[2] in {"table", "virtual"} - and not str(row[1]).startswith("sqlite_") - and not str(row[1]).startswith("semantic_index_vec_") - } - required = { - "semantic_index_config", - "semantic_index_docs", - "semantic_index_vec", - } - if version != SCHEMA_VERSION or tables != required: - raise SemanticIndexError( - "Incompatible PIFS Summary Projection schema; migrate this workspace " - "with pifs-data/scripts/migrate_pifs_workspace.py before opening it." - ) - info = self.info() + info = self.validate_schema() if info["metadata"] != expected_identity: raise SemanticIndexError( "Incompatible PIFS Summary Embedding Profile; migrate the workspace or " @@ -131,6 +78,51 @@ def validate(self, expected_identity: dict[str, Any]) -> dict[str, Any]: ) return info + def validate_schema(self) -> dict[str, Any]: + try: + with self.connect(read_only=True) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + tables = self._logical_tables(conn) + config = self._config(conn) + info = self._info_from_connection(conn, config) + actual = self._schema_signature(conn, tables) + dimension = int(info["dimension"]) + metadata = info["metadata"] + if ( + version != SCHEMA_VERSION + or tables != SUMMARY_TABLES + or set(config) != SUMMARY_CONFIG_KEYS + or config["adapter"] != "sqlite-vec" + or not config["adapter_version"] + or dimension <= 0 + or set(metadata) != {"base_url", "model", "dimensions"} + or not isinstance(metadata["base_url"], str) + or not metadata["base_url"].strip() + or not isinstance(metadata["model"], str) + or not metadata["model"].strip() + or isinstance(metadata["dimensions"], bool) + or int(metadata["dimensions"]) != dimension + or actual["vec_dimension"] != dimension + ): + raise self._incompatible_schema_error() + with self._memory_connection() as expected_conn: + self._create_schema( + expected_conn, + dimension=dimension, + metadata=metadata, + ) + expected = self._schema_signature( + expected_conn, + self._logical_tables(expected_conn), + ) + if actual != expected: + raise self._incompatible_schema_error() + except SemanticIndexError: + raise + except (json.JSONDecodeError, sqlite3.Error, TypeError, ValueError) as exc: + raise self._incompatible_schema_error() from exc + return info + def upsert_many(self, records: list[SemanticIndexRecord]) -> int: if not records: return 0 @@ -312,27 +304,8 @@ def _search_fetch_k( return min(4096, max(limit, limit * max(fetch_multiplier, 1))) def info(self) -> dict[str, Any]: - with self.connect() as conn: - config = { - row["key"]: row["value"] - for row in conn.execute( - "SELECT key, value FROM semantic_index_config ORDER BY key" - ).fetchall() - } - count = conn.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] - parsed_metadata: dict[str, Any] - try: - parsed_metadata = json.loads(config.get("metadata", "{}")) - except json.JSONDecodeError: - parsed_metadata = {} - return { - "db_path": str(self.db_path), - "adapter": config.get("adapter", "sqlite-vec"), - "adapter_version": config.get("adapter_version", ""), - "dimension": int(config.get("dimension", "0") or 0), - "document_count": count, - "metadata": parsed_metadata, - } + with self.connect(read_only=True) as conn: + return self._info_from_connection(conn, self._config(conn)) def dimension(self) -> int: with self.connect() as conn: @@ -345,14 +318,186 @@ def dimension(self) -> int: ) return int(row["value"]) - def connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) + def connect(self, *, read_only: bool = False) -> sqlite3.Connection: + if read_only: + conn = sqlite3.connect( + f"{self.db_path.resolve().as_uri()}?mode=ro", + uri=True, + ) + else: + conn = sqlite3.connect(self.db_path) conn.row_factory = sqlite3.Row + self._load_extension(conn) + return conn + + @staticmethod + def _load_extension(conn: sqlite3.Connection) -> None: conn.enable_load_extension(True) sqlite_vec.load(conn) conn.enable_load_extension(False) + + @classmethod + def _memory_connection(cls) -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + cls._load_extension(conn) return conn + @staticmethod + def _create_schema( + conn: sqlite3.Connection, + *, + dimension: int, + metadata: dict[str, Any], + ) -> None: + conn.executescript( + """ + DROP TABLE IF EXISTS semantic_index_vec; + DROP TABLE IF EXISTS semantic_index_docs; + DROP TABLE IF EXISTS semantic_index_config; + CREATE TABLE semantic_index_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE semantic_index_docs ( + rowid INTEGER PRIMARY KEY, + file_ref TEXT NOT NULL UNIQUE, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + conn.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + f"vec0(source_type TEXT partition key, embedding float[{dimension}])" + ) + config = { + "dimension": str(dimension), + "adapter": "sqlite-vec", + "adapter_version": sqlite_vec.__version__, + "metadata": json.dumps(metadata, ensure_ascii=False, sort_keys=True), + } + conn.executemany( + "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", + sorted(config.items()), + ) + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + conn.commit() + + @staticmethod + def _logical_tables(conn: sqlite3.Connection) -> set[str]: + return { + str(row[1]) + for row in conn.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} + and not str(row[1]).startswith("sqlite_") + and not str(row[1]).startswith("semantic_index_vec_") + } + + @staticmethod + def _config(conn: sqlite3.Connection) -> dict[str, str]: + return { + str(row["key"]): str(row["value"]) + for row in conn.execute( + "SELECT key, value FROM semantic_index_config ORDER BY key" + ) + } + + def _info_from_connection( + self, + conn: sqlite3.Connection, + config: dict[str, str], + ) -> dict[str, Any]: + metadata = json.loads(config.get("metadata", "{}")) + if not isinstance(metadata, dict): + raise self._incompatible_schema_error() + return { + "db_path": str(self.db_path), + "adapter": config.get("adapter", ""), + "adapter_version": config.get("adapter_version", ""), + "dimension": int(config.get("dimension", "0") or 0), + "document_count": int( + conn.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] + ), + "metadata": metadata, + } + + @staticmethod + def _schema_signature( + conn: sqlite3.Connection, + tables: set[str], + ) -> dict[str, Any]: + columns = { + table: tuple( + tuple(row) + for row in conn.execute(f'PRAGMA table_xinfo("{table}")') + ) + for table in sorted(tables) + } + indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} + foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} + for table in sorted(tables): + table_indexes = [] + for row in conn.execute(f'PRAGMA index_list("{table}")'): + name = str(row[1]) + origin = str(row[3]) + index_columns = tuple( + str(column[2]) + for column in conn.execute(f'PRAGMA index_info("{name}")') + ) + table_indexes.append( + ( + name if origin == "c" else None, + int(row[2]), + origin, + int(row[4]), + index_columns, + ) + ) + indexes[table] = tuple(sorted(table_indexes, key=repr)) + foreign_keys[table] = tuple( + sorted( + ( + str(row[2]), + str(row[3]), + str(row[4]), + str(row[5]), + str(row[6]), + str(row[7]), + ) + for row in conn.execute(f'PRAGMA foreign_key_list("{table}")') + ) + ) + vec_row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' " + "AND name = 'semantic_index_vec'" + ).fetchone() + match = VEC_DIMENSION_RE.search(str(vec_row[0] if vec_row else "")) + return { + "tables": tables, + "columns": columns, + "indexes": indexes, + "foreign_keys": foreign_keys, + "vec_dimension": int(match.group(1)) if match else None, + } + + @staticmethod + def _incompatible_schema_error() -> SemanticIndexError: + return SemanticIndexError( + "Incompatible PIFS Summary Projection schema; migrate this workspace " + "with pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + @staticmethod def text_hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index 5c22c3a1f..754fe54c4 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -102,16 +102,15 @@ def __init__( self.index = SQLiteVecSemanticIndex( self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" ) - summary_exists = self.index.db_path.exists() and self.index.db_path.stat().st_size > 0 + summary_exists = self.index.db_path.exists() cache_path = self.index_dir / "embedding_cache.sqlite" - cache_exists = cache_path.exists() and cache_path.stat().st_size > 0 - if summary_exists: + cache_exists = cache_path.exists() + if summary_exists and cache_exists: self.index.validate(self.profile.identity) self.embedding_cache = EmbeddingCache(cache_path, create=False) - elif cache_exists: - self.embedding_cache = EmbeddingCache(cache_path, create=False) + elif summary_exists or cache_exists: raise RuntimeError( - "PIFS Summary Projection is missing; migrate this workspace with " + "PIFS Summary Projection topology is incomplete; migrate this workspace with " "pifs-data/scripts/migrate_pifs_workspace.py before opening it." ) elif create: @@ -208,45 +207,29 @@ def _embedder(self) -> Any: class EmbeddingCache: - COLUMNS = { - "base_url", - "model", - "dimensions", - "text_hash", - "vector_blob", - "created_at", - } - def __init__(self, db_path: Path, *, create: bool) -> None: self.db_path = db_path - exists = self.db_path.exists() and self.db_path.stat().st_size > 0 + exists = self.db_path.exists() if exists: self._validate() elif create: self.db_path.parent.mkdir(parents=True, exist_ok=True) with self.connect() as connection: - connection.executescript( - f""" - CREATE TABLE embedding_cache ( - base_url TEXT NOT NULL, - model TEXT NOT NULL, - dimensions INTEGER NOT NULL CHECK(dimensions > 0), - text_hash TEXT NOT NULL, - vector_blob BLOB NOT NULL, - created_at TEXT DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(base_url, model, dimensions, text_hash) - ); - PRAGMA user_version = {SCHEMA_VERSION}; - """ - ) + self._create_schema(connection) else: raise RuntimeError( "PIFS embedding cache is missing; migrate this workspace with " "pifs-data/scripts/migrate_pifs_workspace.py before opening it." ) - def connect(self) -> sqlite3.Connection: - connection = sqlite3.connect(self.db_path) + def connect(self, *, read_only: bool = False) -> sqlite3.Connection: + if read_only: + connection = sqlite3.connect( + f"{self.db_path.resolve().as_uri()}?mode=ro", + uri=True, + ) + else: + connection = sqlite3.connect(self.db_path) connection.row_factory = sqlite3.Row return connection @@ -320,24 +303,126 @@ def embed_texts( return [cached[text_hash] for text_hash in hashes] def _validate(self) -> None: - with self.connect() as connection: - version = int(connection.execute("PRAGMA user_version").fetchone()[0]) - tables = { - str(row[0]) + try: + with self.connect(read_only=True) as connection: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + actual = self._schema_signature(connection) + invalid_rows = int( + connection.execute( + "SELECT COUNT(*) FROM embedding_cache " + "WHERE dimensions <= 0 OR length(vector_blob) != dimensions * 4 " + "OR trim(base_url) = '' OR trim(model) = '' OR trim(text_hash) = ''" + ).fetchone()[0] + ) + with sqlite3.connect(":memory:") as expected_connection: + expected_connection.row_factory = sqlite3.Row + self._create_schema(expected_connection) + expected = self._schema_signature(expected_connection) + except sqlite3.Error as exc: + raise self._incompatible_schema_error() from exc + if version != SCHEMA_VERSION or actual != expected or invalid_rows: + raise self._incompatible_schema_error() + + @staticmethod + def _create_schema(connection: sqlite3.Connection) -> None: + connection.executescript( + f""" + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(base_url, model, dimensions, text_hash) + ); + PRAGMA user_version = {SCHEMA_VERSION}; + """ + ) + + @staticmethod + def _schema_signature(connection: sqlite3.Connection) -> dict[str, Any]: + tables = { + str(row[0]) + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + if tables != {"embedding_cache"}: + return {"tables": tables} + columns = tuple( + tuple(row) + for row in connection.execute('PRAGMA table_xinfo("embedding_cache")') + ) + indexes = [] + for row in connection.execute('PRAGMA index_list("embedding_cache")'): + name = str(row[1]) + origin = str(row[3]) + index_columns = tuple( + str(column[2]) + for column in connection.execute(f'PRAGMA index_info("{name}")') + ) + indexes.append( + ( + name if origin == "c" else None, + int(row[2]), + origin, + int(row[4]), + index_columns, + ) + ) + foreign_keys = tuple( + sorted( + ( + str(row[2]), + str(row[3]), + str(row[4]), + str(row[5]), + str(row[6]), + str(row[7]), + ) for row in connection.execute( - "SELECT name FROM sqlite_master " - "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + 'PRAGMA foreign_key_list("embedding_cache")' ) - } - columns = { - str(row[1]) - for row in connection.execute("PRAGMA table_info(embedding_cache)") - } - if version != SCHEMA_VERSION or tables != {"embedding_cache"} or columns != self.COLUMNS: - raise RuntimeError( - "Incompatible PIFS embedding cache schema; migrate this workspace with " - "pifs-data/scripts/migrate_pifs_workspace.py before opening it." ) + ) + sql = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' " + "AND name = 'embedding_cache'" + ).fetchone()[0] + return { + "tables": tables, + "columns": columns, + "indexes": tuple(sorted(indexes, key=repr)), + "foreign_keys": foreign_keys, + "sql": " ".join(str(sql).lower().split()), + } + + @staticmethod + def _incompatible_schema_error() -> RuntimeError: + return RuntimeError( + "Incompatible PIFS embedding cache schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + +def validate_projection_topology(index_dir: str | Path) -> bool: + index_dir = Path(index_dir).expanduser() + summary_path = index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" + cache_path = index_dir / "embedding_cache.sqlite" + summary_exists = summary_path.exists() + cache_exists = cache_path.exists() + if not summary_exists and not cache_exists: + return False + if not summary_exists or not cache_exists: + raise RuntimeError( + "PIFS Summary Projection topology is incomplete; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + SQLiteVecSemanticIndex(summary_path).validate_schema() + EmbeddingCache(cache_path, create=False) + return True class EmbeddingClient: diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py index b7c80484a..9bdfe881f 100644 --- a/pageindex/filesystem/store.py +++ b/pageindex/filesystem/store.py @@ -39,30 +39,26 @@ def connect(self) -> sqlite3.Connection: def initialize_schema(self) -> None: if self.db_path.exists() and self.db_path.stat().st_size > 0: - with self.connect() as conn: - version = int(conn.execute("PRAGMA user_version").fetchone()[0]) - tables = { - str(row[0]) - for row in conn.execute( - "SELECT name FROM sqlite_master " - "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" - ) - } - if version != SCHEMA_VERSION or tables != CATALOG_TABLES: - raise RuntimeError( - "Incompatible PIFS catalog schema; migrate this workspace with " - "pifs-data/scripts/migrate_pifs_workspace.py before opening it " - f"(expected version {SCHEMA_VERSION} and tables " - f"{sorted(CATALOG_TABLES)}, found version {version} and tables " - f"{sorted(tables)})." - ) + try: + with self._readonly_connection(self.db_path) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + actual = self._schema_signature(conn) + with sqlite3.connect(":memory:") as expected_conn: + expected_conn.row_factory = sqlite3.Row + self._create_current_schema(expected_conn) + expected = self._schema_signature(expected_conn) + except sqlite3.Error as exc: + raise self._incompatible_schema_error() from exc + if version != SCHEMA_VERSION or actual != expected: + raise self._incompatible_schema_error() return with self.connect() as conn: self._create_current_schema(conn) self.ensure_folder(conn, "/") conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") - def _create_current_schema(self, conn: sqlite3.Connection) -> None: + @staticmethod + def _create_current_schema(conn: sqlite3.Connection) -> None: conn.executescript( """ CREATE TABLE IF NOT EXISTS files ( @@ -135,6 +131,82 @@ def _create_current_schema(self, conn: sqlite3.Connection) -> None: """ ) + @staticmethod + def _readonly_connection(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect( + f"{path.resolve().as_uri()}?mode=ro", + uri=True, + ) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + return connection + + @staticmethod + def _schema_signature(conn: sqlite3.Connection) -> dict[str, Any]: + tables = { + str(row[0]) + for row in conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + columns = { + table: tuple( + tuple(row) + for row in conn.execute(f'PRAGMA table_xinfo("{table}")') + ) + for table in sorted(tables) + } + indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} + foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} + for table in sorted(tables): + table_indexes = [] + for row in conn.execute(f'PRAGMA index_list("{table}")'): + name = str(row[1]) + origin = str(row[3]) + index_columns = tuple( + str(column[2]) + for column in conn.execute(f'PRAGMA index_info("{name}")') + ) + table_indexes.append( + ( + name if origin == "c" else None, + int(row[2]), + origin, + int(row[4]), + index_columns, + ) + ) + indexes[table] = tuple(sorted(table_indexes, key=repr)) + foreign_keys[table] = tuple( + sorted( + ( + str(row[2]), + str(row[3]), + str(row[4]), + str(row[5]), + str(row[6]), + str(row[7]), + ) + for row in conn.execute(f'PRAGMA foreign_key_list("{table}")') + ) + ) + return { + "tables": tables, + "columns": columns, + "indexes": indexes, + "foreign_keys": foreign_keys, + } + + @staticmethod + def _incompatible_schema_error() -> RuntimeError: + return RuntimeError( + "Incompatible PIFS catalog schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it " + f"(expected exact schema version {SCHEMA_VERSION} with tables " + f"{sorted(CATALOG_TABLES)})." + ) + @staticmethod def _json_object(value: Any) -> dict[str, Any]: try: diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 6220fe8a3..910580137 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -1,8 +1,6 @@ import json import hashlib import sqlite3 -import subprocess -import sys from pathlib import Path from types import SimpleNamespace @@ -115,6 +113,40 @@ def logical_tables(connection): } +def workspace_state(workspace): + workspace = Path(workspace) + if not workspace.exists(): + return {} + return { + path.relative_to(workspace).as_posix(): ( + "directory" + if path.is_dir() + else hashlib.sha256(path.read_bytes()).hexdigest() + ) + for path in sorted(workspace.rglob("*")) + } + + +def create_projection_v2(workspace): + from pageindex.filesystem.semantic_projection import ( + SummaryEmbeddingProfile, + SummaryProjection, + ) + + projection_dir = Path(workspace) / "artifacts" / "projection_indexes" + SummaryProjection( + projection_dir, + profile=SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="test-embedding", + dimensions=3, + api_key="runtime-only", + ), + create=True, + ) + return projection_dir + + def catalog_file_count(workspace): with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] @@ -180,6 +212,47 @@ def test_cli_rejects_legacy_catalog_without_mutating_it(tmp_path, capsys): assert hashlib.sha256(database.read_bytes()).hexdigest() == before +@pytest.mark.parametrize( + "mutation", + ["extra_column", "missing_index", "missing_primary_and_foreign_keys"], +) +def test_cli_rejects_pseudo_v2_catalog_without_mutating_workspace( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + database = workspace / "filesystem.sqlite" + with sqlite3.connect(database) as connection: + if mutation == "extra_column": + connection.execute("ALTER TABLE files ADD COLUMN legacy_provider TEXT") + elif mutation == "missing_index": + connection.execute("DROP INDEX idx_files_external_id") + else: + connection.executescript( + """ + ALTER TABLE file_folders RENAME TO legacy_file_folders; + CREATE TABLE file_folders ( + file_ref TEXT NOT NULL, + folder_id TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_file_folders; + CREATE INDEX idx_file_folders_folder ON file_folders(folder_id); + """ + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + def test_cli_rejects_partial_legacy_projection_without_creating_summary( tmp_path, monkeypatch, capsys ): @@ -210,6 +283,145 @@ def test_cli_rejects_partial_legacy_projection_without_creating_summary( assert catalog_file_count(workspace) == 0 +def test_cli_preflights_partial_projection_before_creating_catalog(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + projection_dir = workspace / "artifacts" / "projection_indexes" + projection_dir.mkdir(parents=True) + cache_path = projection_dir / "embedding_cache.sqlite" + with sqlite3.connect(cache_path) as connection: + connection.execute( + "CREATE TABLE embedding_cache(provider TEXT, model TEXT, text_hash TEXT)" + ) + connection.execute("PRAGMA user_version = 1") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert not (workspace / "filesystem.sqlite").exists() + + +@pytest.mark.parametrize( + "mutation", + [ + "summary_extra_column", + "summary_missing_primary_and_unique", + "summary_missing_index", + "summary_extra_config_key", + "summary_vec_dimension_mismatch", + "cache_extra_column", + "cache_missing_primary_key", + ], +) +def test_cli_rejects_pseudo_v2_projection_without_mutating_workspace( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + projection_dir = create_projection_v2(workspace) + summary_path = projection_dir / "summary.sqlite" + cache_path = projection_dir / "embedding_cache.sqlite" + if mutation.startswith("summary_"): + with sqlite3.connect(summary_path) as connection: + if mutation == "summary_extra_column": + connection.execute( + "ALTER TABLE semantic_index_docs ADD COLUMN legacy_provider TEXT" + ) + elif mutation == "summary_missing_primary_and_unique": + connection.executescript( + """ + DROP INDEX idx_semantic_index_docs_external_id; + DROP INDEX idx_semantic_index_docs_source_type; + ALTER TABLE semantic_index_docs RENAME TO legacy_semantic_index_docs; + CREATE TABLE semantic_index_docs ( + rowid INTEGER, + file_ref TEXT NOT NULL, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_semantic_index_docs; + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + elif mutation == "summary_missing_index": + connection.execute("DROP INDEX idx_semantic_index_docs_external_id") + elif mutation == "summary_extra_config_key": + connection.execute( + "INSERT INTO semantic_index_config(key, value) " + "VALUES ('legacy_provider', 'openai')" + ) + else: + connection.execute( + "UPDATE semantic_index_config SET value = '4' WHERE key = 'dimension'" + ) + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + ( + json.dumps( + { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 4, + } + ), + ), + ) + else: + with sqlite3.connect(cache_path) as connection: + if mutation == "cache_extra_column": + connection.execute( + "ALTER TABLE embedding_cache ADD COLUMN provider TEXT" + ) + else: + connection.executescript( + """ + ALTER TABLE embedding_cache RENAME TO legacy_embedding_cache; + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_embedding_cache; + """ + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert not (workspace / "filesystem.sqlite").exists() + + +def test_cli_allows_fresh_listing_when_projection_is_completely_absent(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert (workspace / "filesystem.sqlite").is_file() + assert not (workspace / "artifacts" / "projection_indexes").exists() + + def test_pageindex_is_the_only_public_python_entry_point_for_pifs(): import pageindex import pageindex.filesystem as filesystem_module @@ -1016,164 +1228,6 @@ def test_set_workspace_persists_runtime_config_without_credentials_or_provider( } -def test_external_migration_fixture_reopens_without_reembedding( - tmp_path, monkeypatch, capsys -): - from pageindex.filesystem.cli import main - - install_network_fakes(monkeypatch) - write_embedding_config(tmp_path, monkeypatch) - workspace = tmp_path / "legacy-workspace" - source = tmp_path / "notes.md" - source.write_text("alpha evidence", encoding="utf-8") - assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 - capsys.readouterr() - assert main( - [ - "--workspace", - str(workspace), - "setmeta", - "/documents/notes.md", - '{"year": 2024}', - ] - ) == 0 - capsys.readouterr() - - projection_dir = workspace / "artifacts" / "projection_indexes" - with sqlite3.connect(workspace / "filesystem.sqlite") as connection: - connection.execute("CREATE TABLE metadata_schema(name TEXT)") - connection.execute("CREATE TABLE semantic_folders(folder_id TEXT)") - connection.execute("CREATE TABLE file_fts(file_ref TEXT, body TEXT)") - connection.execute("PRAGMA user_version = 1") - with sqlite3.connect(projection_dir / "summary.sqlite") as connection: - connection.execute( - "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", - ( - json.dumps( - { - "embedding_provider": "openai", - "embedding_model": "test-embedding", - "embedding_dimensions": 3, - } - ), - ), - ) - connection.execute( - "CREATE INDEX idx_semantic_index_docs_file_ref " - "ON semantic_index_docs(file_ref)" - ) - connection.execute("PRAGMA user_version = 1") - with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as connection: - connection.executescript( - """ - ALTER TABLE embedding_cache RENAME TO embedding_cache_v2; - CREATE TABLE embedding_cache ( - provider TEXT NOT NULL, - model TEXT NOT NULL, - text_hash TEXT NOT NULL, - dimension INTEGER NOT NULL, - vector_blob BLOB, - vector_json TEXT, - created_at TEXT, - PRIMARY KEY(provider, model, text_hash) - ); - INSERT INTO embedding_cache( - provider, model, text_hash, dimension, - vector_blob, vector_json, created_at - ) - SELECT - 'openai', model || ':dimensions=' || dimensions, - text_hash, dimensions, vector_blob, NULL, created_at - FROM embedding_cache_v2; - DROP TABLE embedding_cache_v2; - PRAGMA user_version = 1; - """ - ) - - script = Path( - "/Users/chengjie/Projects/pifs-data/scripts/migrate_pifs_workspace.py" - ) - migration_command = [ - sys.executable, - str(script), - "--workspace", - str(workspace), - "--embedding-base-url", - "https://EXAMPLE.invalid/v1/", - "--embedding-model", - "test-embedding", - ] - migrated = subprocess.run( - migration_command, - check=False, - capture_output=True, - text=True, - ) - assert migrated.returncode == 0, migrated.stderr - report = json.loads(migrated.stdout) - assert report["status"] == "complete" - assert report["filesystem"]["files"] == 1 - assert report["summary"] == {"documents": 1, "vectors": 1, "dimensions": 3} - assert report["embedding_cache"] == {"rows": 1} - - databases = [ - workspace / "filesystem.sqlite", - projection_dir / "summary.sqlite", - projection_dir / "embedding_cache.sqlite", - ] - hashes = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases} - repeated = subprocess.run( - migration_command, - check=False, - capture_output=True, - text=True, - ) - assert repeated.returncode == 0, repeated.stderr - assert json.loads(repeated.stdout)["status"] == "already_migrated" - assert { - path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases - } == hashes - - with sqlite3.connect(databases[0]) as catalog: - assert logical_tables(catalog) == { - "files", - "folders", - "file_folders", - "metadata_fields", - "metadata_values", - } - with sqlite3.connect(databases[1]) as summary: - assert summary.execute("PRAGMA user_version").fetchone()[0] == 2 - metadata = json.loads( - summary.execute( - "SELECT value FROM semantic_index_config WHERE key = 'metadata'" - ).fetchone()[0] - ) - assert metadata == { - "base_url": "https://example.invalid/v1", - "model": "test-embedding", - "dimensions": 3, - } - with sqlite3.connect(databases[2]) as cache: - assert logical_tables(cache) == {"embedding_cache"} - assert cache.execute("SELECT COUNT(*) FROM embedding_cache").fetchone()[0] == 1 - - backup_roots = list((workspace / ".pifs-migration-backups").iterdir()) - assert len(backup_roots) == 1 - assert json.loads( - (backup_roots[0] / "migration.json").read_text(encoding="utf-8") - )["status"] == "complete" - assert not list(workspace.rglob("*.migrating-*")) - assert not list(workspace.rglob("*.rollback-*")) - - assert main(["--workspace", str(workspace), "tree", "/documents/@year=2024"]) == 0 - tree = json.loads(capsys.readouterr().out)["data"]["tree"] - assert tree["files"][0]["path"] == "/documents/@year=2024/notes.md" - assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 0 - documents = json.loads(capsys.readouterr().out)["data"]["documents"] - assert [document["title"] for document in documents] == ["notes.md"] - - def test_public_example_is_a_short_supported_cli_walkthrough(): example = Path(__file__).parents[1] / "examples" / "pifs_demo.py" source = example.read_text(encoding="utf-8") From 121bef95e2b7fa07bf9efd937d1e32b5343209d9 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 16:06:45 +0800 Subject: [PATCH 03/14] fix(filesystem): close PIFS migration validation gaps Require an exact catalog whenever projection databases exist, validate canonical vec0 and persisted embedding identities, and align the external migration's staged and idempotent checks with the runtime schema contract. --- pageindex/filesystem/_embedding_identity.py | 33 +++++ pageindex/filesystem/_projection_topology.py | 24 +++ pageindex/filesystem/_sqlite_schema.py | 75 ++++++++++ pageindex/filesystem/core.py | 19 ++- pageindex/filesystem/semantic_index.py | 83 ++++------- pageindex/filesystem/semantic_projection.py | 135 +++++------------ pageindex/filesystem/store.py | 88 +++-------- tests/test_pifs_simplification_acceptance.py | 146 +++++++++++++++++++ 8 files changed, 380 insertions(+), 223 deletions(-) create mode 100644 pageindex/filesystem/_embedding_identity.py create mode 100644 pageindex/filesystem/_projection_topology.py create mode 100644 pageindex/filesystem/_sqlite_schema.py diff --git a/pageindex/filesystem/_embedding_identity.py b/pageindex/filesystem/_embedding_identity.py new file mode 100644 index 000000000..a55445f8d --- /dev/null +++ b/pageindex/filesystem/_embedding_identity.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from urllib.parse import urlsplit, urlunsplit + + +DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" + + +def normalize_base_url(value: str | None) -> str: + raw = str(value or DEFAULT_OPENAI_BASE_URL).strip() + parsed = urlsplit(raw) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("embedding base URL must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError( + "embedding base URL must not contain credentials, query parameters, or a fragment" + ) + return urlunsplit( + ( + parsed.scheme.lower(), + parsed.netloc.lower(), + parsed.path.rstrip("/"), + "", + "", + ) + ) + + +def normalize_model(value: str | None) -> str: + model = str(value or "").strip() + if not model: + raise ValueError("embedding model must not be empty") + return model diff --git a/pageindex/filesystem/_projection_topology.py b/pageindex/filesystem/_projection_topology.py new file mode 100644 index 000000000..9418c8d1c --- /dev/null +++ b/pageindex/filesystem/_projection_topology.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from pathlib import Path + + +def projection_database_paths(index_dir: str | Path) -> tuple[Path, Path]: + index_dir = Path(index_dir).expanduser() + return index_dir / "summary.sqlite", index_dir / "embedding_cache.sqlite" + + +def projection_database_pair( + index_dir: str | Path, +) -> tuple[Path, Path] | None: + summary_path, cache_path = projection_database_paths(index_dir) + summary_exists = summary_path.exists() + cache_exists = cache_path.exists() + if not summary_exists and not cache_exists: + return None + if not summary_exists or not cache_exists: + raise RuntimeError( + "PIFS Summary Projection topology is incomplete; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + return summary_path, cache_path diff --git a/pageindex/filesystem/_sqlite_schema.py b/pageindex/filesystem/_sqlite_schema.py new file mode 100644 index 000000000..691184391 --- /dev/null +++ b/pageindex/filesystem/_sqlite_schema.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import sqlite3 +from typing import Any + + +def regular_table_names(connection: sqlite3.Connection) -> set[str]: + return { + str(row[0]) + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + + +def sqlite_schema_signature( + connection: sqlite3.Connection, + tables: set[str], +) -> dict[str, Any]: + columns = { + table: tuple( + tuple(row) + for row in connection.execute(f'PRAGMA table_xinfo("{table}")') + ) + for table in sorted(tables) + } + indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} + foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} + for table in sorted(tables): + table_indexes = [] + for row in connection.execute(f'PRAGMA index_list("{table}")'): + name = str(row[1]) + origin = str(row[3]) + index_columns = tuple( + str(column[2]) + for column in connection.execute(f'PRAGMA index_info("{name}")') + ) + table_indexes.append( + ( + name if origin == "c" else None, + int(row[2]), + origin, + int(row[4]), + index_columns, + ) + ) + indexes[table] = tuple(sorted(table_indexes, key=repr)) + foreign_keys[table] = tuple( + sorted( + ( + str(row[2]), + str(row[3]), + str(row[4]), + str(row[5]), + str(row[6]), + str(row[7]), + ) + for row in connection.execute(f'PRAGMA foreign_key_list("{table}")') + ) + ) + return { + "tables": tables, + "columns": columns, + "indexes": indexes, + "foreign_keys": foreign_keys, + } + + +def normalized_table_sql(connection: sqlite3.Connection, table: str) -> str: + row = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + (table,), + ).fetchone() + return " ".join(str(row[0] if row else "").lower().split()) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 401186c6f..05653a963 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -8,6 +8,10 @@ from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import quote, unquote, urlparse +from ._projection_topology import ( + projection_database_pair, + projection_database_paths, +) from .metadata import MetadataQueryEngine from .store import ( SQLiteFileSystemStore, @@ -73,14 +77,15 @@ def __init__( if summary_projection_index_dir is not None else self.workspace / "artifacts" / "projection_indexes" ) - summary_path = self.summary_projection_index_dir / "summary.sqlite" - cache_path = self.summary_projection_index_dir / "embedding_cache.sqlite" - if summary_path.exists() != cache_path.exists(): - raise RuntimeError( - "PIFS Summary Projection topology is incomplete; migrate this workspace with " - "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + summary_path, cache_path = projection_database_paths( + self.summary_projection_index_dir + ) + if summary_path.exists() or cache_path.exists(): + SQLiteFileSystemStore.validate_existing_database( + self.workspace / "filesystem.sqlite" ) - if summary_path.exists(): + database_pair = projection_database_pair(self.summary_projection_index_dir) + if database_pair is not None: from .semantic_projection import validate_projection_topology validate_projection_topology(self.summary_projection_index_dir) diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py index 7a9fc39ce..a430454e9 100644 --- a/pageindex/filesystem/semantic_index.py +++ b/pageindex/filesystem/semantic_index.py @@ -10,6 +10,9 @@ import sqlite_vec +from ._embedding_identity import normalize_base_url, normalize_model +from ._sqlite_schema import sqlite_schema_signature + class SemanticIndexError(RuntimeError): pass @@ -22,7 +25,17 @@ class SemanticIndexError(RuntimeError): "semantic_index_vec", } SUMMARY_CONFIG_KEYS = {"adapter", "adapter_version", "dimension", "metadata"} -VEC_DIMENSION_RE = re.compile(r"\bembedding\s+float\[(\d+)\]", re.IGNORECASE) +VEC0_DECLARATION_RE = re.compile( + r""" + ^\s*CREATE\s+VIRTUAL\s+TABLE\s+[\"`\[]?semantic_index_vec[\"`\]]? + \s+USING\s+vec0\s*\(\s* + source_type\s+TEXT\s+PARTITION\s+KEY\s*,\s* + embedding\s+FLOAT\s*\[\s*(\d+)\s*\]\s* + (?:distance_metric\s*=\s*([A-Za-z0-9_-]+)\s*)? + \)\s*$ + """, + re.IGNORECASE | re.VERBOSE, +) @dataclass(frozen=True) @@ -98,11 +111,15 @@ def validate_schema(self) -> dict[str, Any]: or set(metadata) != {"base_url", "model", "dimensions"} or not isinstance(metadata["base_url"], str) or not metadata["base_url"].strip() + or normalize_base_url(metadata["base_url"]) != metadata["base_url"] or not isinstance(metadata["model"], str) or not metadata["model"].strip() + or normalize_model(metadata["model"]) != metadata["model"] or isinstance(metadata["dimensions"], bool) or int(metadata["dimensions"]) != dimension - or actual["vec_dimension"] != dimension + or actual["vec0"] is None + or actual["vec0"]["dimension"] != dimension + or actual["vec0"]["distance_metric"] != "l2" ): raise self._incompatible_schema_error() with self._memory_connection() as expected_conn: @@ -437,59 +454,23 @@ def _schema_signature( conn: sqlite3.Connection, tables: set[str], ) -> dict[str, Any]: - columns = { - table: tuple( - tuple(row) - for row in conn.execute(f'PRAGMA table_xinfo("{table}")') - ) - for table in sorted(tables) - } - indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} - foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} - for table in sorted(tables): - table_indexes = [] - for row in conn.execute(f'PRAGMA index_list("{table}")'): - name = str(row[1]) - origin = str(row[3]) - index_columns = tuple( - str(column[2]) - for column in conn.execute(f'PRAGMA index_info("{name}")') - ) - table_indexes.append( - ( - name if origin == "c" else None, - int(row[2]), - origin, - int(row[4]), - index_columns, - ) - ) - indexes[table] = tuple(sorted(table_indexes, key=repr)) - foreign_keys[table] = tuple( - sorted( - ( - str(row[2]), - str(row[3]), - str(row[4]), - str(row[5]), - str(row[6]), - str(row[7]), - ) - for row in conn.execute(f'PRAGMA foreign_key_list("{table}")') - ) - ) + signature = sqlite_schema_signature(conn, tables) vec_row = conn.execute( "SELECT sql FROM sqlite_master WHERE type = 'table' " "AND name = 'semantic_index_vec'" ).fetchone() - match = VEC_DIMENSION_RE.search(str(vec_row[0] if vec_row else "")) - return { - "tables": tables, - "columns": columns, - "indexes": indexes, - "foreign_keys": foreign_keys, - "vec_dimension": int(match.group(1)) if match else None, - } + match = VEC0_DECLARATION_RE.fullmatch(str(vec_row[0] if vec_row else "")) + signature["vec0"] = ( + { + "partition_key": ("source_type", "text"), + "embedding_type": "float", + "dimension": int(match.group(1)), + "distance_metric": str(match.group(2) or "l2").lower(), + } + if match + else None + ) + return signature @staticmethod def _incompatible_schema_error() -> SemanticIndexError: diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index 754fe54c4..cd42b0f78 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -6,8 +6,18 @@ from dataclasses import dataclass from pathlib import Path from typing import Any -from urllib.parse import urlsplit, urlunsplit +from ._embedding_identity import ( + DEFAULT_OPENAI_BASE_URL, + normalize_base_url, + normalize_model, +) +from ._projection_topology import projection_database_pair +from ._sqlite_schema import ( + normalized_table_sql, + regular_table_names, + sqlite_schema_signature, +) from .core import DEFAULT_EMBEDDING_DIMENSIONS from .semantic_index import ( SCHEMA_VERSION, @@ -16,30 +26,9 @@ ) -DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" SUMMARY_INDEX_NAME = "summary" -def normalize_base_url(value: str | None) -> str: - raw = str(value or DEFAULT_OPENAI_BASE_URL).strip() - parsed = urlsplit(raw) - if parsed.scheme not in {"http", "https"} or not parsed.netloc: - raise ValueError("embedding base URL must be an absolute HTTP(S) URL") - if parsed.username or parsed.password or parsed.query or parsed.fragment: - raise ValueError( - "embedding base URL must not contain credentials, query parameters, or a fragment" - ) - return urlunsplit( - ( - parsed.scheme.lower(), - parsed.netloc.lower(), - parsed.path.rstrip("/"), - "", - "", - ) - ) - - @dataclass(frozen=True) class SummaryEmbeddingProfile: base_url: str | None = DEFAULT_OPENAI_BASE_URL @@ -49,9 +38,7 @@ class SummaryEmbeddingProfile: api_key: str | None = None def __post_init__(self) -> None: - model = str(self.model or "").strip() - if not model: - raise ValueError("embedding model must not be empty") + model = normalize_model(self.model) if int(self.dimensions) <= 0: raise ValueError("embedding dimensions must be positive") if float(self.timeout) <= 0: @@ -102,17 +89,11 @@ def __init__( self.index = SQLiteVecSemanticIndex( self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" ) - summary_exists = self.index.db_path.exists() cache_path = self.index_dir / "embedding_cache.sqlite" - cache_exists = cache_path.exists() - if summary_exists and cache_exists: + database_pair = projection_database_pair(self.index_dir) + if database_pair is not None: self.index.validate(self.profile.identity) self.embedding_cache = EmbeddingCache(cache_path, create=False) - elif summary_exists or cache_exists: - raise RuntimeError( - "PIFS Summary Projection topology is incomplete; migrate this workspace with " - "pifs-data/scripts/migrate_pifs_workspace.py before opening it." - ) elif create: self.index_dir.mkdir(parents=True, exist_ok=True) self.index.reset( @@ -314,13 +295,26 @@ def _validate(self) -> None: "OR trim(base_url) = '' OR trim(model) = '' OR trim(text_hash) = ''" ).fetchone()[0] ) + identities = connection.execute( + "SELECT DISTINCT base_url, model FROM embedding_cache" + ).fetchall() with sqlite3.connect(":memory:") as expected_connection: expected_connection.row_factory = sqlite3.Row self._create_schema(expected_connection) expected = self._schema_signature(expected_connection) - except sqlite3.Error as exc: + invalid_identity = any( + normalize_base_url(row["base_url"]) != row["base_url"] + or normalize_model(row["model"]) != row["model"] + for row in identities + ) + except (sqlite3.Error, ValueError) as exc: raise self._incompatible_schema_error() from exc - if version != SCHEMA_VERSION or actual != expected or invalid_rows: + if ( + version != SCHEMA_VERSION + or actual != expected + or invalid_rows + or invalid_identity + ): raise self._incompatible_schema_error() @staticmethod @@ -342,62 +336,12 @@ def _create_schema(connection: sqlite3.Connection) -> None: @staticmethod def _schema_signature(connection: sqlite3.Connection) -> dict[str, Any]: - tables = { - str(row[0]) - for row in connection.execute( - "SELECT name FROM sqlite_master " - "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" - ) - } + tables = regular_table_names(connection) if tables != {"embedding_cache"}: return {"tables": tables} - columns = tuple( - tuple(row) - for row in connection.execute('PRAGMA table_xinfo("embedding_cache")') - ) - indexes = [] - for row in connection.execute('PRAGMA index_list("embedding_cache")'): - name = str(row[1]) - origin = str(row[3]) - index_columns = tuple( - str(column[2]) - for column in connection.execute(f'PRAGMA index_info("{name}")') - ) - indexes.append( - ( - name if origin == "c" else None, - int(row[2]), - origin, - int(row[4]), - index_columns, - ) - ) - foreign_keys = tuple( - sorted( - ( - str(row[2]), - str(row[3]), - str(row[4]), - str(row[5]), - str(row[6]), - str(row[7]), - ) - for row in connection.execute( - 'PRAGMA foreign_key_list("embedding_cache")' - ) - ) - ) - sql = connection.execute( - "SELECT sql FROM sqlite_master WHERE type = 'table' " - "AND name = 'embedding_cache'" - ).fetchone()[0] - return { - "tables": tables, - "columns": columns, - "indexes": tuple(sorted(indexes, key=repr)), - "foreign_keys": foreign_keys, - "sql": " ".join(str(sql).lower().split()), - } + signature = sqlite_schema_signature(connection, tables) + signature["sql"] = normalized_table_sql(connection, "embedding_cache") + return signature @staticmethod def _incompatible_schema_error() -> RuntimeError: @@ -409,17 +353,10 @@ def _incompatible_schema_error() -> RuntimeError: def validate_projection_topology(index_dir: str | Path) -> bool: index_dir = Path(index_dir).expanduser() - summary_path = index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" - cache_path = index_dir / "embedding_cache.sqlite" - summary_exists = summary_path.exists() - cache_exists = cache_path.exists() - if not summary_exists and not cache_exists: + database_pair = projection_database_pair(index_dir) + if database_pair is None: return False - if not summary_exists or not cache_exists: - raise RuntimeError( - "PIFS Summary Projection topology is incomplete; migrate this workspace with " - "pifs-data/scripts/migrate_pifs_workspace.py before opening it." - ) + summary_path, cache_path = database_pair SQLiteVecSemanticIndex(summary_path).validate_schema() EmbeddingCache(cache_path, create=False) return True diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py index 9bdfe881f..b1e6afb1b 100644 --- a/pageindex/filesystem/store.py +++ b/pageindex/filesystem/store.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import Any, Iterable, Optional +from ._sqlite_schema import regular_table_names, sqlite_schema_signature from .types import FileEntry, MetadataField SCHEMA_VERSION = 2 @@ -39,24 +40,31 @@ def connect(self) -> sqlite3.Connection: def initialize_schema(self) -> None: if self.db_path.exists() and self.db_path.stat().st_size > 0: - try: - with self._readonly_connection(self.db_path) as conn: - version = int(conn.execute("PRAGMA user_version").fetchone()[0]) - actual = self._schema_signature(conn) - with sqlite3.connect(":memory:") as expected_conn: - expected_conn.row_factory = sqlite3.Row - self._create_current_schema(expected_conn) - expected = self._schema_signature(expected_conn) - except sqlite3.Error as exc: - raise self._incompatible_schema_error() from exc - if version != SCHEMA_VERSION or actual != expected: - raise self._incompatible_schema_error() + self.validate_existing_database(self.db_path) return with self.connect() as conn: self._create_current_schema(conn) self.ensure_folder(conn, "/") conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + @classmethod + def validate_existing_database(cls, db_path: str | Path) -> None: + db_path = Path(db_path) + if not db_path.is_file() or db_path.stat().st_size <= 0: + raise cls._incompatible_schema_error() + try: + with cls._readonly_connection(db_path) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + actual = cls._schema_signature(conn) + with sqlite3.connect(":memory:") as expected_conn: + expected_conn.row_factory = sqlite3.Row + cls._create_current_schema(expected_conn) + expected = cls._schema_signature(expected_conn) + except sqlite3.Error as exc: + raise cls._incompatible_schema_error() from exc + if version != SCHEMA_VERSION or actual != expected: + raise cls._incompatible_schema_error() + @staticmethod def _create_current_schema(conn: sqlite3.Connection) -> None: conn.executescript( @@ -143,60 +151,8 @@ def _readonly_connection(path: Path) -> sqlite3.Connection: @staticmethod def _schema_signature(conn: sqlite3.Connection) -> dict[str, Any]: - tables = { - str(row[0]) - for row in conn.execute( - "SELECT name FROM sqlite_master " - "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" - ) - } - columns = { - table: tuple( - tuple(row) - for row in conn.execute(f'PRAGMA table_xinfo("{table}")') - ) - for table in sorted(tables) - } - indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} - foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} - for table in sorted(tables): - table_indexes = [] - for row in conn.execute(f'PRAGMA index_list("{table}")'): - name = str(row[1]) - origin = str(row[3]) - index_columns = tuple( - str(column[2]) - for column in conn.execute(f'PRAGMA index_info("{name}")') - ) - table_indexes.append( - ( - name if origin == "c" else None, - int(row[2]), - origin, - int(row[4]), - index_columns, - ) - ) - indexes[table] = tuple(sorted(table_indexes, key=repr)) - foreign_keys[table] = tuple( - sorted( - ( - str(row[2]), - str(row[3]), - str(row[4]), - str(row[5]), - str(row[6]), - str(row[7]), - ) - for row in conn.execute(f'PRAGMA foreign_key_list("{table}")') - ) - ) - return { - "tables": tables, - "columns": columns, - "indexes": indexes, - "foreign_keys": foreign_keys, - } + tables = regular_table_names(conn) + return sqlite_schema_signature(conn, tables) @staticmethod def _incompatible_schema_error() -> RuntimeError: diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 910580137..b57bc3e66 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -147,6 +147,16 @@ def create_projection_v2(workspace): return projection_dir +def connect_summary_database(path): + import sqlite_vec + + connection = sqlite3.connect(path) + connection.enable_load_extension(True) + sqlite_vec.load(connection) + connection.enable_load_extension(False) + return connection + + def catalog_file_count(workspace): with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] @@ -305,6 +315,29 @@ def test_cli_preflights_partial_projection_before_creating_catalog(tmp_path, cap assert not (workspace / "filesystem.sqlite").exists() +@pytest.mark.parametrize("catalog_state", ["missing", "zero_byte"]) +def test_cli_rejects_projection_pair_without_valid_catalog_before_mutating_workspace( + catalog_state, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + create_projection_v2(workspace) + if catalog_state == "zero_byte": + (workspace / "filesystem.sqlite").write_bytes(b"") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + if catalog_state == "missing": + assert not (workspace / "filesystem.sqlite").exists() + else: + assert (workspace / "filesystem.sqlite").stat().st_size == 0 + + @pytest.mark.parametrize( "mutation", [ @@ -411,6 +444,115 @@ def test_cli_rejects_pseudo_v2_projection_without_mutating_workspace( assert not (workspace / "filesystem.sqlite").exists() +def test_cli_rejects_noncanonical_vec0_distance_without_mutating_workspace( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + with connect_summary_database(projection_dir / "summary.sqlite") as connection: + connection.execute("DROP TABLE semantic_index_vec") + connection.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + "vec0(source_type TEXT partition key, " + "embedding float[3] distance_metric=cosine)" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_accepts_canonical_vec0_declaration_with_spacing_and_case_variation( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + with connect_summary_database(projection_dir / "summary.sqlite") as connection: + connection.execute("DROP TABLE semantic_index_vec") + connection.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + "VEC0( source_type TEXT PARTITION KEY , embedding FLOAT [ 3 ] )" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "identity_case", + [ + "summary_base_url", + "summary_model", + "cache_base_url", + "cache_model", + ], +) +def test_cli_rejects_noncanonical_persisted_embedding_identity_without_mutation( + identity_case, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + if identity_case.startswith("summary_"): + with sqlite3.connect(projection_dir / "summary.sqlite") as connection: + metadata = json.loads( + connection.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + if identity_case == "summary_base_url": + metadata["base_url"] = "HTTPS://EXAMPLE.INVALID/v1/" + else: + metadata["model"] = " test-embedding " + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + (json.dumps(metadata, sort_keys=True),), + ) + else: + base_url = ( + "HTTPS://EXAMPLE.INVALID/v1/" + if identity_case == "cache_base_url" + else "https://example.invalid/v1" + ) + model = ( + " test-embedding " + if identity_case == "cache_model" + else "test-embedding" + ) + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as connection: + connection.execute( + "INSERT INTO embedding_cache(" + "base_url, model, dimensions, text_hash, vector_blob" + ") VALUES (?, ?, 3, 'cache-hash', ?)", + (base_url, model, b"\0" * 12), + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + def test_cli_allows_fresh_listing_when_projection_is_completely_absent(tmp_path, capsys): from pageindex.filesystem.cli import main @@ -421,6 +563,10 @@ def test_cli_allows_fresh_listing_when_projection_is_completely_absent(tmp_path, assert (workspace / "filesystem.sqlite").is_file() assert not (workspace / "artifacts" / "projection_indexes").exists() + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert not (workspace / "artifacts" / "projection_indexes").exists() + def test_pageindex_is_the_only_public_python_entry_point_for_pifs(): import pageindex From 120b1ea5f7a92191da8d5ec287c436c0e74396bf Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 16:26:36 +0800 Subject: [PATCH 04/14] fix(filesystem): validate workspace consistency Fail closed before mutating an existing catalog and reject projection rows that do not reference active catalog files. Validate the canonical root and cross-database references at workspace open. Refs #47 --- .../filesystem/_workspace_consistency.py | 72 +++++++++++++++++++ pageindex/filesystem/core.py | 5 ++ pageindex/filesystem/store.py | 5 +- tests/test_pifs_simplification_acceptance.py | 63 ++++++++++++++++ 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 pageindex/filesystem/_workspace_consistency.py diff --git a/pageindex/filesystem/_workspace_consistency.py b/pageindex/filesystem/_workspace_consistency.py new file mode 100644 index 000000000..aa8a5ef9f --- /dev/null +++ b/pageindex/filesystem/_workspace_consistency.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + + +def _readonly_connection(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + return connection + + +def _inconsistent_workspace_error(detail: str) -> RuntimeError: + return RuntimeError( + f"Inconsistent PIFS workspace: {detail}; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + +def validate_catalog_root(catalog_path: str | Path) -> None: + try: + with _readonly_connection(Path(catalog_path)) as catalog: + root = catalog.execute( + "SELECT folder_id, parent_id, name FROM folders WHERE path = '/'" + ).fetchone() + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "the catalog canonical root folder could not be read" + ) from exc + if ( + root is None + or root["folder_id"] != "folder_root" + or root["parent_id"] is not None + or root["name"] != "/" + ): + raise _inconsistent_workspace_error( + "the catalog is missing its canonical root folder" + ) + + +def validate_workspace_consistency( + catalog_path: str | Path, + summary_path: str | Path, +) -> None: + catalog_path = Path(catalog_path) + validate_catalog_root(catalog_path) + try: + with _readonly_connection(catalog_path) as catalog: + active_file_refs = { + str(row[0]) + for row in catalog.execute( + "SELECT file_ref FROM files WHERE deleted_at IS NULL" + ) + } + with _readonly_connection(Path(summary_path)) as summary: + projected_file_refs = { + str(row[0]) + for row in summary.execute( + "SELECT file_ref FROM semantic_index_docs" + ) + } + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "catalog and Summary Projection references could not be read" + ) from exc + orphaned = sorted(projected_file_refs - active_file_refs) + if orphaned: + shown = ", ".join(orphaned[:5]) + raise _inconsistent_workspace_error( + "Summary Projection file_ref values do not reference active catalog files: " + f"{shown}" + ) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 05653a963..109bbcc14 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -87,8 +87,13 @@ def __init__( database_pair = projection_database_pair(self.summary_projection_index_dir) if database_pair is not None: from .semantic_projection import validate_projection_topology + from ._workspace_consistency import validate_workspace_consistency validate_projection_topology(self.summary_projection_index_dir) + validate_workspace_consistency( + self.workspace / "filesystem.sqlite", + database_pair[0], + ) self.store = SQLiteFileSystemStore(self.workspace) self.metadata = MetadataQueryEngine(self.store) self.summary_projection: Any | None = None diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py index b1e6afb1b..358247a3f 100644 --- a/pageindex/filesystem/store.py +++ b/pageindex/filesystem/store.py @@ -39,7 +39,7 @@ def connect(self) -> sqlite3.Connection: return conn def initialize_schema(self) -> None: - if self.db_path.exists() and self.db_path.stat().st_size > 0: + if self.db_path.exists() or self.db_path.is_symlink(): self.validate_existing_database(self.db_path) return with self.connect() as conn: @@ -64,6 +64,9 @@ def validate_existing_database(cls, db_path: str | Path) -> None: raise cls._incompatible_schema_error() from exc if version != SCHEMA_VERSION or actual != expected: raise cls._incompatible_schema_error() + from ._workspace_consistency import validate_catalog_root + + validate_catalog_root(db_path) @staticmethod def _create_current_schema(conn: sqlite3.Connection) -> None: diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index b57bc3e66..7d6742b64 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -157,6 +157,27 @@ def connect_summary_database(path): return connection +def insert_projection_document(summary_path, *, file_ref="orphan_projection_ref"): + import sqlite_vec + + with connect_summary_database(summary_path) as connection: + connection.execute( + """ + INSERT INTO semantic_index_docs( + rowid, file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + ) VALUES (1, ?, 'orphan-doc', 'markdown', 'orphan.md', + 'orphan-hash', 14, '{}') + """, + (file_ref,), + ) + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (1, 'markdown', ?)", + (sqlite_vec.serialize_float32([1.0, 0.0, 0.0]),), + ) + + def catalog_file_count(workspace): with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] @@ -338,6 +359,48 @@ def test_cli_rejects_projection_pair_without_valid_catalog_before_mutating_works assert (workspace / "filesystem.sqlite").stat().st_size == 0 +def test_cli_rejects_existing_zero_byte_catalog_without_projection_or_mutation( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + workspace.mkdir() + catalog = workspace / "filesystem.sqlite" + catalog.write_bytes(b"") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert catalog.stat().st_size == 0 + + +@pytest.mark.parametrize("mutation", ["orphan_projection", "missing_root"]) +def test_cli_rejects_inconsistent_catalog_projection_relationships_without_mutation( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + insert_projection_document(projection_dir / "summary.sqlite") + if mutation == "missing_root": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute("DELETE FROM folders WHERE path = '/'") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + @pytest.mark.parametrize( "mutation", [ From 30df72209d6bcceecc824d9b2cf26cb45f334699 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 16:39:55 +0800 Subject: [PATCH 05/14] fix(filesystem): reject broken projection symlinks Treat existing symlink entries as projection paths even when their targets are missing. Share the same presence rule between core preflight and topology validation so damaged projections fail closed without workspace mutation. Refs #47 --- pageindex/filesystem/_projection_topology.py | 9 +++-- pageindex/filesystem/core.py | 5 ++- tests/test_pifs_simplification_acceptance.py | 36 +++++++++++++++++++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/pageindex/filesystem/_projection_topology.py b/pageindex/filesystem/_projection_topology.py index 9418c8d1c..8c209548c 100644 --- a/pageindex/filesystem/_projection_topology.py +++ b/pageindex/filesystem/_projection_topology.py @@ -8,12 +8,17 @@ def projection_database_paths(index_dir: str | Path) -> tuple[Path, Path]: return index_dir / "summary.sqlite", index_dir / "embedding_cache.sqlite" +def projection_database_path_present(path: str | Path) -> bool: + path = Path(path) + return path.exists() or path.is_symlink() + + def projection_database_pair( index_dir: str | Path, ) -> tuple[Path, Path] | None: summary_path, cache_path = projection_database_paths(index_dir) - summary_exists = summary_path.exists() - cache_exists = cache_path.exists() + summary_exists = projection_database_path_present(summary_path) + cache_exists = projection_database_path_present(cache_path) if not summary_exists and not cache_exists: return None if not summary_exists or not cache_exists: diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 109bbcc14..8e63a4c03 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -10,6 +10,7 @@ from ._projection_topology import ( projection_database_pair, + projection_database_path_present, projection_database_paths, ) from .metadata import MetadataQueryEngine @@ -80,7 +81,9 @@ def __init__( summary_path, cache_path = projection_database_paths( self.summary_projection_index_dir ) - if summary_path.exists() or cache_path.exists(): + summary_present = projection_database_path_present(summary_path) + cache_present = projection_database_path_present(cache_path) + if summary_present or cache_present: SQLiteFileSystemStore.validate_existing_database( self.workspace / "filesystem.sqlite" ) diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 7d6742b64..13f2e4237 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -119,7 +119,9 @@ def workspace_state(workspace): return {} return { path.relative_to(workspace).as_posix(): ( - "directory" + f"symlink:{path.readlink()}" + if path.is_symlink() + else "directory" if path.is_dir() else hashlib.sha256(path.read_bytes()).hexdigest() ) @@ -359,6 +361,38 @@ def test_cli_rejects_projection_pair_without_valid_catalog_before_mutating_works assert (workspace / "filesystem.sqlite").stat().st_size == 0 +@pytest.mark.parametrize( + "projection_state", + ["both_broken", "broken_summary_only", "broken_summary_with_valid_cache"], +) +def test_cli_rejects_broken_projection_symlinks_without_mutating_workspace( + projection_state, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = workspace / "artifacts" / "projection_indexes" + if projection_state == "broken_summary_with_valid_cache": + create_projection_v2(workspace) + (projection_dir / "summary.sqlite").unlink() + else: + projection_dir.mkdir(parents=True) + (projection_dir / "summary.sqlite").symlink_to("missing-summary.sqlite") + if projection_state == "both_broken": + (projection_dir / "embedding_cache.sqlite").symlink_to( + "missing-cache.sqlite" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + def test_cli_rejects_existing_zero_byte_catalog_without_projection_or_mutation( tmp_path, capsys ): From 81ee9baebf3ddd9948193ae08ace371113f4a344 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 17:00:57 +0800 Subject: [PATCH 06/14] fix(filesystem): preserve actionable file locators Derive setmeta output from the updated document instead of echoing its input target. Reuse the tree scope collision mapping for physical browse and stable locators so duplicate display names remain ranked and resolvable. Refs #47 --- pageindex/filesystem/cli.py | 2 +- pageindex/filesystem/core.py | 33 ++--- tests/test_pifs_cli.py | 6 +- tests/test_pifs_simplification_acceptance.py | 121 +++++++++++++++++++ 4 files changed, 146 insertions(+), 16 deletions(-) diff --git a/pageindex/filesystem/cli.py b/pageindex/filesystem/cli.py index 2ebbed2f5..47b9610f1 100644 --- a/pageindex/filesystem/cli.py +++ b/pageindex/filesystem/cli.py @@ -384,7 +384,7 @@ def _run_setmeta(argv: list[str], *, workspace: str) -> int: filesystem = _filesystem_from_workspace(workspace) info = filesystem.set_metadata(args.target, metadata, clear=args.clear) document = { - "path": args.target if args.target.startswith("/") else info.get("path"), + "path": info.get("path"), "document_id": info.get("external_id"), "title": info.get("title"), "status": info.get("pageindex_tree_status"), diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 8e63a4c03..e55ff7c8f 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -699,7 +699,20 @@ def set_metadata( metadata=replacement, metadata_status=dict(info.get("metadata_status") or {}), ) - return self.store.file_info(file_ref) + updated = self.store.file_info(file_ref) + entry = self.store.get_file(file_ref) + folder_paths = [folder["path"] for folder in updated.get("folders", [])] + folder_path = self._preferred_folder_path( + folder_paths, + entry.folder_path, + entry.folder_path, + ) + updated["path"] = self._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + return updated def pageindex_structure( self, @@ -1470,19 +1483,11 @@ def _stable_file_locator( ).strip() if not title: raise RuntimeError(f"browse cannot build a virtual path for {file_ref}: missing title") - target = self._join_virtual_file_path(folder_path, title.strip("/")) - try: - resolved_file_ref = self.store.resolve_file_ref(target) - except KeyError as exc: - raise RuntimeError( - f"browse produced an unresolved virtual path for {file_ref}: {target}" - ) from exc - if resolved_file_ref != file_ref: - raise RuntimeError( - "browse produced a non-idempotent virtual path: " - f"{target} resolved to {resolved_file_ref}, expected {file_ref}" - ) - return target + return self.scope_file_locator( + self.resolve_query_scope(folder_path), + file_ref, + title, + ) @staticmethod def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: diff --git a/tests/test_pifs_cli.py b/tests/test_pifs_cli.py index 5453504b3..34e35bbae 100644 --- a/tests/test_pifs_cli.py +++ b/tests/test_pifs_cli.py @@ -239,7 +239,11 @@ def test_cli_setmeta_clear_uses_empty_object(monkeypatch, capsys, tmp_path): class FakeSetMetaFileSystem(FakeFileSystem): def set_metadata(self, target, metadata, *, clear=False): calls.append((self.workspace, target, metadata, clear)) - return {"file_ref": "file_report", "metadata": metadata} + return { + "path": "/documents/report.md", + "file_ref": "file_report", + "metadata": metadata, + } monkeypatch.setattr(cli, "PageIndexFileSystem", FakeSetMetaFileSystem) diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 13f2e4237..89bc03fcb 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -972,6 +972,84 @@ def test_setmeta_translates_identity_at_the_cli_boundary( assert document["metadata"]["year"] == 2024 +def test_setmeta_by_file_ref_returns_an_actionable_path_without_leaking_identity( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + file_ref = connection.execute("SELECT file_ref FROM files").fetchone()[0] + + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + file_ref, + '{"year": 2024}', + ] + ) == 0 + output = capsys.readouterr().out + document = json.loads(output) + + assert document["path"] == "/documents/notes.md" + assert file_ref not in output + assert main(["--workspace", str(workspace), "stat", document["path"]]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == document["path"] + + +@pytest.mark.parametrize("operation", ["replace", "clear"]) +def test_metadata_scoped_setmeta_returns_the_post_update_actionable_path( + operation, tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + old_path = "/documents/@year=2024/notes.md" + command = ["--workspace", str(workspace), "setmeta"] + if operation == "clear": + command.extend(["--clear", old_path]) + else: + command.extend([old_path, '{"year": 2025}']) + + assert main(command) == 0 + document = json.loads(capsys.readouterr().out) + + assert document["path"] == "/documents/notes.md" + assert document["path"] != old_path + if operation == "clear": + assert "year" not in document["metadata"] + else: + assert document["metadata"]["year"] == 2025 + assert main(["--workspace", str(workspace), "stat", document["path"]]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == document["path"] + + def test_tree_file_records_use_only_command_identity_names( tmp_path, monkeypatch, capsys ): @@ -1174,6 +1252,49 @@ def test_cli_add_rejects_duplicate_target_without_changing_owned_document( assert owned[0].read_text(encoding="utf-8") == "first alpha evidence" +def test_physical_browse_preserves_collision_aware_tree_locators( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for filename, content in ( + ("alpha.md", "alpha evidence"), + ("beta.md", "beta evidence"), + ): + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE file_folders SET metadata_json = ?", + (json.dumps({"display_name": "same.md"}),), + ) + + assert main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) == 0 + tree_files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + tree_paths = {row["path"] for row in tree_files} + assert tree_paths == { + "/documents/same.md~1", + "/documents/same.md~2", + } + + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 0 + browse_documents = json.loads(capsys.readouterr().out)["data"]["documents"] + browse_paths = [row["path"] for row in browse_documents] + + assert len(browse_documents) == 2 + assert len(set(browse_paths)) == 2 + assert set(browse_paths) == tree_paths + for path in browse_paths: + assert main(["--workspace", str(workspace), "stat", path]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == path + + def test_browse_recursively_returns_one_global_ranked_page_of_ten( tmp_path, monkeypatch, capsys ): From 0447889e00645beeaf26357ab9c3933f7b85e978 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 17:36:09 +0800 Subject: [PATCH 07/14] fix(filesystem): complete projections for registrations Open or create the Summary Projection for Python registrations, require ready status before returning, and roll back fresh batches when projection completion fails. Clean partial fresh projection databases when schema creation aborts. --- pageindex/filesystem/core.py | 12 +- pageindex/filesystem/semantic_projection.py | 24 ++- tests/pifs_markdown_fixture.py | 18 ++ tests/test_pifs_simplification_acceptance.py | 179 +++++++++++++++++++ 4 files changed, 222 insertions(+), 11 deletions(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index e55ff7c8f..a5f7a24e3 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -227,6 +227,8 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: record for record in records if record["file_ref"] not in preexisting_file_refs ] try: + if records: + self._ensure_add_completion_defaults() self._register_custom_metadata_fields(records) self.store.insert_files(records) for record in records: @@ -237,6 +239,7 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: metadata=record["metadata"], metadata_status=record["metadata_status"], ) + self._require_add_summary_projection_ready(record) self._sync_owned_raw_artifact(record) except KeyError: continue @@ -1217,19 +1220,18 @@ def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: if not summary: return False if self.summary_projection is None: - self._refresh_record_metadata_status(record) - return True + raise RuntimeError("PIFS Summary Projection is not open") try: result = self.summary_projection.upsert_summary(record) except Exception as exc: summary_index["status"] = "failed" summary_index["error"] = str(exc) self._refresh_record_metadata_status(record) - return True + raise RuntimeError( + f"PIFS failed to build summary projection index: {exc}" + ) from exc summary_index.clear() summary_index.update({"requested": True, **result}) - if summary_index.get("status") != "ready": - summary_index["status"] = "ready" self._refresh_record_metadata_status(record) return True diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index cd42b0f78..0fbaef15e 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -12,7 +12,7 @@ normalize_base_url, normalize_model, ) -from ._projection_topology import projection_database_pair +from ._projection_topology import projection_database_pair, projection_database_paths from ._sqlite_schema import ( normalized_table_sql, regular_table_names, @@ -96,14 +96,26 @@ def __init__( self.embedding_cache = EmbeddingCache(cache_path, create=False) elif create: self.index_dir.mkdir(parents=True, exist_ok=True) - self.index.reset( - dimension=self.profile.dimensions, - metadata=self.profile.identity, - ) - self.embedding_cache = EmbeddingCache(cache_path, create=True) + try: + self.index.reset( + dimension=self.profile.dimensions, + metadata=self.profile.identity, + ) + self.embedding_cache = EmbeddingCache(cache_path, create=True) + except Exception: + self._cleanup_failed_create() + raise else: raise RuntimeError("PIFS Summary Projection is not available") + def _cleanup_failed_create(self) -> None: + for path in projection_database_paths(self.index_dir): + for suffix in ("", "-journal", "-shm", "-wal"): + try: + Path(f"{path}{suffix}").unlink() + except FileNotFoundError: + continue + def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: summary = str((record.get("metadata") or {}).get("summary") or "").strip() if not summary: diff --git a/tests/pifs_markdown_fixture.py b/tests/pifs_markdown_fixture.py index cc7dc3525..deda5bcac 100644 --- a/tests/pifs_markdown_fixture.py +++ b/tests/pifs_markdown_fixture.py @@ -38,6 +38,14 @@ def get_page_content(self, doc_id, pages): return json.dumps(self.documents[doc_id]["pages"]) +class FakeEmbeddingClient: + def __init__(self, dimensions: int): + self.dimensions = dimensions + + def embed(self, texts: list[str]) -> list[list[float]]: + return [[1.0, *([0.0] * (self.dimensions - 1))] for _ in texts] + + def register_markdown( filesystem, tmp_path: Path, @@ -53,6 +61,16 @@ def register_markdown( client = FakePageIndexClient() filesystem._test_pageindex_client = client filesystem._pageindex_client = lambda: client + if filesystem.summary_projection is None: + from pageindex.filesystem.semantic_projection import SummaryProjection + + profile = filesystem._summary_embedding_profile() + filesystem.summary_projection = SummaryProjection( + filesystem.summary_projection_index_dir, + profile=profile, + embedder=FakeEmbeddingClient(profile.dimensions), + create=True, + ) filename = title if title and title.endswith(".md") else f"{external_id}.md" source = tmp_path / filename source.write_text(text or f"{external_id} alpha evidence", encoding="utf-8") diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 89bc03fcb..70af86f72 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -185,6 +185,27 @@ def catalog_file_count(workspace): return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] +def open_test_filesystem(workspace): + from pageindex.filesystem import PageIndexFileSystem + + return PageIndexFileSystem( + workspace, + summary_projection_embedding_base_url="https://example.invalid/v1", + summary_projection_embedding_model="test-embedding", + summary_projection_embedding_dimensions=3, + summary_projection_embedding_api_key="runtime-secret", + ) + + +def nested_path_with_length(root, target_length): + path = Path(root) + while len(str(path)) < target_length: + component_length = min(200, target_length - len(str(path)) - 1) + path /= "x" * component_length + assert len(str(path)) == target_length + return path + + def test_cli_rejects_removed_ls_command_with_structured_error(tmp_path, capsys): from pageindex.filesystem.cli import main @@ -1223,6 +1244,164 @@ def create(self, *, model, input, dimensions): ).fetchone()[0] == 0 +def test_register_file_completes_summary_projection_for_immediate_and_reopen_browse( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + + projection_dir = workspace / "artifacts" / "projection_indexes" + assert (projection_dir / "summary.sqlite").is_file() + assert (projection_dir / "embedding_cache.sqlite").is_file() + assert filesystem.store.get_file(file_ref).metadata_status["summary_projection"]["status"] == "ready" + assert file_ref in { + row["file_ref"] + for row in filesystem.browse_semantic_files("/documents", "alpha")["data"] + } + + reopened = open_test_filesystem(workspace) + assert file_ref in { + row["file_ref"] + for row in reopened.browse_semantic_files("/documents", "alpha")["data"] + } + + +def test_register_file_opens_existing_projection_and_upserts_second_document( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + first_ref = filesystem.register_file( + storage_uri=first.as_uri(), + folder_path="/documents", + title=first.name, + content_type="text/markdown", + content=first.read_text(encoding="utf-8"), + ) + + reopened = open_test_filesystem(workspace) + second_ref = reopened.register_file( + storage_uri=second.as_uri(), + folder_path="/documents", + title=second.name, + content_type="text/markdown", + content=second.read_text(encoding="utf-8"), + ) + + assert reopened.store.get_file(second_ref).metadata_status["summary_projection"]["status"] == "ready" + assert second_ref in { + row["file_ref"] + for row in reopened.browse_semantic_files("/documents", "beta")["data"] + } + with connect_summary_database( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] == 2 + assert connection.execute("SELECT COUNT(*) FROM semantic_index_vec").fetchone()[0] == 2 + assert set(reopened.store.file_refs_for_scope()) == {first_ref, second_ref} + + +def test_register_files_rolls_back_new_batch_when_second_projection_upsert_fails( + tmp_path, monkeypatch +): + import openai + + install_network_fakes(monkeypatch) + + class FailingSecondOpenAI(FakeOpenAI): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = 0 + + def create(self, *, model, input, dimensions): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingSecondOpenAI) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + filesystem.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + } + for source in (first, second) + ] + ) + + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "text").glob("*")) + assert not list((workspace / "artifacts" / "raw").glob("*")) + with connect_summary_database( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] == 0 + assert connection.execute("SELECT COUNT(*) FROM semantic_index_vec").fetchone()[0] == 0 + + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_rolls_back_partial_projection_creation_and_reopens( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + # SQLite's Unix VFS accepts summary.sqlite at this path length but rejects + # the eight-character-longer embedding_cache.sqlite path. + workspace = nested_path_with_length(tmp_path / "long-workspace", 453) + filesystem = open_test_filesystem(workspace) + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + + with pytest.raises(sqlite3.OperationalError, match="unable to open database file"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + + projection_dir = workspace / "artifacts" / "projection_indexes" + assert not (projection_dir / "summary.sqlite").exists() + assert not (projection_dir / "embedding_cache.sqlite").exists() + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "text").glob("*")) + assert not list((workspace / "artifacts" / "raw").glob("*")) + + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + + def test_cli_add_rejects_duplicate_target_without_changing_owned_document( tmp_path, monkeypatch, capsys ): From eea969c1bb8f32b040805ba9d435d195fb210a54 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 18:00:46 +0800 Subject: [PATCH 08/14] fix(filesystem): restore registration rollback baseline Track only folders, metadata fields, and embedding cache keys introduced by a failed registration batch, then remove them without disturbing preexisting workspace state. Generalize Summary Projection lifecycle readiness messages for add and registration. --- pageindex/filesystem/core.py | 81 +++++++-- pageindex/filesystem/semantic_projection.py | 71 ++++++++ pageindex/filesystem/store.py | 20 +++ tests/test_pifs_simplification_acceptance.py | 170 ++++++++++++++++++- 4 files changed, 320 insertions(+), 22 deletions(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index a5f7a24e3..6847ed20a 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -156,7 +156,7 @@ def add_file( ) if self.store.file_basename_exists_in_folder(folder_path, filename): raise FileExistsError(f"File already exists at {virtual_path}") - self._ensure_add_completion_defaults() + self._ensure_summary_projection() add_created_folder_paths = self._add_created_folder_paths(folder_path) file_ref = make_file_ref(virtual_path.strip("/")) uploads_dir = self.workspace / "artifacts" / "uploads" @@ -202,7 +202,7 @@ def add_file( metadata=record["metadata"], metadata_status=record["metadata_status"], ) - self._require_add_summary_projection_ready(record) + self._require_summary_projection_ready(record, operation="add") self._sync_owned_raw_artifact(record) self._ensure_add_semantic_retrieval_ready() except Exception: @@ -226,9 +226,26 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: new_records = [ record for record in records if record["file_ref"] not in preexisting_file_refs ] + created_folder_paths = sorted( + { + path + for record in new_records + for path in self._add_created_folder_paths(record["folder_path"]) + }, + key=lambda path: (path.count("/"), path), + ) + new_metadata_fields = { + name + for name in self._custom_metadata_field_names(records) + if not self.store.metadata_field_exists(name) + } + batch_cache_keys: set[Any] = set() + preexisting_cache_keys: set[Any] = set() try: if records: - self._ensure_add_completion_defaults() + projection = self._ensure_summary_projection() + batch_cache_keys = projection.cache_keys_for_records(new_records) + preexisting_cache_keys = projection.existing_cache_keys(batch_cache_keys) self._register_custom_metadata_fields(records) self.store.insert_files(records) for record in records: @@ -239,20 +256,28 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: metadata=record["metadata"], metadata_status=record["metadata_status"], ) - self._require_add_summary_projection_ready(record) + self._require_summary_projection_ready( + record, + operation="registration", + ) self._sync_owned_raw_artifact(record) except KeyError: continue except Exception: self._cleanup_add_summary_projection(new_records) + self._cleanup_summary_projection_cache( + batch_cache_keys - preexisting_cache_keys + ) for record in new_records: self._cleanup_add_catalog_record(str(record["file_ref"])) + self._cleanup_add_created_folders(created_folder_paths) + self._cleanup_new_metadata_fields(new_metadata_fields) self._cleanup_failed_register_artifacts(records) raise return [record["file_ref"] for record in records] - def _ensure_add_completion_defaults(self) -> None: - self._open_summary_projection(create=True) + def _ensure_summary_projection(self) -> Any: + return self._open_summary_projection(create=True) def _ensure_add_semantic_retrieval_ready(self) -> None: projection = self._open_summary_projection(create=False) @@ -1009,14 +1034,21 @@ def _require_add_pageindex_ready(self, record: dict[str, Any]) -> None: ) raise RuntimeError(f"pifs add failed to build PageIndex tree: {message}") - def _require_add_summary_projection_ready(self, record: dict[str, Any]) -> None: + def _require_summary_projection_ready( + self, + record: dict[str, Any], + *, + operation: str, + ) -> None: summary_projection = (record.get("metadata_status") or {}).get("summary_projection") if not summary_projection or not summary_projection.get("requested"): - raise RuntimeError("pifs add requires a requested summary projection index") + raise RuntimeError( + f"PIFS {operation} requires a requested summary projection index" + ) if summary_projection.get("status") != "ready": detail = summary_projection.get("error") or summary_projection.get("status") raise RuntimeError( - f"pifs add failed to build summary projection index: {detail}" + f"PIFS {operation} failed to build summary projection index: {detail}" ) def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: @@ -1281,6 +1313,14 @@ def _cleanup_add_summary_projection(self, records: list[dict[str, Any]]) -> None except Exception: continue + def _cleanup_summary_projection_cache(self, keys: set[Any]) -> None: + if not keys or self.summary_projection is None: + return + try: + self.summary_projection.delete_cache_keys(keys) + except Exception: + return + def _cleanup_add_created_folders(self, folder_paths: list[str]) -> None: for folder_path in reversed(folder_paths): try: @@ -1288,6 +1328,13 @@ def _cleanup_add_created_folders(self, folder_paths: list[str]) -> None: except Exception: continue + def _cleanup_new_metadata_fields(self, names: set[str]) -> None: + for name in sorted(names): + try: + self.store.delete_metadata_field_if_unreferenced(name) + except Exception: + continue + def _pageindex_cache_doc_ids(self) -> set[str]: workspace = self.pageindex_client_workspace doc_ids = {path.stem for path in workspace.glob("*.json") if path.name != "_meta.json"} @@ -1504,15 +1551,23 @@ def _validate_register_metadata(metadata: dict[str, Any]) -> None: raise ValueError("summary is managed by PageIndex doc_description") def _register_custom_metadata_fields(self, records: list[dict[str, Any]]) -> None: - fields = {} + fields = { + name: {} + for name in self._custom_metadata_field_names(records) + if not self.store.metadata_field_exists(name) + } + if fields: + self.metadata.register_schema({"fields": fields}, source="user") + + def _custom_metadata_field_names(self, records: list[dict[str, Any]]) -> set[str]: + fields = set() for record in records: for name in SQLiteFileSystemStore.indexed_metadata_values( record.get("metadata", {}) ): if self.metadata.FIELD_RE.match(str(name)): - fields[str(name)] = {} - if fields: - self.metadata.register_schema({"fields": fields}, source="user") + fields.add(str(name)) + return fields @staticmethod def _metadata_status_state(*, metadata: dict[str, Any]) -> dict[str, Any]: diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index 0fbaef15e..7a87f6343 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -27,6 +27,7 @@ SUMMARY_INDEX_NAME = "summary" +EmbeddingCacheKey = tuple[str, str, int, str] @dataclass(frozen=True) @@ -149,6 +150,28 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: def delete_summary(self, file_ref: str) -> int: return self.index.delete_file_refs([file_ref]) + def cache_keys_for_records( + self, + records: list[dict[str, Any]], + ) -> set[EmbeddingCacheKey]: + summaries = [ + str((record.get("metadata") or {}).get("summary") or "").strip() + for record in records + ] + return self.embedding_cache.keys_for_texts( + [summary for summary in summaries if summary], + profile=self.profile, + ) + + def existing_cache_keys( + self, + keys: set[EmbeddingCacheKey], + ) -> set[EmbeddingCacheKey]: + return self.embedding_cache.existing_keys(keys) + + def delete_cache_keys(self, keys: set[EmbeddingCacheKey]) -> int: + return self.embedding_cache.delete_keys(keys) + def search( self, query: str, @@ -295,6 +318,54 @@ def embed_texts( cached[hashes[index]] = vector return [cached[text_hash] for text_hash in hashes] + @staticmethod + def keys_for_texts( + texts: list[str], + *, + profile: SummaryEmbeddingProfile, + ) -> set[EmbeddingCacheKey]: + return { + ( + str(profile.base_url), + profile.model, + profile.dimensions, + SQLiteVecSemanticIndex.text_hash(text), + ) + for text in texts + } + + def existing_keys( + self, + keys: set[EmbeddingCacheKey], + ) -> set[EmbeddingCacheKey]: + existing: set[EmbeddingCacheKey] = set() + with self.connect(read_only=True) as connection: + for key in sorted(keys): + if connection.execute( + """ + SELECT 1 + FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + key, + ).fetchone() is not None: + existing.add(key) + return existing + + def delete_keys(self, keys: set[EmbeddingCacheKey]) -> int: + if not keys: + return 0 + with self.connect() as connection: + before = connection.total_changes + connection.executemany( + """ + DELETE FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + sorted(keys), + ) + return connection.total_changes - before + def _validate(self) -> None: try: with self.connect(read_only=True) as connection: diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py index 358247a3f..31450e67d 100644 --- a/pageindex/filesystem/store.py +++ b/pageindex/filesystem/store.py @@ -555,6 +555,26 @@ def metadata_field_exists(self, name: str) -> bool: ).fetchone() return row is not None + def delete_metadata_field_if_unreferenced(self, name: str) -> bool: + with self.connect() as conn: + row = conn.execute( + "SELECT field_id FROM metadata_fields WHERE name = ?", + (name,), + ).fetchone() + if row is None: + return False + referenced = conn.execute( + "SELECT 1 FROM metadata_values WHERE field_id = ? LIMIT 1", + (row["field_id"],), + ).fetchone() + if referenced is not None: + return False + cursor = conn.execute( + "DELETE FROM metadata_fields WHERE field_id = ?", + (row["field_id"],), + ) + return cursor.rowcount > 0 + def list_metadata_fields(self) -> list[MetadataField]: with self.connect() as conn: rows = conn.execute( diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 70af86f72..ae62cde89 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -185,6 +185,102 @@ def catalog_file_count(workspace): return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] +def registration_logical_state(workspace): + workspace = Path(workspace) + catalog_path = workspace / "filesystem.sqlite" + projection_dir = workspace / "artifacts" / "projection_indexes" + summary_path = projection_dir / "summary.sqlite" + cache_path = projection_dir / "embedding_cache.sqlite" + + with sqlite3.connect(catalog_path) as connection: + catalog = { + "files": connection.execute( + """ + SELECT file_ref, external_id, storage_uri, title, descriptor, + content_type, source_type, fingerprint, text_artifact_path, + raw_artifact_path, pageindex_doc_id, pageindex_tree_status, + metadata_json, metadata_status_json, deleted_at + FROM files + ORDER BY file_ref + """ + ).fetchall(), + "folders": connection.execute( + """ + SELECT folder_id, parent_id, name, path, description, kind, metadata_json + FROM folders + ORDER BY path + """ + ).fetchall(), + "metadata_fields": connection.execute( + """ + SELECT field_id, name, description, source + FROM metadata_fields + ORDER BY name + """ + ).fetchall(), + "metadata_values": connection.execute( + """ + SELECT file_ref, field_id, value_text + FROM metadata_values + ORDER BY file_ref, field_id, value_text + """ + ).fetchall(), + } + + summary = {"docs": [], "vec": []} + if summary_path.is_file(): + with connect_summary_database(summary_path) as connection: + summary = { + "docs": connection.execute( + """ + SELECT rowid, file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + FROM semantic_index_docs + ORDER BY rowid + """ + ).fetchall(), + "vec": connection.execute( + """ + SELECT rowid, source_type, hex(embedding) + FROM semantic_index_vec + ORDER BY rowid + """ + ).fetchall(), + } + + cache = [] + if cache_path.is_file(): + with sqlite3.connect(cache_path) as connection: + cache = connection.execute( + """ + SELECT base_url, model, dimensions, text_hash, hex(vector_blob) + FROM embedding_cache + ORDER BY base_url, model, dimensions, text_hash + """ + ).fetchall() + + artifacts = {} + for artifact_kind in ("text", "raw"): + artifact_dir = workspace / "artifacts" / artifact_kind + if artifact_dir.is_dir(): + artifacts[artifact_kind] = { + path.relative_to(artifact_dir).as_posix(): hashlib.sha256( + path.read_bytes() + ).hexdigest() + for path in sorted(artifact_dir.rglob("*")) + if path.is_file() + } + else: + artifacts[artifact_kind] = {} + + return { + "catalog": catalog, + "summary": summary, + "cache": cache, + "artifacts": artifacts, + } + + def open_test_filesystem(workspace): from pageindex.filesystem import PageIndexFileSystem @@ -1342,35 +1438,91 @@ def create(self, *, model, input, dimensions): second = tmp_path / "second.md" first.write_text("alpha registration evidence", encoding="utf-8") second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) with pytest.raises(RuntimeError, match="summary projection.*second registration"): filesystem.register_files( [ { "storage_uri": source.as_uri(), - "folder_path": "/documents", + "folder_path": "/documents/new", "title": source.name, "content_type": "text/markdown", "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, } for source in (first, second) ] ) - assert catalog_file_count(workspace) == 0 - assert not list((workspace / "artifacts" / "text").glob("*")) - assert not list((workspace / "artifacts" / "raw").glob("*")) - with connect_summary_database( - workspace / "artifacts" / "projection_indexes" / "summary.sqlite" - ) as connection: - assert connection.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] == 0 - assert connection.execute("SELECT COUNT(*) FROM semantic_index_vec").fetchone()[0] == 0 + assert registration_logical_state(workspace) == baseline reopened = open_test_filesystem(workspace) assert reopened.store.folder_info("/")["path"] == "/" assert reopened.store.file_refs_for_scope() == [] +def test_register_files_rollback_preserves_preexisting_catalog_projection_and_cache( + tmp_path, monkeypatch +): + import openai + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + existing_dir = tmp_path / "existing" + batch_dir = tmp_path / "batch" + existing_dir.mkdir() + batch_dir.mkdir() + existing = existing_dir / "shared.md" + shared = batch_dir / "shared.md" + second = batch_dir / "second.md" + existing.write_text("alpha registration evidence", encoding="utf-8") + shared.write_text(existing.read_text(encoding="utf-8"), encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=existing.as_uri(), + folder_path="/documents", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + filesystem.metadata.register_schema( + {"fields": {"year": {"description": "preexisting definition"}}}, + source="manual", + ) + baseline = registration_logical_state(workspace) + + class FailingBetaOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + if any("beta" in str(text).lower() for text in input): + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingBetaOpenAI) + reopened = open_test_filesystem(workspace) + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + reopened.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024, "batch_only": "new"}, + } + for source in (shared, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + strict_reopen = open_test_filesystem(workspace) + assert strict_reopen.store.get_file(existing_ref).metadata["year"] == 2023 + + def test_register_file_rolls_back_partial_projection_creation_and_reopens( tmp_path, monkeypatch ): From 867fd8f778f6f54d3ce4e6d71968e889f2348b1d Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 18:17:03 +0800 Subject: [PATCH 09/14] fix(filesystem): restore PageIndex registration artifacts Capture PageIndex document IDs before registration preparation and remove only IDs introduced by a failed batch. Prepare records inside the rollback boundary so PageIndex after-write failures also restore the logical workspace baseline. --- pageindex/filesystem/core.py | 83 +++++++++++------- pageindex/filesystem/semantic_projection.py | 20 ++--- tests/test_pifs_simplification_acceptance.py | 92 ++++++++++++++++---- 3 files changed, 134 insertions(+), 61 deletions(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 6847ed20a..41691e799 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from ..client import PageIndexClient + from .semantic_projection import _EmbeddingCacheKey PROJECTION_INDEX_STATUSES = { "not_indexed", @@ -207,11 +208,11 @@ def add_file( self._ensure_add_semantic_retrieval_ready() except Exception: if catalog_inserted: - self._cleanup_add_catalog_record(file_ref) - self._cleanup_add_summary_projection(records) + self._cleanup_catalog_record(file_ref) + self._cleanup_summary_projection_records(records) self._cleanup_failed_register_artifacts(records) - self._cleanup_add_pageindex_cache(records, preexisting_pageindex_doc_ids) - self._cleanup_add_created_folders(add_created_folder_paths) + self._cleanup_pageindex_cache(records, preexisting_pageindex_doc_ids) + self._cleanup_created_folders(add_created_folder_paths) if final_dir_created: shutil.rmtree(final_dir, ignore_errors=True) raise @@ -221,27 +222,35 @@ def add_file( return info def register_files(self, files: list[dict[str, Any]]) -> list[str]: - records = [self._prepare_file_record(file) for file in files] - preexisting_file_refs = self._existing_file_refs(records) - new_records = [ - record for record in records if record["file_ref"] not in preexisting_file_refs - ] - created_folder_paths = sorted( - { - path - for record in new_records - for path in self._add_created_folder_paths(record["folder_path"]) - }, - key=lambda path: (path.count("/"), path), - ) - new_metadata_fields = { - name - for name in self._custom_metadata_field_names(records) - if not self.store.metadata_field_exists(name) - } - batch_cache_keys: set[Any] = set() - preexisting_cache_keys: set[Any] = set() + preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() + records: list[dict[str, Any]] = [] + new_records: list[dict[str, Any]] = [] + created_folder_paths: list[str] = [] + new_metadata_fields: set[str] = set() + batch_cache_keys: set[_EmbeddingCacheKey] = set() + preexisting_cache_keys: set[_EmbeddingCacheKey] = set() try: + for file in files: + records.append(self._prepare_file_record(file)) + preexisting_file_refs = self._existing_file_refs(records) + new_records = [ + record + for record in records + if record["file_ref"] not in preexisting_file_refs + ] + created_folder_paths = sorted( + { + path + for record in new_records + for path in self._add_created_folder_paths(record["folder_path"]) + }, + key=lambda path: (path.count("/"), path), + ) + new_metadata_fields = { + name + for name in self._custom_metadata_field_names(records) + if not self.store.metadata_field_exists(name) + } if records: projection = self._ensure_summary_projection() batch_cache_keys = projection.cache_keys_for_records(new_records) @@ -264,15 +273,19 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: except KeyError: continue except Exception: - self._cleanup_add_summary_projection(new_records) + self._cleanup_summary_projection_records(new_records) self._cleanup_summary_projection_cache( batch_cache_keys - preexisting_cache_keys ) for record in new_records: - self._cleanup_add_catalog_record(str(record["file_ref"])) - self._cleanup_add_created_folders(created_folder_paths) + self._cleanup_catalog_record(str(record["file_ref"])) + self._cleanup_created_folders(created_folder_paths) self._cleanup_new_metadata_fields(new_metadata_fields) self._cleanup_failed_register_artifacts(records) + self._cleanup_pageindex_cache( + records, + preexisting_pageindex_doc_ids, + ) raise return [record["file_ref"] for record in records] @@ -1281,7 +1294,7 @@ def _cleanup_failed_register_artifacts(self, records: list[dict[str, Any]]) -> N if record.get("_pifs_owned_raw_artifact") and record.get("raw_artifact_path"): self._unlink_artifact(record["raw_artifact_path"]) - def _cleanup_add_catalog_record(self, file_ref: str) -> None: + def _cleanup_catalog_record(self, file_ref: str) -> None: try: self.store.delete_file(file_ref) except Exception: @@ -1300,7 +1313,10 @@ def _existing_file_refs(self, records: list[dict[str, Any]]) -> set[str]: existing.add(file_ref) return existing - def _cleanup_add_summary_projection(self, records: list[dict[str, Any]]) -> None: + def _cleanup_summary_projection_records( + self, + records: list[dict[str, Any]], + ) -> None: projection = self.summary_projection if projection is None: return @@ -1313,7 +1329,10 @@ def _cleanup_add_summary_projection(self, records: list[dict[str, Any]]) -> None except Exception: continue - def _cleanup_summary_projection_cache(self, keys: set[Any]) -> None: + def _cleanup_summary_projection_cache( + self, + keys: set[_EmbeddingCacheKey], + ) -> None: if not keys or self.summary_projection is None: return try: @@ -1321,7 +1340,7 @@ def _cleanup_summary_projection_cache(self, keys: set[Any]) -> None: except Exception: return - def _cleanup_add_created_folders(self, folder_paths: list[str]) -> None: + def _cleanup_created_folders(self, folder_paths: list[str]) -> None: for folder_path in reversed(folder_paths): try: self.store.delete_empty_folder(folder_path) @@ -1349,7 +1368,7 @@ def _pageindex_cache_doc_ids(self) -> set[str]: doc_ids.update(str(doc_id) for doc_id in payload) return doc_ids - def _cleanup_add_pageindex_cache( + def _cleanup_pageindex_cache( self, records: list[dict[str, Any]], preexisting_doc_ids: set[str], diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index 7a87f6343..4347469f6 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -27,7 +27,7 @@ SUMMARY_INDEX_NAME = "summary" -EmbeddingCacheKey = tuple[str, str, int, str] +_EmbeddingCacheKey = tuple[str, str, int, str] @dataclass(frozen=True) @@ -153,7 +153,7 @@ def delete_summary(self, file_ref: str) -> int: def cache_keys_for_records( self, records: list[dict[str, Any]], - ) -> set[EmbeddingCacheKey]: + ) -> set[_EmbeddingCacheKey]: summaries = [ str((record.get("metadata") or {}).get("summary") or "").strip() for record in records @@ -165,11 +165,11 @@ def cache_keys_for_records( def existing_cache_keys( self, - keys: set[EmbeddingCacheKey], - ) -> set[EmbeddingCacheKey]: + keys: set[_EmbeddingCacheKey], + ) -> set[_EmbeddingCacheKey]: return self.embedding_cache.existing_keys(keys) - def delete_cache_keys(self, keys: set[EmbeddingCacheKey]) -> int: + def delete_cache_keys(self, keys: set[_EmbeddingCacheKey]) -> int: return self.embedding_cache.delete_keys(keys) def search( @@ -323,7 +323,7 @@ def keys_for_texts( texts: list[str], *, profile: SummaryEmbeddingProfile, - ) -> set[EmbeddingCacheKey]: + ) -> set[_EmbeddingCacheKey]: return { ( str(profile.base_url), @@ -336,9 +336,9 @@ def keys_for_texts( def existing_keys( self, - keys: set[EmbeddingCacheKey], - ) -> set[EmbeddingCacheKey]: - existing: set[EmbeddingCacheKey] = set() + keys: set[_EmbeddingCacheKey], + ) -> set[_EmbeddingCacheKey]: + existing: set[_EmbeddingCacheKey] = set() with self.connect(read_only=True) as connection: for key in sorted(keys): if connection.execute( @@ -352,7 +352,7 @@ def existing_keys( existing.add(key) return existing - def delete_keys(self, keys: set[EmbeddingCacheKey]) -> int: + def delete_keys(self, keys: set[_EmbeddingCacheKey]) -> int: if not keys: return 0 with self.connect() as connection: diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index ae62cde89..c06f8b80e 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -42,7 +42,8 @@ def fake_index(self, file_path, mode="auto"): if path.suffix.lower() in {".md", ".markdown"} else "pdf alpha evidence" ) - document_id = f"pageindex_{path.stem}" + path_digest = hashlib.sha256(str(path.resolve()).encode("utf-8")).hexdigest() + document_id = f"pageindex_{path_digest[:16]}" document = { "id": document_id, "type": "pdf" if path.suffix.lower() == ".pdf" else "md", @@ -62,24 +63,8 @@ def fake_index(self, file_path, mode="auto"): "pages": [{"page": 1, "content": text}], } self.documents[document_id] = document - self.workspace.mkdir(parents=True, exist_ok=True) - (self.workspace / f"{document_id}.json").write_text( - json.dumps(document), encoding="utf-8" - ) - (self.workspace / "_meta.json").write_text( - json.dumps( - { - document_id: { - "type": document["type"], - "doc_name": path.name, - "doc_description": document["doc_description"], - "path": str(path.resolve()), - "line_count": document["line_count"], - } - } - ), - encoding="utf-8", - ) + if self.workspace: + self._save_doc(document_id) return document_id monkeypatch.setattr(openai, "OpenAI", FakeOpenAI) @@ -273,6 +258,22 @@ def registration_logical_state(workspace): else: artifacts[artifact_kind] = {} + pageindex_dir = workspace / "artifacts" / "pageindex_client" + meta_path = pageindex_dir / "_meta.json" + artifacts["pageindex_client"] = { + "meta": json.loads(meta_path.read_text(encoding="utf-8")) + if meta_path.is_file() + else {}, + "documents": { + path.name: { + "json": json.loads(path.read_text(encoding="utf-8")), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for path in sorted(pageindex_dir.glob("*.json")) + if path.name != "_meta.json" + }, + } + return { "catalog": catalog, "summary": summary, @@ -1521,6 +1522,59 @@ def create(self, *, model, input, dimensions): assert registration_logical_state(workspace) == baseline strict_reopen = open_test_filesystem(workspace) assert strict_reopen.store.get_file(existing_ref).metadata["year"] == 2023 + structure = strict_reopen.pageindex_structure(existing_ref) + assert structure["available"] is True + assert structure["structure"][0]["title"] == existing.stem + + +def test_register_files_rolls_back_partial_pageindex_preparation( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + calls = 0 + + def fail_after_second_pageindex_write(self, file_path, mode="auto"): + nonlocal calls + calls += 1 + document_id = successful_index(self, file_path, mode=mode) + if calls == 2: + raise RuntimeError("PageIndex extraction failed for second registration") + return document_id + + monkeypatch.setattr(PageIndexClient, "index", fail_after_second_pageindex_write) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises( + RuntimeError, + match="PageIndex extraction failed for second registration", + ): + filesystem.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + } + for source in (first, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] def test_register_file_rolls_back_partial_projection_creation_and_reopens( From 89c62087f2467dd4be32e772d6587714910e147e Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 18:38:19 +0800 Subject: [PATCH 10/14] fix(filesystem): preserve registration rollback state Validate copied caller metadata before PageIndex and artifact preparation. Capture exact registration-owned artifact baselines so prepare-stage failures restore existing bytes and remove only newly created paths. --- pageindex/filesystem/core.py | 54 +++++- tests/test_pifs_simplification_acceptance.py | 172 ++++++++++++++++++- 2 files changed, 222 insertions(+), 4 deletions(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 41691e799..00332d5e9 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -223,6 +223,7 @@ def add_file( def register_files(self, files: list[dict[str, Any]]) -> list[str]: preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() + artifact_baselines: dict[Path, bytes | None] = {} records: list[dict[str, Any]] = [] new_records: list[dict[str, Any]] = [] created_folder_paths: list[str] = [] @@ -231,7 +232,12 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: preexisting_cache_keys: set[_EmbeddingCacheKey] = set() try: for file in files: - records.append(self._prepare_file_record(file)) + records.append( + self._prepare_file_record( + file, + artifact_baselines=artifact_baselines, + ) + ) preexisting_file_refs = self._existing_file_refs(records) new_records = [ record @@ -281,11 +287,11 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: self._cleanup_catalog_record(str(record["file_ref"])) self._cleanup_created_folders(created_folder_paths) self._cleanup_new_metadata_fields(new_metadata_fields) - self._cleanup_failed_register_artifacts(records) self._cleanup_pageindex_cache( records, preexisting_pageindex_doc_ids, ) + self._restore_registration_artifact_baselines(artifact_baselines) raise return [record["file_ref"] for record in records] @@ -1064,11 +1070,21 @@ def _require_summary_projection_ready( f"PIFS {operation} failed to build summary projection index: {detail}" ) - def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: + def _prepare_file_record( + self, + file: dict[str, Any], + *, + artifact_baselines: dict[Path, bytes | None] | None = None, + ) -> dict[str, Any]: storage_uri = file["storage_uri"] metadata = file.get("metadata") or {} if not isinstance(metadata, dict): raise ValueError("metadata must be a JSON object") + metadata = dict(metadata) + try: + json.dumps(metadata, ensure_ascii=False) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON serializable") from exc self._validate_register_metadata(metadata) external_id = file.get("external_id") content = file.get("content") or "" @@ -1091,6 +1107,12 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: file_ref = make_file_ref( str(external_id or self._join_virtual_file_path(folder_path, title).strip("/")) ) + if artifact_baselines is not None: + self._capture_registration_artifact_baselines( + file_ref, + file, + artifact_baselines, + ) ( pageindex_doc_id, pageindex_tree_status, @@ -1294,6 +1316,32 @@ def _cleanup_failed_register_artifacts(self, records: list[dict[str, Any]]) -> N if record.get("_pifs_owned_raw_artifact") and record.get("raw_artifact_path"): self._unlink_artifact(record["raw_artifact_path"]) + def _capture_registration_artifact_baselines( + self, + file_ref: str, + file: dict[str, Any], + baselines: dict[Path, bytes | None], + ) -> None: + paths = [] + if file.get("text_artifact_path") is None: + paths.append(self.store.text_dir / f"{file_ref}.txt") + if file.get("raw_artifact_path") is None and file.get("write_raw_artifact", True): + paths.append(self.store.raw_dir / f"{file_ref}.json") + for path in paths: + if path not in baselines: + baselines[path] = path.read_bytes() if path.is_file() else None + + def _restore_registration_artifact_baselines( + self, + baselines: dict[Path, bytes | None], + ) -> None: + for path, content in baselines.items(): + if content is None: + self._unlink_artifact(path) + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + def _cleanup_catalog_record(self, file_ref: str) -> None: try: self.store.delete_file(file_ref) diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index c06f8b80e..e5573333b 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -1349,6 +1349,8 @@ def test_register_file_completes_summary_projection_for_immediate_and_reopen_bro source = tmp_path / "notes.md" source.write_text("alpha registration evidence", encoding="utf-8") filesystem = open_test_filesystem(workspace) + caller_metadata = {"year": 2024, "labels": {"team": "alpha"}} + caller_metadata_baseline = json.loads(json.dumps(caller_metadata)) file_ref = filesystem.register_file( storage_uri=source.as_uri(), @@ -1356,12 +1358,17 @@ def test_register_file_completes_summary_projection_for_immediate_and_reopen_bro title=source.name, content_type="text/markdown", content=source.read_text(encoding="utf-8"), + metadata=caller_metadata, ) projection_dir = workspace / "artifacts" / "projection_indexes" assert (projection_dir / "summary.sqlite").is_file() assert (projection_dir / "embedding_cache.sqlite").is_file() - assert filesystem.store.get_file(file_ref).metadata_status["summary_projection"]["status"] == "ready" + stored = filesystem.store.get_file(file_ref) + assert caller_metadata == caller_metadata_baseline + assert "summary" not in caller_metadata + assert stored.metadata["summary"].startswith("Summary for notes.md:") + assert stored.metadata_status["summary_projection"]["status"] == "ready" assert file_ref in { row["file_ref"] for row in filesystem.browse_semantic_files("/documents", "alpha")["data"] @@ -1577,6 +1584,169 @@ def fail_after_second_pageindex_write(self, file_path, mode="auto"): assert reopened.store.file_refs_for_scope() == [] +def test_register_files_prepare_failure_restores_existing_owned_artifacts( + tmp_path, monkeypatch +): + from pageindex.filesystem.commands import PIFSCommandExecutor + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + existing = tmp_path / "existing.md" + replacement = tmp_path / "replacement.md" + existing.write_text("alpha original registration evidence", encoding="utf-8") + replacement.write_text("beta replacement registration evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=existing.as_uri(), + external_id="doc-existing", + folder_path="/documents", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + existing_entry = filesystem.store.get_file(existing_ref) + text_path = Path(existing_entry.text_artifact_path) + raw_path = Path(existing_entry.raw_artifact_path) + original_text = text_path.read_bytes() + original_raw = raw_path.read_bytes() + original_pageindex_doc_id = existing_entry.pageindex_doc_id + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="requires PageIndex extraction"): + filesystem.register_files( + [ + { + "storage_uri": replacement.as_uri(), + "external_id": "doc-existing", + "folder_path": "/documents/replacement", + "title": replacement.name, + "content_type": "text/markdown", + "content": replacement.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": "https://example.invalid/unresolvable.md", + "external_id": "doc-failing", + "folder_path": "/documents/new", + "title": "unresolvable.md", + "content_type": "text/markdown", + "content": "must not be registered", + "metadata": {"year": 2025}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + assert text_path.read_bytes() == original_text + assert raw_path.read_bytes() == original_raw + + reopened = open_test_filesystem(workspace) + reopened_entry = reopened.store.get_file(existing_ref) + assert reopened_entry.pageindex_doc_id == original_pageindex_doc_id + executor = PIFSCommandExecutor(reopened) + locator = "/documents/existing.md" + assert json.loads(executor.execute(f"stat {locator}"))["success"] is True + cat = json.loads(executor.execute(f"cat {locator} --structure")) + assert cat["success"] is True + assert cat["data"]["pagination"]["available"] is True + grep = json.loads(executor.execute(f"grep alpha {locator}")) + assert grep["success"] is True + assert grep["data"]["matches"] + structure = reopened.pageindex_structure(existing_ref) + assert structure["available"] is True + assert structure["structure"][0]["title"] == existing.stem + + +def test_register_file_rejects_non_json_metadata_before_side_effects( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + indexed_paths = [] + + def record_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path).name) + return successful_index(self, file_path, mode=mode) + + monkeypatch.setattr(PageIndexClient, "index", record_index) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(ValueError, match="metadata must be JSON serializable"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents/new", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata={"not_json": object()}, + ) + + assert registration_logical_state(workspace) == baseline + assert indexed_paths == [] + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_rejects_second_non_json_metadata_and_rolls_back_batch( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + indexed_paths = [] + + def record_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path).name) + return successful_index(self, file_path, mode=mode) + + monkeypatch.setattr(PageIndexClient, "index", record_index) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(ValueError, match="metadata must be JSON serializable"): + filesystem.register_files( + [ + { + "storage_uri": first.as_uri(), + "folder_path": "/documents/new", + "title": first.name, + "content_type": "text/markdown", + "content": first.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": second.as_uri(), + "folder_path": "/documents/new", + "title": second.name, + "content_type": "text/markdown", + "content": second.read_text(encoding="utf-8"), + "metadata": {"not_json": object()}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + assert indexed_paths == [first.name] + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + def test_register_file_rolls_back_partial_projection_creation_and_reopens( tmp_path, monkeypatch ): From 2b235bd552844e9ef2e7ca038de3f5a30c737a39 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 18:52:46 +0800 Subject: [PATCH 11/14] fix(filesystem): preflight registration mutations Validate every batch metadata payload before PageIndex work begins. Reuse one canonical raw-artifact policy for both success-path writes and rollback baseline capture. --- pageindex/filesystem/core.py | 64 ++++++++++++----- tests/test_pifs_simplification_acceptance.py | 75 +++++++++++++++++++- 2 files changed, 118 insertions(+), 21 deletions(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 00332d5e9..6c4e09dcb 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -222,6 +222,13 @@ def add_file( return info def register_files(self, files: list[dict[str, Any]]) -> list[str]: + files = [ + { + **file, + "metadata": self._validated_register_metadata(file.get("metadata")), + } + for file in files + ] preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() artifact_baselines: dict[Path, bytes | None] = {} records: list[dict[str, Any]] = [] @@ -1077,15 +1084,7 @@ def _prepare_file_record( artifact_baselines: dict[Path, bytes | None] | None = None, ) -> dict[str, Any]: storage_uri = file["storage_uri"] - metadata = file.get("metadata") or {} - if not isinstance(metadata, dict): - raise ValueError("metadata must be a JSON object") - metadata = dict(metadata) - try: - json.dumps(metadata, ensure_ascii=False) - except (TypeError, ValueError) as exc: - raise ValueError("metadata must be JSON serializable") from exc - self._validate_register_metadata(metadata) + metadata = self._validated_register_metadata(file.get("metadata")) external_id = file.get("external_id") content = file.get("content") or "" folder_path = normalize_path(file.get("folder_path") or "/") @@ -1233,12 +1232,10 @@ def _raw_artifact_payload( def _sync_owned_raw_artifact(self, record: dict[str, Any]) -> None: raw_artifact_path = record.get("raw_artifact_path") - if not raw_artifact_path: - return - default_raw_artifact_path = self.store.raw_dir / f"{record['file_ref']}.json" - if Path(raw_artifact_path).expanduser().resolve(strict=False) != ( - default_raw_artifact_path.resolve(strict=False) - ): + if self._managed_raw_artifact_path( + str(record["file_ref"]), + raw_artifact_path, + ) is None: return record["raw_artifact_path"] = str( self.store.write_raw_artifact( @@ -1325,12 +1322,30 @@ def _capture_registration_artifact_baselines( paths = [] if file.get("text_artifact_path") is None: paths.append(self.store.text_dir / f"{file_ref}.txt") - if file.get("raw_artifact_path") is None and file.get("write_raw_artifact", True): - paths.append(self.store.raw_dir / f"{file_ref}.json") + raw_artifact_path = file.get("raw_artifact_path") + if raw_artifact_path is None and file.get("write_raw_artifact", True): + raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" + managed_raw_path = self._managed_raw_artifact_path(file_ref, raw_artifact_path) + if managed_raw_path is not None: + paths.append(managed_raw_path) for path in paths: if path not in baselines: baselines[path] = path.read_bytes() if path.is_file() else None + def _managed_raw_artifact_path( + self, + file_ref: str, + raw_artifact_path: Any, + ) -> Path | None: + if not raw_artifact_path: + return None + default_path = self.store.raw_dir / f"{file_ref}.json" + if Path(raw_artifact_path).expanduser().resolve(strict=False) != ( + default_path.resolve(strict=False) + ): + return None + return default_path + def _restore_registration_artifact_baselines( self, baselines: dict[Path, bytes | None], @@ -1613,9 +1628,20 @@ def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: ) @staticmethod - def _validate_register_metadata(metadata: dict[str, Any]) -> None: - if "summary" in metadata: + def _validated_register_metadata(metadata: Any) -> dict[str, Any]: + if metadata is None: + validated = {} + elif not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + else: + validated = dict(metadata) + try: + json.dumps(validated, ensure_ascii=False) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON serializable") from exc + if "summary" in validated: raise ValueError("summary is managed by PageIndex doc_description") + return validated def _register_custom_metadata_fields(self, records: list[dict[str, Any]]) -> None: fields = { diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index e5573333b..4f3137c94 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -1659,6 +1659,77 @@ def test_register_files_prepare_failure_restores_existing_owned_artifacts( assert structure["structure"][0]["title"] == existing.stem +@pytest.mark.parametrize("raw_path_kind", ["canonical", "custom"]) +def test_register_files_rollback_follows_explicit_raw_artifact_management_policy( + raw_path_kind, tmp_path, monkeypatch +): + import openai + from pageindex.filesystem.store import make_file_ref + + install_network_fakes(monkeypatch) + + class FailingSecondOpenAI(FakeOpenAI): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = 0 + + def create(self, *, model, input, dimensions): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingSecondOpenAI) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + first_external_id = f"doc-{raw_path_kind}-raw" + first_file_ref = make_file_ref(first_external_id) + raw_dir = workspace / "artifacts" / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_path = raw_dir / ( + f"{first_file_ref}.json" + if raw_path_kind == "canonical" + else "custom-sentinel.json" + ) + sentinel = f"{raw_path_kind} raw sentinel".encode() + raw_path.write_bytes(sentinel) + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + filesystem.register_files( + [ + { + "storage_uri": first.as_uri(), + "external_id": first_external_id, + "folder_path": "/documents/new", + "title": first.name, + "content_type": "text/markdown", + "content": first.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + "raw_artifact_path": str(raw_path), + }, + { + "storage_uri": second.as_uri(), + "external_id": "doc-second-raw", + "folder_path": "/documents/new", + "title": second.name, + "content_type": "text/markdown", + "content": second.read_text(encoding="utf-8"), + "metadata": {"year": 2025}, + }, + ] + ) + + assert raw_path.read_bytes() == sentinel + assert registration_logical_state(workspace) == baseline + reopened = open_test_filesystem(workspace) + assert reopened.store.file_refs_for_scope() == [] + + def test_register_file_rejects_non_json_metadata_before_side_effects( tmp_path, monkeypatch ): @@ -1696,7 +1767,7 @@ def record_index(self, file_path, mode="auto"): assert reopened.store.file_refs_for_scope() == [] -def test_register_files_rejects_second_non_json_metadata_and_rolls_back_batch( +def test_register_files_preflights_all_metadata_before_preparing_batch( tmp_path, monkeypatch ): from pageindex import PageIndexClient @@ -1741,7 +1812,7 @@ def record_index(self, file_path, mode="auto"): ) assert registration_logical_state(workspace) == baseline - assert indexed_paths == [first.name] + assert indexed_paths == [] reopened = open_test_filesystem(workspace) assert reopened.store.folder_info("/")["path"] == "/" assert reopened.store.file_refs_for_scope() == [] From 4a784f7bd8fe4e7162a70cf8a5784bacfc362248 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 19:03:01 +0800 Subject: [PATCH 12/14] fix(filesystem): require complete workspace projection Validate active catalog and Summary Projection references in both directions. Existing catalogs with active files now fail closed when the projection pair is absent, while fresh and deleted-only catalog listing remains lazy. --- .../filesystem/_workspace_consistency.py | 30 ++++++ pageindex/filesystem/core.py | 14 ++- tests/test_pifs_simplification_acceptance.py | 101 ++++++++++++++++++ 3 files changed, 140 insertions(+), 5 deletions(-) diff --git a/pageindex/filesystem/_workspace_consistency.py b/pageindex/filesystem/_workspace_consistency.py index aa8a5ef9f..41fbb412a 100644 --- a/pageindex/filesystem/_workspace_consistency.py +++ b/pageindex/filesystem/_workspace_consistency.py @@ -70,3 +70,33 @@ def validate_workspace_consistency( "Summary Projection file_ref values do not reference active catalog files: " f"{shown}" ) + missing = sorted(active_file_refs - projected_file_refs) + if missing: + shown = ", ".join(missing[:5]) + raise _inconsistent_workspace_error( + "active catalog file_ref values are missing from the Summary Projection: " + f"{shown}" + ) + + +def validate_catalog_without_projection(catalog_path: str | Path) -> None: + catalog_path = Path(catalog_path) + validate_catalog_root(catalog_path) + try: + with _readonly_connection(catalog_path) as catalog: + active_file_refs = sorted( + str(row[0]) + for row in catalog.execute( + "SELECT file_ref FROM files WHERE deleted_at IS NULL" + ) + ) + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "active catalog references could not be read" + ) from exc + if active_file_refs: + shown = ", ".join(active_file_refs[:5]) + raise _inconsistent_workspace_error( + "active catalog files require a complete Summary Projection: " + f"{shown}" + ) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 6c4e09dcb..a825a63b4 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -82,12 +82,12 @@ def __init__( summary_path, cache_path = projection_database_paths( self.summary_projection_index_dir ) + catalog_path = self.workspace / "filesystem.sqlite" + catalog_present = catalog_path.exists() or catalog_path.is_symlink() summary_present = projection_database_path_present(summary_path) cache_present = projection_database_path_present(cache_path) - if summary_present or cache_present: - SQLiteFileSystemStore.validate_existing_database( - self.workspace / "filesystem.sqlite" - ) + if catalog_present or summary_present or cache_present: + SQLiteFileSystemStore.validate_existing_database(catalog_path) database_pair = projection_database_pair(self.summary_projection_index_dir) if database_pair is not None: from .semantic_projection import validate_projection_topology @@ -95,9 +95,13 @@ def __init__( validate_projection_topology(self.summary_projection_index_dir) validate_workspace_consistency( - self.workspace / "filesystem.sqlite", + catalog_path, database_pair[0], ) + elif catalog_present: + from ._workspace_consistency import validate_catalog_without_projection + + validate_catalog_without_projection(catalog_path) self.store = SQLiteFileSystemStore(self.workspace) self.metadata = MetadataQueryEngine(self.store) self.summary_projection: Any | None = None diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 4f3137c94..40296708f 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -165,6 +165,23 @@ def insert_projection_document(summary_path, *, file_ref="orphan_projection_ref" ) +def delete_projection_document(summary_path, file_ref): + with connect_summary_database(summary_path) as connection: + row = connection.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchone() + assert row is not None + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", + (row[0],), + ) + connection.execute( + "DELETE FROM semantic_index_docs WHERE rowid = ?", + (row[0],), + ) + + def catalog_file_count(workspace): with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] @@ -553,6 +570,90 @@ def test_cli_rejects_inconsistent_catalog_projection_relationships_without_mutat assert workspace_state(workspace) == before +@pytest.mark.parametrize( + ("catalog_state", "projection_state", "should_fail"), + [ + ("active", "projected", False), + ("active", "missing", True), + ("deleted", "missing", False), + ("deleted", "projected", True), + ], +) +def test_runtime_requires_complete_projection_for_every_active_catalog_file( + catalog_state, projection_state, should_fail, tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha consistency evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + if catalog_state == "deleted": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE files SET deleted_at = '2026-01-02 03:04:05' " + "WHERE file_ref = ?", + (file_ref,), + ) + if projection_state == "missing": + delete_projection_document( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite", + file_ref, + ) + before = workspace_state(workspace) + + if should_fail: + with pytest.raises(RuntimeError, match="migrate_pifs_workspace.py"): + open_test_filesystem(workspace) + else: + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize(("catalog_state", "should_fail"), [("active", True), ("deleted", False)]) +def test_runtime_requires_projection_pair_when_catalog_only_has_active_files( + catalog_state, should_fail, tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha catalog-only evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + if catalog_state == "deleted": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE files SET deleted_at = '2026-01-02 03:04:05' " + "WHERE file_ref = ?", + (file_ref,), + ) + projection_dir = workspace / "artifacts" / "projection_indexes" + (projection_dir / "summary.sqlite").unlink() + (projection_dir / "embedding_cache.sqlite").unlink() + before = workspace_state(workspace) + + if should_fail: + with pytest.raises(RuntimeError, match="migrate_pifs_workspace.py"): + open_test_filesystem(workspace) + else: + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + @pytest.mark.parametrize( "mutation", [ From 0559180ad91d29afc72ad655992eb55e29da64a2 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 19:48:41 +0800 Subject: [PATCH 13/14] fix(filesystem): close PIFS consistency gaps Restore the canonical metadata-axis paths, load embedding credentials from the configured precedence, and validate projection rows on open. Make add and registration failures restore catalog, projection, cache, PageIndex, and owned artifact state exactly, including updates to existing records. Refs BukeLy/Pageindex#47 --- pageindex/filesystem/agent.py | 5 +- pageindex/filesystem/cli.py | 7 +- pageindex/filesystem/commands.py | 30 +- pageindex/filesystem/core.py | 299 ++++++++--- pageindex/filesystem/semantic_index.py | 28 +- tests/test_pifs_agent_stream.py | 4 + tests/test_pifs_cli.py | 37 ++ tests/test_pifs_simplification_acceptance.py | 490 ++++++++++++++++++- 8 files changed, 803 insertions(+), 97 deletions(-) diff --git a/pageindex/filesystem/agent.py b/pageindex/filesystem/agent.py index cd67efb17..fd4176d6b 100644 --- a/pageindex/filesystem/agent.py +++ b/pageindex/filesystem/agent.py @@ -55,7 +55,8 @@ Use only: tree, browse, stat, cat, and grep. Start broad workspace questions with tree. Then choose a physical folder, inspect metadata-derived virtual axes such as @year or @ticker with tree, and -copy returned scope paths into later commands. Run browse "" for document +select exact metadata values through paths such as @field/value. Copy returned +scope paths into later commands. Run browse "" for document discovery. Use browse -R when low-signal folder or virtual-node labels do not help prune the query, or after scoped browse misses/rephrasing. Browse returns document candidates only, not final evidence. Use stat only for identity, status, or @@ -72,7 +73,7 @@ - Use only tree, browse, stat, cat, and grep. PIFS Listing is tree -L 1. - Folder paths such as /documents are positional command targets; never put folder paths in --where. - Start with tree to understand workspace and folder structure before document retrieval. -- After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect metadata-derived virtual value nodes, and copy returned scope paths instead of handwriting encoded values. +- After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect @field/value nodes, and copy returned scope paths instead of handwriting encoded values. - After choosing a folder or scope, use browse "" for relevance-ranked document candidates; quote multi-word queries, for example browse /documents "Federal Reserve". - Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query. - If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder. diff --git a/pageindex/filesystem/cli.py b/pageindex/filesystem/cli.py index 47b9610f1..8b03a65ba 100644 --- a/pageindex/filesystem/cli.py +++ b/pageindex/filesystem/cli.py @@ -28,6 +28,7 @@ PIFS_WORKSPACE_ENV = "PIFS_WORKSPACE" PERSISTED_CONFIG_KEYS = { "workspace", + "embedding_api_key", "embedding_base_url", "embedding_model", "embedding_dimensions", @@ -198,7 +199,11 @@ def _filesystem_embedding_config() -> dict[str, object]: config_values = _read_config() config: dict[str, object] = {} base_url = config_values.get("embedding_base_url") - api_key = os.environ.get("PIFS_EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY") + api_key = ( + os.environ.get("PIFS_EMBEDDING_API_KEY") + or config_values.get("embedding_api_key") + or os.environ.get("OPENAI_API_KEY") + ) model = config_values.get("embedding_model") dimensions = _optional_int( "embedding_dimensions", diff --git a/pageindex/filesystem/commands.py b/pageindex/filesystem/commands.py index 828048429..22cfdd15e 100644 --- a/pageindex/filesystem/commands.py +++ b/pageindex/filesystem/commands.py @@ -95,12 +95,10 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: root = self._scope_root(scope, file_count=self.filesystem.scope_file_count(scope)) root["folders"] = [ { - "path": self._join_scope_path( + "path": self._metadata_value_path( scope.path.rsplit("/", 1)[0] or "/", - ( - f"@{self.filesystem.encode_scope_segment(scope.metadata_axis)}=" - f"{self.filesystem.encode_scope_segment(row['value'])}" - ), + scope.metadata_axis, + row["value"], ), "name": str(row["value"]), "type": "metadata_value", @@ -124,7 +122,7 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: }, } next_steps = [ - f'browse {shlex.quote(scope.path)}= ""' + f'browse {shlex.quote(scope.path)}/ ""' ] return data, next_steps page_size = self.TREE_VALUE_PAGE_SIZE @@ -197,7 +195,7 @@ def _cmd_browse(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: scope = self.filesystem.resolve_query_scope(path) if scope.metadata_axis is not None: raise PIFSCommandError( - "Metadata axis paths require @field=value; run tree /@field to inspect values." + "Metadata axis paths require @field/value; run tree /@field to inspect values." ) merged_filter = self.filesystem.merge_scope_filter(scope, where) effective_recursive = recursive or bool(scope.metadata_filter) @@ -544,15 +542,19 @@ def _scope_root(self, scope: Any, *, file_count: int) -> dict[str, Any]: def _scoped_folder_path(self, scope: Any, folder_path: str) -> str: path = self._normalize_folder_path(folder_path) for field, value in getattr(scope, "metadata_filter", {}).items(): - path = self._join_scope_path( - path, - ( - f"@{self.filesystem.encode_scope_segment(field)}=" - f"{self.filesystem.encode_scope_segment(value)}" - ), - ) + path = self._metadata_value_path(path, field, value) return path + def _metadata_value_path(self, base: str, field: object, value: object) -> str: + axis_path = self._join_scope_path( + base, + f"@{self.filesystem.encode_scope_segment(field)}", + ) + return self._join_scope_path( + axis_path, + self.filesystem.encode_scope_segment(value), + ) + @staticmethod def _join_scope_path(base: str, segment: str) -> str: base_path = str(base or "/").rstrip("/") diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index a825a63b4..df04e7ab0 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -4,6 +4,7 @@ import os import shutil import tempfile +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import quote, unquote, urlparse @@ -49,6 +50,24 @@ } +@dataclass +class _RegistrationRollbackSnapshot: + preexisting_pageindex_doc_ids: set[str] + artifact_baselines: dict[Path, bytes | None] = field(default_factory=dict) + records: list[dict[str, Any]] = field(default_factory=list) + new_records: list[dict[str, Any]] = field(default_factory=list) + created_folder_paths: list[str] = field(default_factory=list) + new_metadata_fields: set[str] = field(default_factory=set) + new_cache_keys: set[_EmbeddingCacheKey] = field(default_factory=set) + catalog_rows: dict[str, dict[str, Any]] = field(default_factory=dict) + membership_rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + metadata_value_rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + projection_rows: dict[ + str, + tuple[dict[str, Any], dict[str, Any]], + ] = field(default_factory=dict) + + def strip_pageindex_text_fields(value: Any) -> Any: if isinstance(value, list): return [strip_pageindex_text_fields(item) for item in value] @@ -161,7 +180,7 @@ def add_file( ) if self.store.file_basename_exists_in_folder(folder_path, filename): raise FileExistsError(f"File already exists at {virtual_path}") - self._ensure_summary_projection() + projection = self._ensure_summary_projection() add_created_folder_paths = self._add_created_folder_paths(folder_path) file_ref = make_file_ref(virtual_path.strip("/")) uploads_dir = self.workspace / "artifacts" / "uploads" @@ -170,6 +189,8 @@ def add_file( final_dir_created = False catalog_inserted = False records: list[dict[str, Any]] = [] + cache_keys: set[_EmbeddingCacheKey] = set() + preexisting_cache_keys: set[_EmbeddingCacheKey] = set() preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() uploads_dir.mkdir(parents=True, exist_ok=True) @@ -197,6 +218,8 @@ def add_file( } ) records = [record] + cache_keys = projection.cache_keys_for_records(records) + preexisting_cache_keys = projection.existing_cache_keys(cache_keys) self._require_add_pageindex_ready(record) self._register_custom_metadata_fields(records) self.store.insert_files(records) @@ -214,6 +237,9 @@ def add_file( if catalog_inserted: self._cleanup_catalog_record(file_ref) self._cleanup_summary_projection_records(records) + self._cleanup_summary_projection_cache( + cache_keys - preexisting_cache_keys + ) self._cleanup_failed_register_artifacts(records) self._cleanup_pageindex_cache(records, preexisting_pageindex_doc_ids) self._cleanup_created_folders(add_created_folder_paths) @@ -233,48 +259,45 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: } for file in files ] - preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() - artifact_baselines: dict[Path, bytes | None] = {} - records: list[dict[str, Any]] = [] - new_records: list[dict[str, Any]] = [] - created_folder_paths: list[str] = [] - new_metadata_fields: set[str] = set() - batch_cache_keys: set[_EmbeddingCacheKey] = set() - preexisting_cache_keys: set[_EmbeddingCacheKey] = set() + rollback = _RegistrationRollbackSnapshot( + preexisting_pageindex_doc_ids=self._pageindex_cache_doc_ids() + ) try: for file in files: - records.append( + rollback.records.append( self._prepare_file_record( file, - artifact_baselines=artifact_baselines, + artifact_baselines=rollback.artifact_baselines, ) ) - preexisting_file_refs = self._existing_file_refs(records) - new_records = [ + projection = self._ensure_summary_projection() if rollback.records else None + self._capture_existing_registration_rows(rollback, projection) + rollback.new_records = [ record - for record in records - if record["file_ref"] not in preexisting_file_refs + for record in rollback.records + if record["file_ref"] not in rollback.catalog_rows ] - created_folder_paths = sorted( + rollback.created_folder_paths = sorted( { path - for record in new_records + for record in rollback.records for path in self._add_created_folder_paths(record["folder_path"]) }, key=lambda path: (path.count("/"), path), ) - new_metadata_fields = { + rollback.new_metadata_fields = { name - for name in self._custom_metadata_field_names(records) + for name in self._custom_metadata_field_names(rollback.records) if not self.store.metadata_field_exists(name) } - if records: - projection = self._ensure_summary_projection() - batch_cache_keys = projection.cache_keys_for_records(new_records) - preexisting_cache_keys = projection.existing_cache_keys(batch_cache_keys) - self._register_custom_metadata_fields(records) - self.store.insert_files(records) - for record in records: + if projection is not None: + batch_cache_keys = projection.cache_keys_for_records(rollback.records) + rollback.new_cache_keys = ( + batch_cache_keys - projection.existing_cache_keys(batch_cache_keys) + ) + self._register_custom_metadata_fields(rollback.records) + self.store.insert_files(rollback.records) + for record in rollback.records: try: if self._complete_summary_projection_index(record): self.store.update_file_metadata_status( @@ -290,21 +313,23 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: except KeyError: continue except Exception: - self._cleanup_summary_projection_records(new_records) - self._cleanup_summary_projection_cache( - batch_cache_keys - preexisting_cache_keys - ) - for record in new_records: + self._cleanup_summary_projection_records(rollback.new_records) + self._restore_existing_registration_projection(rollback) + self._cleanup_summary_projection_cache(rollback.new_cache_keys) + for record in rollback.new_records: self._cleanup_catalog_record(str(record["file_ref"])) - self._cleanup_created_folders(created_folder_paths) - self._cleanup_new_metadata_fields(new_metadata_fields) + self._restore_existing_registration_catalog(rollback) + self._cleanup_created_folders(rollback.created_folder_paths) + self._cleanup_new_metadata_fields(rollback.new_metadata_fields) self._cleanup_pageindex_cache( - records, - preexisting_pageindex_doc_ids, + rollback.records, + rollback.preexisting_pageindex_doc_ids, + ) + self._restore_registration_artifact_baselines( + rollback.artifact_baselines ) - self._restore_registration_artifact_baselines(artifact_baselines) raise - return [record["file_ref"] for record in records] + return [record["file_ref"] for record in rollback.records] def _ensure_summary_projection(self) -> Any: return self._open_summary_projection(create=True) @@ -378,12 +403,16 @@ def resolve_query_scope(self, path: str) -> PIFSQueryScope: raise KeyError(f"Unknown folder path: {normalized}") raise ValueError( "Metadata axes must come after the physical folder prefix; " - "inspect the physical folder first, then append @field=value buckets. " + "inspect the physical folder first, then append @field/value segments. " "Use the path returned by tree for values containing '/'." ) axis_segment = segment[1:] - encoded_field, separator, encoded_value = axis_segment.partition("=") - field = unquote(encoded_field) + if "=" in axis_segment: + raise ValueError( + "Metadata virtual paths use @field/value; run tree /@field " + "and copy the returned path." + ) + field = unquote(axis_segment) self.metadata.validate_field_name(field) if not self.store.metadata_field_exists(field): raise ValueError("Unknown metadata axis; run tree to inspect available @field axes.") @@ -392,24 +421,22 @@ def resolve_query_scope(self, path: str) -> PIFSQueryScope: "A metadata field can appear only once in a scope path; " "choose one value or use browse --where for advanced predicates." ) - if not separator: - if index + 1 != len(remainder): - raise ValueError( - "Metadata axis inspection must be the final path segment; " - "choose a value with @field=value before appending another axis." - ) + value_index = index + 1 + if value_index == len(remainder): return PIFSQueryScope( path=normalized, folder_path=folder_path, metadata_filter=metadata_filter, metadata_axis=field, ) - if not encoded_value: + encoded_value = remainder[value_index] + if encoded_value.startswith("@"): raise ValueError( - "Metadata value cannot be empty; run tree /@field to inspect values." + "Metadata axis inspection must be the final path segment; " + "choose a value with @field/value before appending another axis." ) metadata_filter[field] = unquote(encoded_value) - index += 1 + index += 2 return PIFSQueryScope( path=normalized, @@ -425,7 +452,7 @@ def merge_scope_filter( parsed = self.metadata.parse_filter(metadata_filter) if scope.metadata_axis is not None: raise ValueError( - "Metadata axis paths require @field=value; run tree /@field to inspect values." + "Metadata axis paths require @field/value; run tree /@field to inspect values." ) if not parsed: return dict(scope.metadata_filter) or None @@ -1332,6 +1359,18 @@ def _capture_registration_artifact_baselines( managed_raw_path = self._managed_raw_artifact_path(file_ref, raw_artifact_path) if managed_raw_path is not None: paths.append(managed_raw_path) + try: + existing = self.store.get_file(file_ref) + except KeyError: + existing = None + if existing is not None and existing.pageindex_doc_id: + paths.extend( + [ + self.pageindex_client_workspace / "_meta.json", + self.pageindex_client_workspace + / f"{existing.pageindex_doc_id}.json", + ] + ) for path in paths: if path not in baselines: baselines[path] = path.read_bytes() if path.is_file() else None @@ -1367,18 +1406,154 @@ def _cleanup_catalog_record(self, file_ref: str) -> None: except Exception: return - def _existing_file_refs(self, records: list[dict[str, Any]]) -> set[str]: - existing: set[str] = set() - for record in records: - file_ref = str(record.get("file_ref") or "") - if not file_ref: - continue - try: - self.store.get_file(file_ref) - except KeyError: - continue - existing.add(file_ref) - return existing + def _capture_existing_registration_rows( + self, + snapshot: _RegistrationRollbackSnapshot, + projection: Any | None, + ) -> None: + file_refs = sorted( + { + str(record.get("file_ref") or "") + for record in snapshot.records + if str(record.get("file_ref") or "") + } + ) + with self.store.connect() as connection: + for file_ref in file_refs: + file_row = connection.execute( + "SELECT * FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + if file_row is None: + continue + snapshot.catalog_rows[file_ref] = dict(file_row) + snapshot.membership_rows[file_ref] = [ + dict(row) + for row in connection.execute( + "SELECT * FROM file_folders WHERE file_ref = ? ORDER BY folder_id", + (file_ref,), + ) + ] + snapshot.metadata_value_rows[file_ref] = [ + dict(row) + for row in connection.execute( + "SELECT * FROM metadata_values WHERE file_ref = ? " + "ORDER BY field_id, value_text, created_at", + (file_ref,), + ) + ] + if not snapshot.catalog_rows: + return + if projection is None: + raise RuntimeError( + "PIFS registration cannot snapshot existing files without a Summary Projection" + ) + with projection.index.connect(read_only=True) as connection: + for file_ref in sorted(snapshot.catalog_rows): + doc = connection.execute( + "SELECT * FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchone() + vector = ( + None + if doc is None + else connection.execute( + "SELECT rowid, source_type, embedding " + "FROM semantic_index_vec WHERE rowid = ?", + (doc["rowid"],), + ).fetchone() + ) + if doc is None or vector is None: + raise RuntimeError( + "PIFS registration found an incomplete existing Summary Projection; " + "migrate this workspace before retrying." + ) + vector_row = dict(vector) + vector_row["embedding"] = bytes(vector_row["embedding"]) + snapshot.projection_rows[file_ref] = (dict(doc), vector_row) + + def _restore_existing_registration_catalog( + self, + snapshot: _RegistrationRollbackSnapshot, + ) -> None: + if not snapshot.catalog_rows: + return + with self.store.connect() as connection: + for file_ref in sorted(snapshot.catalog_rows): + connection.execute( + "DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,) + ) + connection.execute( + "DELETE FROM file_folders WHERE file_ref = ?", (file_ref,) + ) + connection.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) + self._insert_registration_snapshot_row( + connection, + "files", + snapshot.catalog_rows[file_ref], + ) + for row in snapshot.membership_rows[file_ref]: + self._insert_registration_snapshot_row( + connection, + "file_folders", + row, + ) + for row in snapshot.metadata_value_rows[file_ref]: + self._insert_registration_snapshot_row( + connection, + "metadata_values", + row, + ) + + def _restore_existing_registration_projection( + self, + snapshot: _RegistrationRollbackSnapshot, + ) -> None: + if not snapshot.projection_rows: + return + if self.summary_projection is None: + raise RuntimeError( + "PIFS registration cannot restore an unopened Summary Projection" + ) + with self.summary_projection.index.connect() as connection: + for file_ref in sorted(snapshot.projection_rows): + current = connection.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchall() + for row in current: + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", + (row["rowid"],), + ) + connection.execute( + "DELETE FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ) + doc, vector = snapshot.projection_rows[file_ref] + self._insert_registration_snapshot_row( + connection, + "semantic_index_docs", + doc, + ) + self._insert_registration_snapshot_row( + connection, + "semantic_index_vec", + vector, + ) + + @staticmethod + def _insert_registration_snapshot_row( + connection: Any, + table: str, + row: dict[str, Any], + ) -> None: + columns = list(row) + placeholders = ", ".join("?" for _ in columns) + connection.execute( + f"INSERT INTO {table}({', '.join(columns)}) VALUES ({placeholders})", + [row[column] for column in columns], + ) def _cleanup_summary_projection_records( self, diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py index a430454e9..420793e4b 100644 --- a/pageindex/filesystem/semantic_index.py +++ b/pageindex/filesystem/semantic_index.py @@ -99,7 +99,8 @@ def validate_schema(self) -> dict[str, Any]: config = self._config(conn) info = self._info_from_connection(conn, config) actual = self._schema_signature(conn, tables) - dimension = int(info["dimension"]) + dimension = int(info["dimension"]) + self._validate_projection_rows(conn, dimension=dimension) metadata = info["metadata"] if ( version != SCHEMA_VERSION @@ -472,6 +473,31 @@ def _schema_signature( ) return signature + @classmethod + def _validate_projection_rows( + cls, + conn: sqlite3.Connection, + *, + dimension: int, + ) -> None: + docs = { + int(row["rowid"]): str(row["source_type"]) + for row in conn.execute( + "SELECT rowid, source_type FROM semantic_index_docs" + ) + } + vectors: dict[int, str] = {} + expected_bytes = dimension * 4 + for row in conn.execute( + "SELECT rowid, source_type, embedding FROM semantic_index_vec" + ): + rowid = int(row["rowid"]) + if len(bytes(row["embedding"])) != expected_bytes: + raise cls._incompatible_schema_error() + vectors[rowid] = str(row["source_type"]) + if docs != vectors: + raise cls._incompatible_schema_error() + @staticmethod def _incompatible_schema_error() -> SemanticIndexError: return SemanticIndexError( diff --git a/tests/test_pifs_agent_stream.py b/tests/test_pifs_agent_stream.py index d2c664586..244e8249b 100644 --- a/tests/test_pifs_agent_stream.py +++ b/tests/test_pifs_agent_stream.py @@ -281,6 +281,10 @@ def test_system_prompt_sets_workspace_identity_and_scope(self): self.assertIn("VectifyAI Team", AGENT_SYSTEM_PROMPT) self.assertIn("current PageIndex FileSystem\nworkspace", AGENT_SYSTEM_PROMPT) self.assertIn("unrelated to the current workspace", AGENT_SYSTEM_PROMPT) + self.assertIn("@field/value", AGENT_TOOL_POLICY) + self.assertIn("@field/value", BASH_TOOL_DESCRIPTION) + self.assertNotIn("@field=value", AGENT_TOOL_POLICY) + self.assertNotIn("@field=value", BASH_TOOL_DESCRIPTION) self.assertIn("do not answer it as\na general-purpose assistant", AGENT_SYSTEM_PROMPT) self.assertIn("workspace-related topic question", AGENT_SYSTEM_PROMPT) self.assertIn("clarify only after the persistence protocol", AGENT_SYSTEM_PROMPT) diff --git a/tests/test_pifs_cli.py b/tests/test_pifs_cli.py index 34e35bbae..7c2ae441a 100644 --- a/tests/test_pifs_cli.py +++ b/tests/test_pifs_cli.py @@ -93,6 +93,7 @@ def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): "embedding_dimensions": 3072, "embedding_timeout": 12.5, "embedding_base_url": "https://example.invalid/openai/", + "embedding_api_key": "config-key", } ), encoding="utf-8", @@ -104,6 +105,7 @@ def __init__(self, workspace, **kwargs): self.kwargs = kwargs monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") + monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") monkeypatch.setattr(cli, "PageIndexFileSystem", ConfiguredFileSystem) @@ -121,6 +123,41 @@ def __init__(self, workspace, **kwargs): assert os.environ["PIFS_EMBEDDING_BASE_URL"] == "https://ignored.invalid/" +@pytest.mark.parametrize( + ("config_key", "pifs_key", "openai_key", "expected"), + [ + ("config-key", None, None, "config-key"), + ("config-key", "pifs-key", "openai-key", "pifs-key"), + ("config-key", None, "openai-key", "config-key"), + (None, None, "openai-key", "openai-key"), + ], +) +def test_cli_embedding_api_key_precedence( + config_key, pifs_key, openai_key, expected, monkeypatch, tmp_path +): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + config_path.parent.mkdir(parents=True, exist_ok=True) + config = {"workspace": str(tmp_path / "workspace")} + if config_key is not None: + config["embedding_api_key"] = config_key + config_path.write_text(json.dumps(config), encoding="utf-8") + if pifs_key is None: + monkeypatch.delenv("PIFS_EMBEDDING_API_KEY", raising=False) + else: + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", pifs_key) + if openai_key is None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + else: + monkeypatch.setenv("OPENAI_API_KEY", openai_key) + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + + filesystem = cli._filesystem_from_workspace(str(tmp_path / "workspace")) + + assert filesystem.kwargs["summary_projection_embedding_api_key"] == expected + + def test_cli_passthrough_invokes_pifs_command_executor(monkeypatch, capsys, tmp_path): from pageindex.filesystem import cli diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 40296708f..3df7f351a 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -654,6 +654,64 @@ def test_runtime_requires_projection_pair_when_catalog_only_has_active_files( assert workspace_state(workspace) == before +@pytest.mark.parametrize( + "mutation", + ["missing_vector", "extra_vector", "source_type_mismatch", "wrong_blob_length"], +) +def test_runtime_rejects_projection_document_vector_mismatches_without_mutation( + mutation, tmp_path, monkeypatch +): + import sqlite_vec + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha vector consistency evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + summary_path = workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + with connect_summary_database(summary_path) as connection: + row = connection.execute( + "SELECT rowid, source_type, embedding FROM semantic_index_vec" + ).fetchone() + assert row is not None + if mutation == "missing_vector": + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", (row[0],) + ) + elif mutation == "extra_vector": + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (999, 'markdown', ?)", + (sqlite_vec.serialize_float32([0.0, 0.0, 1.0]),), + ) + elif mutation == "source_type_mismatch": + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", (row[0],) + ) + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (?, 'pdf', ?)", + (row[0], row[2]), + ) + else: + connection.execute( + "UPDATE semantic_index_vec_vector_chunks00 SET vectors = zeroblob(4)" + ) + before = workspace_state(workspace) + + with pytest.raises(RuntimeError, match="migrate this workspace"): + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + @pytest.mark.parametrize( "mutation", [ @@ -909,13 +967,18 @@ def test_cli_add_creates_migration_compatible_summary_and_cache_v2( from pageindex.filesystem.cli import main install_network_fakes(monkeypatch) - write_embedding_config(tmp_path, monkeypatch) + config = write_embedding_config(tmp_path, monkeypatch) + config_values = json.loads(config.read_text(encoding="utf-8")) + config_values["embedding_api_key"] = "config-secret" + config.write_text(json.dumps(config_values), encoding="utf-8") + monkeypatch.delenv("PIFS_EMBEDDING_API_KEY") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) source = tmp_path / "notes.md" source.write_text("alpha evidence", encoding="utf-8") workspace = tmp_path / "workspace" assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 - capsys.readouterr() + output = capsys.readouterr().out projection_dir = workspace / "artifacts" / "projection_indexes" with sqlite3.connect(projection_dir / "summary.sqlite") as summary: @@ -958,7 +1021,8 @@ def test_cli_add_creates_migration_compatible_summary_and_cache_v2( projection_dir / "summary.sqlite", projection_dir / "embedding_cache.sqlite", ): - assert b"runtime-secret" not in database.read_bytes() + assert b"config-secret" not in database.read_bytes() + assert "config-secret" not in output def test_cli_add_reports_path_without_persistence_identity(tmp_path, monkeypatch, capsys): @@ -1108,7 +1172,229 @@ def test_cli_cat_reads_structure_without_internal_identity(tmp_path, monkeypatch ] -def test_tree_virtual_value_uses_one_axis_equals_value_segment( +def create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + records = [ + ( + "current.md", + "alpha current evidence", + "/documents", + {"year": 2024, "ticker": "AAPL", "sector": "finance/tech"}, + ), + ( + "filing.md", + "alpha filing evidence", + "/documents/sec-filings", + {"year": 2024, "ticker": "AAPL", "doc_type": "10-K"}, + ), + ( + "prior.md", + "alpha prior evidence", + "/documents", + {"year": 2023, "ticker": "MSFT"}, + ), + ("archive.md", "archive evidence", "/archive", {"region": "EMEA"}), + ] + for filename, content, folder, metadata in records: + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/{filename}", + json.dumps(metadata), + ] + ) == 0 + capsys.readouterr() + return workspace + + +def test_tree_folder_trailing_slash_lists_scope_local_metadata_axes( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + workspace = create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys) + + payloads = [] + for folder_path in ("/documents", "/documents/"): + assert main( + ["--workspace", str(workspace), "tree", folder_path, "-L", "1"] + ) == 0 + payloads.append(json.loads(capsys.readouterr().out)) + + assert payloads[1]["data"] == payloads[0]["data"] + tree = payloads[0]["data"]["tree"] + assert tree["path"] == "/documents" + axes = { + row["name"]: row + for row in tree["folders"] + if row["type"] == "metadata_axis" + } + + assert set(axes) == {"@doc_type", "@sector", "@ticker", "@year"} + assert {name: row["path"] for name, row in axes.items()} == { + "@doc_type": "/documents/@doc_type", + "@sector": "/documents/@sector", + "@ticker": "/documents/@ticker", + "@year": "/documents/@year", + } + assert {row["type"] for row in axes.values()} == {"metadata_axis"} + physical = [row for row in tree["folders"] if row["type"] == "folder"] + assert [(row["name"], row["path"]) for row in physical] == [ + ("sec-filings", "/documents/sec-filings") + ] + assert all(row["type"] != "file" for row in tree["folders"]) + assert "@region" not in axes + + +def test_tree_metadata_axis_trailing_slash_lists_actionable_paginated_values( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + workspace = create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys) + + payloads = [] + for axis_path in ("/documents/@year", "/documents/@year/"): + assert main( + ["--workspace", str(workspace), "tree", axis_path, "-L", "1"] + ) == 0 + payloads.append(json.loads(capsys.readouterr().out)) + + values = payloads[0]["data"]["tree"]["folders"] + assert payloads[1]["data"] == payloads[0]["data"] + assert [(row["name"], row["type"], row["path"]) for row in values] == [ + ("2024", "metadata_value", "/documents/@year/2024"), + ("2023", "metadata_value", "/documents/@year/2023"), + ] + assert payloads[0]["data"]["pagination"] == { + "page": 1, + "page_size": 50, + "has_more": False, + "next_page": None, + } + + selected = values[0]["path"] + scoped_payloads = [] + for value_path in (selected, f"{selected}/"): + assert main( + ["--workspace", str(workspace), "tree", value_path, "-L", "1"] + ) == 0 + scoped_payloads.append(json.loads(capsys.readouterr().out)) + assert scoped_payloads[1]["data"] == scoped_payloads[0]["data"] + selected_tree = scoped_payloads[0]["data"]["tree"] + assert selected_tree["path"] == selected + assert {row["title"] for row in selected_tree["files"]} == { + "current.md", + "filing.md", + } + selected_folders = { + (row["type"], row["name"]): row["path"] + for row in selected_tree["folders"] + } + assert selected_folders == { + ("folder", "sec-filings"): "/documents/sec-filings/@year/2024", + ("metadata_axis", "@doc_type"): "/documents/@year/2024/@doc_type", + ("metadata_axis", "@sector"): "/documents/@year/2024/@sector", + ("metadata_axis", "@ticker"): "/documents/@year/2024/@ticker", + } + assert main( + ["--workspace", str(workspace), "browse", selected, "alpha"] + ) == 0 + documents = json.loads(capsys.readouterr().out)["data"]["documents"] + assert {row["title"] for row in documents} == {"current.md", "filing.md"} + + locator = selected_tree["files"][0]["path"] + for command in ( + ["stat", locator], + ["cat", locator, "--structure"], + ["grep", "alpha", locator], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + capsys.readouterr() + + child_scope = selected_folders[("folder", "sec-filings")] + assert main(["--workspace", str(workspace), "tree", child_scope, "-L", "1"]) == 0 + child_tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert [row["title"] for row in child_tree["files"]] == ["filing.md"] + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@sector/", "-L", "1"] + ) == 0 + encoded = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"][0] + assert encoded["path"] == "/documents/@sector/finance%2Ftech" + assert main( + ["--workspace", str(workspace), "tree", encoded["path"], "-L", "1"] + ) == 0 + capsys.readouterr() + + +def test_tree_metadata_values_keep_fixed_fifty_item_pagination( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "tickers.md" + source.write_text("alpha ticker evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + tickers = [f"T{index:02d}" for index in range(55)] + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/tickers.md", + json.dumps({"ticker": tickers}), + ] + ) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@ticker/", "-L", "1"] + ) == 0 + first = json.loads(capsys.readouterr().out)["data"] + assert [row["value"] for row in first["tree"]["folders"]] == tickers[:50] + assert first["pagination"] == { + "page": 1, + "page_size": 50, + "has_more": True, + "next_page": 2, + } + + assert main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@ticker", + "--page", + "2", + ] + ) == 0 + second = json.loads(capsys.readouterr().out)["data"] + assert [row["value"] for row in second["tree"]["folders"]] == tickers[50:] + assert second["pagination"] == { + "page": 2, + "page_size": 50, + "has_more": False, + "next_page": None, + } + + +def test_metadata_virtual_paths_use_alternating_encoded_field_value_segments( tmp_path, monkeypatch, capsys ): from pageindex.filesystem.cli import main @@ -1127,7 +1413,7 @@ def test_tree_virtual_value_uses_one_axis_equals_value_segment( str(workspace), "setmeta", "/documents/notes.md", - '{"year": 2024}', + '{"year": 2024, "sector": "finance/tech"}', ] ) == 0 @@ -1135,8 +1421,12 @@ def test_tree_virtual_value_uses_one_axis_equals_value_segment( capsys.readouterr() assert main(["--workspace", str(workspace), "tree", "/documents/@year"]) == 0 - values = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"] - assert values[0]["path"] == "/documents/@year=2024" + year_payload = json.loads(capsys.readouterr().out) + values = year_payload["data"]["tree"]["folders"] + assert values[0]["path"] == "/documents/@year/2024" + assert year_payload["next_steps"] == [ + 'browse /documents/@year/ ""' + ] assert ( main( @@ -1144,15 +1434,44 @@ def test_tree_virtual_value_uses_one_axis_equals_value_segment( "--workspace", str(workspace), "tree", - "/documents/@year=2024", + "/documents/@year/2024", "-L", "1", ] ) == 0 ) - files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] - assert files[0]["path"] == "/documents/@year=2024/notes.md" + scoped_tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert scoped_tree["files"][0]["path"] == "/documents/@year/2024/notes.md" + assert [folder["path"] for folder in scoped_tree["folders"]] == [ + "/documents/@year/2024/@sector" + ] + + assert main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@year/2024/@sector", + ] + ) == 0 + sector = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"][0] + assert sector["path"] == "/documents/@year/2024/@sector/finance%2Ftech" + + locator = f"{sector['path']}/notes.md" + for command in ( + ["stat", locator], + ["cat", locator, "--structure"], + ["grep", "alpha", locator], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@year=2024"] + ) == 2 + error = json.loads(capsys.readouterr().out)["error"] + assert "@field/value" in error["message"] def test_setmeta_translates_identity_at_the_cli_boundary( @@ -1248,7 +1567,7 @@ def test_metadata_scoped_setmeta_returns_the_post_update_actionable_path( ] ) == 0 capsys.readouterr() - old_path = "/documents/@year=2024/notes.md" + old_path = "/documents/@year/2024/notes.md" command = ["--workspace", str(workspace), "setmeta"] if operation == "clear": command.extend(["--clear", old_path]) @@ -1323,7 +1642,7 @@ def test_duplicate_virtual_leaves_use_actionable_paths_without_internal_ids( capsys.readouterr() assert main( - ["--workspace", str(workspace), "tree", "/documents/@year=2024", "-L", "1"] + ["--workspace", str(workspace), "tree", "/documents/@year/2024", "-L", "1"] ) == 0 files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] paths = [row["path"] for row in files] @@ -1442,6 +1761,63 @@ def create(self, *, model, input, dimensions): ).fetchone()[0] == 0 +@pytest.mark.parametrize("cache_state", ["fresh", "preexisting_shared_key"]) +def test_add_restores_embedding_cache_when_vec_upsert_fails( + cache_state, tmp_path, monkeypatch +): + from pageindex.filesystem.semantic_index import SQLiteVecSemanticIndex + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + source_dir = tmp_path / "add" + source_dir.mkdir() + source = source_dir / "notes.md" + source.write_text("alpha shared cache evidence", encoding="utf-8") + + if cache_state == "preexisting_shared_key": + existing_dir = tmp_path / "existing" + existing_dir.mkdir() + existing = existing_dir / "notes.md" + existing.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + filesystem.register_file( + storage_uri=existing.as_uri(), + folder_path="/documents/existing", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + ) + else: + create_projection_v2(workspace) + filesystem = open_test_filesystem(workspace) + + baseline = registration_logical_state(workspace) + baseline_uploads = sorted( + path.relative_to(workspace).as_posix() + for path in workspace.glob("artifacts/uploads/**/*") + ) + + def fail_vec_upsert(self, records): + raise RuntimeError("vec upsert unavailable after cache write") + + monkeypatch.setattr(SQLiteVecSemanticIndex, "upsert_many", fail_vec_upsert) + + with pytest.raises(RuntimeError, match="vec upsert unavailable after cache write"): + filesystem.add_file(source, "/documents/new") + + assert registration_logical_state(workspace) == baseline + assert sorted( + path.relative_to(workspace).as_posix() + for path in workspace.glob("artifacts/uploads/**/*") + ) == baseline_uploads + reopened = open_test_filesystem(workspace) + assert registration_logical_state(workspace) == baseline + if cache_state == "preexisting_shared_key": + assert reopened.store.file_refs_for_scope() == filesystem.store.file_refs_for_scope() + else: + assert reopened.store.file_refs_for_scope() == [] + + def test_register_file_completes_summary_projection_for_immediate_and_reopen_browse( tmp_path, monkeypatch ): @@ -1571,6 +1947,84 @@ def create(self, *, model, input, dimensions): assert reopened.store.file_refs_for_scope() == [] +def test_register_files_restores_existing_document_when_later_projection_upsert_fails( + tmp_path, monkeypatch, capsys +): + import openai + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + original = tmp_path / "original.md" + replacement = tmp_path / "replacement.md" + failing = tmp_path / "failing.md" + original.write_text("alpha original evidence", encoding="utf-8") + replacement.write_text("alpha replacement evidence", encoding="utf-8") + failing.write_text("beta second evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=original.as_uri(), + external_id="doc-existing", + folder_path="/documents", + title=original.name, + content_type="text/markdown", + content=original.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + baseline = registration_logical_state(workspace) + + class FailingBetaOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + if any("beta" in str(text).lower() for text in input): + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingBetaOpenAI) + reopened = open_test_filesystem(workspace) + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + reopened.register_files( + [ + { + "storage_uri": replacement.as_uri(), + "external_id": "doc-existing", + "folder_path": "/documents/replacement", + "title": replacement.name, + "content_type": "text/markdown", + "content": replacement.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": failing.as_uri(), + "external_id": "doc-failing", + "folder_path": "/documents/new", + "title": failing.name, + "content_type": "text/markdown", + "content": failing.read_text(encoding="utf-8"), + "metadata": {"year": 2025}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + strict_reopen = open_test_filesystem(workspace) + assert strict_reopen.store.file_refs_for_scope() == [existing_ref] + entry = strict_reopen.store.get_file(existing_ref) + assert entry.metadata["year"] == 2023 + assert entry.storage_uri == original.as_uri() + assert strict_reopen.pageindex_structure(existing_ref)["available"] is True + for command in ( + ["stat", "/documents/original.md"], + ["cat", "/documents/original.md", "--structure"], + ["grep", "original", "/documents/original.md"], + ["browse", "/documents", "original"], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + output = json.loads(capsys.readouterr().out) + assert output["success"] is True + + def test_register_files_rollback_preserves_preexisting_catalog_projection_and_cache( tmp_path, monkeypatch ): @@ -2123,13 +2577,13 @@ def test_browse_respects_physical_and_virtual_scope( "--workspace", str(workspace), "browse", - "/documents/@year=2024", + "/documents/@year/2024", "alpha", ] ) == 0 virtual = json.loads(capsys.readouterr().out)["data"]["documents"] assert [row["title"] for row in virtual] == ["current.md"] - assert virtual[0]["path"] == "/documents/@year=2024/current.md" + assert virtual[0]["path"] == "/documents/@year/2024/current.md" def test_browse_requires_query_and_an_existing_summary_projection( @@ -2285,7 +2739,7 @@ def test_retired_command_forms_return_structured_errors(command, tmp_path, capsy assert payload["error"]["code"] == "invalid_command" -def test_set_workspace_persists_runtime_config_without_credentials_or_provider( +def test_set_workspace_preserves_embedding_api_key_without_exposing_or_persisting_it_elsewhere( tmp_path, monkeypatch, capsys ): from pageindex.filesystem.cli import main @@ -2298,7 +2752,7 @@ def test_set_workspace_persists_runtime_config_without_credentials_or_provider( "embedding_model": "test-embedding", "embedding_dimensions": 3, "embedding_timeout": 12, - "embedding_api_key": "must-not-survive", + "embedding_api_key": "config-secret", "embedding_provider": "legacy-provider", } ), @@ -2308,15 +2762,17 @@ def test_set_workspace_persists_runtime_config_without_credentials_or_provider( workspace = tmp_path / "workspace" assert main(["set", "workspace", str(workspace)]) == 0 - capsys.readouterr() + output = capsys.readouterr().out assert json.loads(config.read_text(encoding="utf-8")) == { + "embedding_api_key": "config-secret", "embedding_base_url": "https://example.invalid/v1", "embedding_dimensions": "3", "embedding_model": "test-embedding", "embedding_timeout": "12", "workspace": str(workspace), } + assert "config-secret" not in output def test_public_example_is_a_short_supported_cli_walkthrough(): From 0b105ec7d3a8af1d30cd59c4e511fdf81c74e8a1 Mon Sep 17 00:00:00 2001 From: BukeLy Date: Tue, 14 Jul 2026 19:52:24 +0800 Subject: [PATCH 14/14] fix(filesystem): encode metadata dot segments Percent-encode dot-only metadata values so tree locators survive path normalization and remain actionable through tree, browse, and stat. Refs BukeLy/Pageindex#47 --- pageindex/filesystem/core.py | 5 ++- tests/test_pifs_simplification_acceptance.py | 46 ++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index df04e7ab0..fb2c48be3 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -613,7 +613,10 @@ def scope_stat(self, path: str) -> dict[str, Any]: @staticmethod def encode_scope_segment(segment: Any) -> str: - return quote(str(segment), safe="") + value = str(segment) + if value in {".", ".."}: + return value.replace(".", "%2E") + return quote(value, safe="") def browse_semantic_files( self, diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py index 3df7f351a..3708f67b2 100644 --- a/tests/test_pifs_simplification_acceptance.py +++ b/tests/test_pifs_simplification_acceptance.py @@ -1394,6 +1394,52 @@ def test_tree_metadata_values_keep_fixed_fifty_item_pagination( } +def test_tree_metadata_dot_values_return_actionable_encoded_scope_paths( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha dot value evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + json.dumps({"tag": [".", ".."]}), + ] + ) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents/@tag"]) == 0 + values = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"] + assert {row["value"]: row["path"] for row in values} == { + ".": "/documents/@tag/%2E", + "..": "/documents/@tag/%2E%2E", + } + + for scope_path in (row["path"] for row in values): + assert main( + ["--workspace", str(workspace), "tree", scope_path, "-L", "1"] + ) == 0 + file_path = json.loads(capsys.readouterr().out)["data"]["tree"]["files"][0][ + "path" + ] + assert main( + ["--workspace", str(workspace), "browse", scope_path, "alpha"] + ) == 0 + document = json.loads(capsys.readouterr().out)["data"]["documents"][0] + assert document["path"] == file_path + assert main(["--workspace", str(workspace), "stat", file_path]) == 0 + capsys.readouterr() + + def test_metadata_virtual_paths_use_alternating_encoded_field_value_segments( tmp_path, monkeypatch, capsys ):