diff --git a/examples/pifs_demo.py b/examples/pifs_demo.py index 4f2887986..9a7e22a78 100644 --- a/examples/pifs_demo.py +++ b/examples/pifs_demo.py @@ -1,781 +1,46 @@ -""" -PageIndex FileSystem (PIFS) agent demo. - -This mirrors examples/agentic_vectorless_rag_demo.py, but exposes a corpus -through the PageIndex FileSystem shell instead of direct PageIndex document -tools. The agent receives one read-only bash-like PIFS tool and must retrieve -evidence through tree, browse, stat, cat --structure, cat --page, and -single-file grep. - -The demo registers supported files under examples/documents. When a matching -examples/documents/results/*_structure.json file exists, it is loaded into the -PIFS workspace's PageIndexClient cache. Files without a cache exercise the -normal PageIndexClient.index() path during register(). - -Requirements: - pip install openai-agents +"""A minimal PageIndex FileSystem CLI walkthrough. -Example: - python examples/pifs_demo.py --stream-mode all --verbose +Configure the Summary Embedding Profile in your PIFS config and provide its API +key through ``PIFS_EMBEDDING_API_KEY`` before running this example. """ from __future__ import annotations import argparse -import hashlib -import json -import os -import re -import shlex -import shutil -import sys -import time +import subprocess from pathlib import Path -from typing import Any - -import PyPDF2 - -sys.path.insert(0, str(Path(__file__).parent.parent)) - -# Keep the local demo quiet in offline environments. -os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "true") -from pageindex import PageIndexClient -from pageindex.filesystem import MetadataGenerator, PageIndexFileSystem, PIFSCommandExecutor -from pageindex.filesystem.agent import run_pifs_agent -from pageindex.filesystem.core import DEFAULT_EMBEDDING_DIMENSIONS - -EXAMPLES_DIR = Path(__file__).parent -DOCUMENTS_DIR = EXAMPLES_DIR / "documents" -WORKSPACE = EXAMPLES_DIR / "pifs_workspace" -DEFAULT_MODEL = os.environ.get("PIFS_DEMO_MODEL", "gpt-5.4") -DEFAULT_METADATA_PROVIDER = os.environ.get("PIFS_DEMO_METADATA_PROVIDER") or os.environ.get( - "PIFS_METADATA_PROVIDER", "openai" -) -DEFAULT_EMBEDDING_PROVIDER = os.environ.get("PIFS_DEMO_EMBEDDING_PROVIDER") or os.environ.get( - "PIFS_EMBEDDING_PROVIDER", "openai" -) -DEFAULT_QUESTION = ( - "Use the PIFS workspace to find the Federal Reserve annual report. " - "Which section covers supervision and regulation, and what page range " - "should I inspect? Cite the document and evidence you used." -) - -PIFS_DEMO_AGENT_PROMPT = """ -You are a PageIndex FileSystem retrieval agent for a local demo workspace. - -Use only the bash tool. It is a read-only PIFS virtual shell, not a real OS -shell. The workspace contains registered example PDFs. - -Retrieval strategy: -- Start with tree to understand the workspace. -- Use concrete PIFS paths from tree/browse output, such as /documents/report.pdf, - or stable file_ref/document ids. Do not invent temporary ref_N aliases. -- Folder paths such as /documents are positional command targets; do not put - folder paths inside --where. -- After choosing a folder, use browse with a required quoted query to find - likely files, for example: - browse /documents "Federal Reserve supervision regulation" -- If the folder is uncertain, use recursive browse from a structural parent, - for example: - browse -R /documents "Federal Reserve supervision regulation" -- Use recursive browse only after inspecting plausible folders or rephrasing. -- browse returns document candidates only; it is not final evidence. -- After browse returns candidates, verify evidence with grep, cat - --structure, or cat --page before answering. -- Use browse --where with JSON metadata DSL when metadata pruning is needed. -- Use grep only for one selected file. -- Do not use find, recursive grep, folder grep, pipes, browse spaces, or stat - schema/field modes. -- Run one evidence command at a time. Do not chain large commands like - cat --structure, grep, and cat --page in one bash call. -- For PDFs, use cat --structure to inspect the PageIndex tree, then - cat --page for evidence, for example: - cat /documents/2023-annual-report.pdf --page 31-35 -- Do not use cat --page as the first inspection command for a selected PDF. - Run cat --structure for that same target first, then choose pages. -- Do not guess cat --page ranges from grep line numbers. -- For page-range questions, use cat --structure to identify the full section - range. Then run cat --page on the smallest useful evidence range, usually the - section start page or first 1-2 pages, before the final answer. Do not print - a broad multi-page section unless the user asks to read the whole section. -- Answer only from PIFS tool output and cite file refs or document ids. -""" +def run(workspace: Path, document: Path, query: str) -> int: + target = f"/documents/{document.name}" + commands = [ + ["add", str(document), "/documents"], + ["tree", "/documents", "-L", "1"], + ["browse", "/documents", query], + ["stat", target], + ["cat", target, "--structure"], + ["cat", target, "--page", "1"], + ["grep", query, target], + ] + for command in commands: + print(f"\n$ pifs {' '.join(command)}") + result = subprocess.run( + ["pifs", "--workspace", str(workspace), *command], + check=False, + ) + if result.returncode: + return result.returncode + return 0 def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="Run a PIFS document retrieval agent demo.") - parser.add_argument("--workspace", type=Path, default=WORKSPACE) - parser.add_argument("--documents-dir", type=Path, default=DOCUMENTS_DIR) - parser.add_argument( - "--document", - action="append", - default=[], - help="Specific document filename or path to register. May be repeated.", - ) - parser.add_argument( - "--max-docs", - type=int, - default=0, - help="Limit number of cached example documents to register. 0 means all.", - ) - parser.add_argument("--reset", action="store_true", help="Delete and rebuild the demo workspace.") - parser.add_argument( - "--prepare-only", - action="store_true", - help="Register documents and print PIFS smoke commands without running the agent.", - ) - parser.add_argument("--question", default=DEFAULT_QUESTION) - parser.add_argument("--model", default=DEFAULT_MODEL) - parser.add_argument( - "--metadata-provider", - default=DEFAULT_METADATA_PROVIDER, - help="Provider used for register-time metadata generation.", - ) - parser.add_argument( - "--metadata-model", - default=os.environ.get("PIFS_METADATA_MODEL", "gpt-5-nano"), - help="Model used for register-time metadata generation.", - ) - parser.add_argument("--stream-mode", default="all", choices=["off", "tools", "model", "all"]) - parser.add_argument("--verbose", action="store_true") - parser.add_argument("--max-turns", type=int, default=12) - parser.add_argument("--max-seconds", type=float, default=90) - parser.add_argument("--reasoning-effort", default=None) - parser.add_argument("--reasoning-summary", default="auto") - parser.add_argument( - "--embedding-provider", - default=DEFAULT_EMBEDDING_PROVIDER, - help="Provider used for register-time summary projection embeddings.", - ) - parser.add_argument( - "--embedding-model", - default=os.environ.get("PIFS_DEMO_EMBEDDING_MODEL", "text-embedding-3-small"), - help="Embedding model used for register-time summary projection.", - ) - parser.add_argument( - "--embedding-dimensions", - type=int, - default=DEFAULT_EMBEDDING_DIMENSIONS, - ) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("document", type=Path, help="PDF or Markdown document") + parser.add_argument("--workspace", type=Path, default=Path("pifs-workspace")) + parser.add_argument("--query", default="key findings") return parser.parse_args() -def require_runtime_environment(*, metadata_provider: str, embedding_provider: str) -> None: - metadata_provider = metadata_provider.lower() - embedding_provider = embedding_provider.lower() - missing: list[str] = [] - if not os.environ.get("OPENAI_API_KEY"): - missing.append("OPENAI_API_KEY for the OpenAI Agents SDK demo agent") - if metadata_provider == "openai" and not ( - os.environ.get("PIFS_METADATA_API_KEY") or os.environ.get("OPENAI_API_KEY") - ): - missing.append("PIFS_METADATA_API_KEY or OPENAI_API_KEY for metadata generation") - if embedding_provider == "openai" and not ( - os.environ.get("PIFS_EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY") - ): - missing.append("PIFS_EMBEDDING_API_KEY or OPENAI_API_KEY for summary embeddings") - if missing: - raise RuntimeError( - "Missing required environment variable(s): " - + "; ".join(missing) - + ". Source your .env or export the required key before running." - ) - - -SUPPORTED_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown", ".txt", ".text"} - - -def discover_documents(documents_dir: Path) -> list[Path]: - return sorted( - path - for path in documents_dir.iterdir() - if path.is_file() and path.suffix.lower() in SUPPORTED_DOCUMENT_SUFFIXES - ) - - -def resolve_requested_documents(documents_dir: Path, requested: list[str]) -> list[Path]: - if not requested: - return discover_documents(documents_dir) - paths: list[Path] = [] - for item in requested: - path = Path(item).expanduser() - if not path.is_absolute(): - path = documents_dir / path - if not path.exists(): - raise FileNotFoundError(f"document not found: {path}") - paths.append(path) - return paths - - -def structure_path_for(document_path: Path, documents_dir: Path) -> Path | None: - path = documents_dir / "results" / f"{document_path.stem}_structure.json" - return path if path.exists() else None - - -def deterministic_doc_id(document_path: Path) -> str: - digest = hashlib.sha1(str(document_path.resolve()).encode("utf-8")).hexdigest()[:16] - return f"pifs_demo_{digest}" - - -def read_pdf_pages(document_path: Path) -> list[dict[str, Any]]: - pages: list[dict[str, Any]] = [] - with document_path.open("rb") as handle: - reader = PyPDF2.PdfReader(handle) - for page_num, page in enumerate(reader.pages, 1): - pages.append({"page": page_num, "content": page.extract_text() or ""}) - return pages - - -def load_structure_json(structure_path: Path) -> dict[str, Any]: - with structure_path.open("r", encoding="utf-8") as handle: - payload = json.load(handle) - if not isinstance(payload, dict) or not isinstance(payload.get("structure"), list): - raise ValueError(f"invalid PageIndex structure cache: {structure_path}") - return payload - - -def seed_pageindex_cache( - filesystem: PageIndexFileSystem, - document_path: Path, - *, - documents_dir: Path, -) -> str | None: - structure_path = structure_path_for(document_path, documents_dir) - if structure_path is None: - return None - - filesystem.pageindex_client_workspace.mkdir(parents=True, exist_ok=True) - meta_path = filesystem.pageindex_client_workspace / "_meta.json" - if not meta_path.exists(): - meta_path.write_text("{}", encoding="utf-8") - client = PageIndexClient(workspace=str(filesystem.pageindex_client_workspace)) - canonical_path = str(document_path.resolve()) - for doc_id, doc in client.documents.items(): - if Path(str(doc.get("path") or "")).resolve(strict=False) == Path(canonical_path): - return doc_id - - payload = load_structure_json(structure_path) - doc_id = deterministic_doc_id(document_path) - suffix = document_path.suffix.lower() - if suffix == ".pdf": - pages = read_pdf_pages(document_path) - client.documents[doc_id] = { - "id": doc_id, - "type": "pdf", - "path": canonical_path, - "doc_name": payload.get("doc_name") or document_path.name, - "doc_description": payload.get("doc_description") or "", - "page_count": len(pages), - "structure": payload["structure"], - "pages": pages, - } - elif suffix in {".md", ".markdown"}: - text = document_path.read_text(encoding="utf-8") - client.documents[doc_id] = { - "id": doc_id, - "type": "md", - "path": canonical_path, - "doc_name": payload.get("doc_name") or document_path.name, - "doc_description": payload.get("doc_description") or "", - "line_count": len(text.splitlines()), - "structure": payload["structure"], - } - else: - return None - client._save_doc(doc_id) - return doc_id - - -def content_type_for(path: Path) -> str: - suffix = path.suffix.lower() - if suffix == ".pdf": - return "application/pdf" - if suffix in {".md", ".markdown"}: - return "text/markdown" - return "text/plain" - - -def external_id_for(path: Path) -> str: - slug = "".join(ch.lower() if ch.isalnum() else "_" for ch in path.stem).strip("_") - slug = "_".join(part for part in slug.split("_") if part) - return f"example_{slug}" - - -def log_progress(message: str, *, indent: int = 0) -> None: - prefix = " " * indent - print(f"{prefix}{message}", flush=True) - - -def register_demo_metadata_schema(filesystem: PageIndexFileSystem) -> None: - filesystem.metadata.register_schema( - { - "fields": { - "source_collection": { - "type": "string", - "description": "Local example corpus collection.", - }, - "file_format": { - "type": "string", - "description": "Source file extension without the leading dot.", - }, - } - }, - source="demo", - ) - - -def backfill_registered_metadata_values(filesystem: PageIndexFileSystem, file_ref: str) -> None: - entry = filesystem.store.get_file(file_ref) - indexed_metadata = dict(entry.metadata or {}) - with filesystem.store.connect() as conn: - filesystem.store.replace_metadata_values(conn, file_ref, indexed_metadata) - - -def configure_summary_projection_backend( - filesystem: PageIndexFileSystem, - *, - embedding_provider: str, - embedding_model: str, - embedding_dimensions: int, -) -> None: - if not (filesystem.summary_projection_index_dir / "summary.sqlite").exists(): - return - filesystem.configure_semantic_projection_retrieval( - filesystem.summary_projection_index_dir, - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - ) - - -def has_ready_register_outputs(filesystem: PageIndexFileSystem, external_id: str) -> bool: - try: - file_ref = filesystem.store.resolve_file_ref(external_id) - entry = filesystem.store.get_file(file_ref) - except KeyError: - return False - status = entry.metadata_status or {} - fields = status.get("fields") or {} - required = ("summary", "doc_type", "domain", "topic") - if any(fields.get(field, {}).get("status") != "generated" for field in required): - return False - summary_projection = (status.get("projection_indexes") or {}).get("summary") or {} - return summary_projection.get("status") == "ready" - - -def register_documents( - filesystem: PageIndexFileSystem, - documents: list[Path], - *, - documents_dir: Path, -) -> list[dict[str, Any]]: - registered: list[dict[str, Any]] = [] - total = len(documents) - for index, document_path in enumerate(documents, 1): - document_path = document_path.resolve() - external_id = external_id_for(document_path) - log_progress(f"[{index}/{total}] {document_path.name}") - log_progress("PageIndex tree cache: checking examples/documents/results", indent=1) - cache_started = time.perf_counter() - cached_doc_id = seed_pageindex_cache( - filesystem, - document_path, - documents_dir=documents_dir, - ) - cache_seconds = time.perf_counter() - cache_started - if cached_doc_id: - log_progress( - f"PageIndex tree cache: ready doc_id={cached_doc_id} ({cache_seconds:.2f}s)", - indent=1, - ) - else: - log_progress( - f"PageIndex tree cache: no cached structure; register() will index if supported ({cache_seconds:.2f}s)", - indent=1, - ) - if has_ready_register_outputs(filesystem, external_id): - file_ref = filesystem.store.resolve_file_ref(external_id) - backfill_registered_metadata_values(filesystem, file_ref) - log_progress( - f"PIFS register: cached file_ref={file_ref}; metadata and summary projection already ready", - indent=1, - ) - registered.append( - { - "file_ref": file_ref, - "external_id": external_id, - "path": str(document_path), - "status": "cached", - "pageindex_doc_id": cached_doc_id, - } - ) - continue - - log_progress( - "PIFS register: running register() -> metadata generation -> summary embedding -> sqlite upsert", - indent=1, - ) - register_started = time.perf_counter() - file_ref = filesystem.register( - storage_uri=document_path.as_uri(), - folder_path="/documents", - external_id=external_id, - title=document_path.name, - content_type=content_type_for(document_path), - source_type="examples-documents", - metadata={ - "title": document_path.name, - "source_collection": "examples/documents", - "file_format": document_path.suffix.lower().lstrip("."), - }, - ) - register_seconds = time.perf_counter() - register_started - entry = filesystem.store.get_file(file_ref) - field_status = { - field: state.get("status") - for field, state in (entry.metadata_status.get("fields") or {}).items() - } - summary_projection = ( - entry.metadata_status.get("projection_indexes", {}).get("summary", {}) - ) - log_progress( - f"PIFS register: done file_ref={file_ref} ({register_seconds:.2f}s)", - indent=1, - ) - log_progress( - f"metadata: {entry.metadata_status.get('status', 'unknown')} fields={field_status}", - indent=1, - ) - log_progress( - "summary projection: " - f"{summary_projection.get('status', 'not_requested')} " - f"index={summary_projection.get('index_path', '')}", - indent=1, - ) - registered.append( - { - "file_ref": file_ref, - "external_id": external_id, - "path": str(document_path), - "status": entry.metadata_status.get("status", "unknown"), - "pageindex_tree_status": entry.pageindex_tree_status, - "pageindex_doc_id": entry.pageindex_doc_id, - } - ) - return registered - - -def print_section(title: str) -> None: - print("\n" + "#" * 78, flush=True) - print(f"# {title}", flush=True) - print("#" * 78, flush=True) - - -def print_step(title: str, detail: str = "") -> None: - print(f"\n>>> {title}", flush=True) - if detail: - print(f" {detail}", flush=True) - - -def sanitize_preview_text(text: str) -> str: - cleaned = str(text).replace("\r", "\n").replace("\f", "\n") - cleaned = "".join( - ch if ch == "\n" or ch == "\t" or ord(ch) >= 32 else " " - for ch in cleaned - ) - return "\n".join( - re.sub(r"[ \t]{2,}", " ", line).strip() - for line in cleaned.splitlines() - ) - - -def compact_lines(text: str, *, max_lines: int = 6, max_chars: int = 900) -> str: - lines = [line for line in sanitize_preview_text(text).splitlines() if line.strip()] - preview = "\n".join(lines[:max_lines]) - if len(preview) > max_chars: - preview = preview[:max_chars].rstrip() + "..." - omitted = len(lines) - min(len(lines), max_lines) - if omitted > 0: - preview += f"\n ... {omitted} more lines" - return preview - - -def find_structure_node(structure: Any, title_fragment: str) -> dict[str, Any] | None: - if isinstance(structure, list): - for item in structure: - found = find_structure_node(item, title_fragment) - if found: - return found - return None - if not isinstance(structure, dict): - return None - if title_fragment.lower() in str(structure.get("title", "")).lower(): - return structure - return find_structure_node(structure.get("nodes", []), title_fragment) - - -def page_range_for_node(node: dict[str, Any] | None) -> str: - if not node: - return "" - ranges: list[tuple[int, int]] = [] - - def collect(item: Any) -> None: - if not isinstance(item, dict): - return - start = item.get("start_index") - end = item.get("end_index") - if isinstance(start, int) and isinstance(end, int): - ranges.append((start, end)) - for child in item.get("nodes") or []: - collect(child) - - collect(node) - if not ranges: - return "" - start = min(item[0] for item in ranges) - end = max(item[1] for item in ranges) - return f"{start}-{end}" if start != end else str(start) - - -def opening_page_range_for_node(node: dict[str, Any] | None, *, max_pages: int = 2) -> str: - if not node: - return "" - ranges: list[tuple[int, int]] = [] - - def collect(item: Any) -> None: - if not isinstance(item, dict): - return - start = item.get("start_index") - end = item.get("end_index") - if isinstance(start, int) and isinstance(end, int): - ranges.append((start, end)) - for child in item.get("nodes") or []: - collect(child) - - collect(node) - if not ranges: - return "" - start = min(item[0] for item in ranges) - end = max(item[1] for item in ranges) - preview_end = min(end, start + max_pages - 1) - return f"{start}-{preview_end}" if start != preview_end else str(start) - - -def execute_json_command(executor: PIFSCommandExecutor, command: str) -> dict[str, Any]: - try: - return json.loads(executor.execute(command)) - except Exception as exc: - return {"ok": False, "error": str(exc), "data": None} - - -def show_capability( - *, - label: str, - command: str, - result: str, - raw: str = "", - verbose: bool = False, -) -> None: - print_step(label, command) - print(f" result: {result}", flush=True) - if verbose and raw: - print(" raw:", flush=True) - print(compact_lines(raw, max_lines=10, max_chars=1600), flush=True) - - -def show_registered_documents(registered: list[dict[str, Any]], *, verbose: bool = False) -> None: - print(f"\nRegistered {len(registered)} document(s):", flush=True) - for item in registered: - print( - " - " - f"{Path(str(item.get('path', ''))).name}: " - f"file_ref={item.get('file_ref')} | " - f"status={item.get('status')} | " - f"pageindex_doc_id={item.get('pageindex_doc_id')}", - flush=True, - ) - if verbose: - print("\nRaw registration records:", flush=True) - print(json.dumps(registered, ensure_ascii=False, indent=2), flush=True) - - -def run_smoke_commands( - filesystem: PageIndexFileSystem, - registered: list[dict[str, Any]], - *, - verbose: bool = False, -) -> None: - executor = PIFSCommandExecutor(filesystem) - - command = "tree / -L 2" - tree = execute_json_command(executor, command) - tree_data = tree.get("data") or {} - tree_root = tree_data.get("tree") or {} - show_capability( - label="Folder structure", - command=command, - result=( - f"{tree_data.get('total_folders', 0)} folders; " - f"root files={tree_root.get('file_count', 0)}" - ), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = "ls /documents" - listing = execute_json_command(executor, command) - listing_data = listing.get("data") or {} - show_capability( - label="List alias", - command=command, - result=f"ls delegates to tree depth={listing_data.get('depth')}", - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = 'browse /documents "Federal Reserve annual report supervision regulation section page range"' - browse = execute_json_command(executor, command) - browse_hits = ((browse.get("data") or {}).get("documents") or []) - if browse_hits: - summary_result = f"{len(browse_hits)} browse candidates; top={browse_hits[0].get('document_id')}" - else: - summary_result = "browse is available, but this tiny two-doc demo returned no candidates" - show_capability( - label="Relevance browse", - command=command, - result=summary_result, - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - first_target = f"/documents/{Path(str(registered[0]['path'])).name}" if registered else None - if not first_target: - return - - command = f"stat {first_target}" - stat = execute_json_command(executor, command) - stat_data = (stat.get("data") or {}).get("document") or {} - show_capability( - label="File stat", - command=command, - result=( - f"{stat_data.get('title')} | tree={stat_data.get('status')} | " - f"metadata_status={(stat_data.get('metadata_status') or {}).get('status')}" - ), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = f"cat {first_target} --structure" - structure_payload = execute_json_command(executor, command) - structure_data = structure_payload.get("data") or {} - structure = structure_data.get("structure") or [] - supervision_node = find_structure_node(structure, "Supervision and Regulation") - supervision_range = page_range_for_node(supervision_node) - show_capability( - label="PageIndex document structure", - command=command, - result=( - "found section 'Supervision and Regulation'" - + (f" with page span {supervision_range}" if supervision_range else "") - ), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - evidence_range = opening_page_range_for_node(supervision_node) or "1-2" - command = f"cat {first_target} --page {evidence_range}" - page = execute_json_command(executor, command) - page_text = str(((page.get("data") or {}).get("content") or {}).get("text") or "") - show_capability( - label="Page evidence", - command=command, - result=compact_lines(page_text, max_lines=3, max_chars=420), - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - command = f'grep "Supervision and Regulation" {shlex.quote(first_target)}' - grep = execute_json_command(executor, command) - grep_hits = ((grep.get("data") or {}).get("matches") or []) - show_capability( - label="Lexical grep", - command=command, - result=f"{len(grep_hits)} real text matches", - raw=executor.execute(command) if verbose else "", - verbose=verbose, - ) - - -def main() -> None: - args = parse_args() - require_runtime_environment( - metadata_provider=args.metadata_provider, - embedding_provider=args.embedding_provider, - ) - workspace = args.workspace.expanduser() - documents_dir = args.documents_dir.expanduser() - if args.reset and workspace.exists(): - shutil.rmtree(workspace) - workspace.mkdir(parents=True, exist_ok=True) - - documents = resolve_requested_documents(documents_dir, args.document) - if args.max_docs > 0: - documents = documents[: args.max_docs] - if not documents: - raise RuntimeError(f"no cached example documents found under {documents_dir}") - - filesystem = PageIndexFileSystem( - workspace, - metadata_generator=MetadataGenerator( - provider=args.metadata_provider, - model=args.metadata_model, - ), - summary_projection_embedding_provider=args.embedding_provider, - summary_projection_embedding_model=args.embedding_model, - summary_projection_embedding_dimensions=args.embedding_dimensions, - ) - register_demo_metadata_schema(filesystem) - - print_section("STEP 1/3 Register Documents") - print(f"Workspace: {workspace}", flush=True) - print(f"Documents: {len(documents)}", flush=True) - registered = register_documents(filesystem, documents, documents_dir=documents_dir) - configure_summary_projection_backend( - filesystem, - embedding_provider=args.embedding_provider, - embedding_model=args.embedding_model, - embedding_dimensions=args.embedding_dimensions, - ) - show_registered_documents(registered, verbose=args.verbose) - - print_section("STEP 2/3 Explore PIFS Tool Surface") - run_smoke_commands(filesystem, registered, verbose=args.verbose) - - if args.prepare_only: - return - - print_section("STEP 3/3 Ask An Agent Using Only PIFS") - print(f"Question: {args.question}", flush=True) - answer = run_pifs_agent( - filesystem, - args.question, - model=args.model, - root="/", - system_prompt=PIFS_DEMO_AGENT_PROMPT, - max_turns=args.max_turns, - max_seconds=args.max_seconds, - verbose=args.verbose, - stream_mode=args.stream_mode, - reasoning_effort=args.reasoning_effort, - reasoning_summary=args.reasoning_summary, - ) - if answer: - print("\nFinal answer:", flush=True) - print(answer, flush=True) - - if __name__ == "__main__": - main() + args = parse_args() + raise SystemExit(run(args.workspace, args.document, args.query)) diff --git a/pageindex/filesystem/__init__.py b/pageindex/filesystem/__init__.py index 24b4c9746..e92b682d2 100644 --- a/pageindex/filesystem/__init__.py +++ b/pageindex/filesystem/__init__.py @@ -1,51 +1,3 @@ -from importlib import import_module -from typing import TYPE_CHECKING - -from .commands import PIFSCommandExecutor from .core import PageIndexFileSystem -from .types import OpenResult, SearchResult - -if TYPE_CHECKING: - from .semantic_projection import SemanticProjectionSearchBackend - from .semantic_projection import SummaryProjectionIndexer - from .semantic_index import ( - RebuildableSemanticIndex, - SemanticIndexRecord, - SemanticSearchResult, - SQLiteVecSemanticIndex, - ) - -_LAZY_EXPORTS = { - "SemanticProjectionSearchBackend": (".semantic_projection", "SemanticProjectionSearchBackend"), - "RebuildableSemanticIndex": (".semantic_index", "RebuildableSemanticIndex"), - "SemanticIndexRecord": (".semantic_index", "SemanticIndexRecord"), - "SemanticSearchResult": (".semantic_index", "SemanticSearchResult"), - "SQLiteVecSemanticIndex": (".semantic_index", "SQLiteVecSemanticIndex"), - "SummaryProjectionIndexer": (".semantic_projection", "SummaryProjectionIndexer"), -} - -__all__ = [ - "OpenResult", - "SemanticProjectionSearchBackend", - "PIFSCommandExecutor", - "PageIndexFileSystem", - "RebuildableSemanticIndex", - "SearchResult", - "SemanticIndexRecord", - "SemanticSearchResult", - "SummaryProjectionIndexer", - "SQLiteVecSemanticIndex", -] - - -def __getattr__(name: str): - if name in _LAZY_EXPORTS: - module_name, attribute_name = _LAZY_EXPORTS[name] - value = getattr(import_module(module_name, __name__), attribute_name) - globals()[name] = value - return value - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - -def __dir__() -> list[str]: - return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS)) +__all__ = ["PageIndexFileSystem"] diff --git a/pageindex/filesystem/_embedding_identity.py b/pageindex/filesystem/_embedding_identity.py new file mode 100644 index 000000000..a55445f8d --- /dev/null +++ b/pageindex/filesystem/_embedding_identity.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from urllib.parse import urlsplit, urlunsplit + + +DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" + + +def normalize_base_url(value: str | None) -> str: + raw = str(value or DEFAULT_OPENAI_BASE_URL).strip() + parsed = urlsplit(raw) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("embedding base URL must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError( + "embedding base URL must not contain credentials, query parameters, or a fragment" + ) + return urlunsplit( + ( + parsed.scheme.lower(), + parsed.netloc.lower(), + parsed.path.rstrip("/"), + "", + "", + ) + ) + + +def normalize_model(value: str | None) -> str: + model = str(value or "").strip() + if not model: + raise ValueError("embedding model must not be empty") + return model diff --git a/pageindex/filesystem/_projection_topology.py b/pageindex/filesystem/_projection_topology.py new file mode 100644 index 000000000..8c209548c --- /dev/null +++ b/pageindex/filesystem/_projection_topology.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + + +def projection_database_paths(index_dir: str | Path) -> tuple[Path, Path]: + index_dir = Path(index_dir).expanduser() + return index_dir / "summary.sqlite", index_dir / "embedding_cache.sqlite" + + +def projection_database_path_present(path: str | Path) -> bool: + path = Path(path) + return path.exists() or path.is_symlink() + + +def projection_database_pair( + index_dir: str | Path, +) -> tuple[Path, Path] | None: + summary_path, cache_path = projection_database_paths(index_dir) + summary_exists = projection_database_path_present(summary_path) + cache_exists = projection_database_path_present(cache_path) + if not summary_exists and not cache_exists: + return None + if not summary_exists or not cache_exists: + raise RuntimeError( + "PIFS Summary Projection topology is incomplete; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + return summary_path, cache_path diff --git a/pageindex/filesystem/_sqlite_schema.py b/pageindex/filesystem/_sqlite_schema.py new file mode 100644 index 000000000..691184391 --- /dev/null +++ b/pageindex/filesystem/_sqlite_schema.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import sqlite3 +from typing import Any + + +def regular_table_names(connection: sqlite3.Connection) -> set[str]: + return { + str(row[0]) + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + + +def sqlite_schema_signature( + connection: sqlite3.Connection, + tables: set[str], +) -> dict[str, Any]: + columns = { + table: tuple( + tuple(row) + for row in connection.execute(f'PRAGMA table_xinfo("{table}")') + ) + for table in sorted(tables) + } + indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} + foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} + for table in sorted(tables): + table_indexes = [] + for row in connection.execute(f'PRAGMA index_list("{table}")'): + name = str(row[1]) + origin = str(row[3]) + index_columns = tuple( + str(column[2]) + for column in connection.execute(f'PRAGMA index_info("{name}")') + ) + table_indexes.append( + ( + name if origin == "c" else None, + int(row[2]), + origin, + int(row[4]), + index_columns, + ) + ) + indexes[table] = tuple(sorted(table_indexes, key=repr)) + foreign_keys[table] = tuple( + sorted( + ( + str(row[2]), + str(row[3]), + str(row[4]), + str(row[5]), + str(row[6]), + str(row[7]), + ) + for row in connection.execute(f'PRAGMA foreign_key_list("{table}")') + ) + ) + return { + "tables": tables, + "columns": columns, + "indexes": indexes, + "foreign_keys": foreign_keys, + } + + +def normalized_table_sql(connection: sqlite3.Connection, table: str) -> str: + row = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + (table,), + ).fetchone() + return " ".join(str(row[0] if row else "").lower().split()) diff --git a/pageindex/filesystem/_workspace_consistency.py b/pageindex/filesystem/_workspace_consistency.py new file mode 100644 index 000000000..41fbb412a --- /dev/null +++ b/pageindex/filesystem/_workspace_consistency.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + + +def _readonly_connection(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + return connection + + +def _inconsistent_workspace_error(detail: str) -> RuntimeError: + return RuntimeError( + f"Inconsistent PIFS workspace: {detail}; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + +def validate_catalog_root(catalog_path: str | Path) -> None: + try: + with _readonly_connection(Path(catalog_path)) as catalog: + root = catalog.execute( + "SELECT folder_id, parent_id, name FROM folders WHERE path = '/'" + ).fetchone() + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "the catalog canonical root folder could not be read" + ) from exc + if ( + root is None + or root["folder_id"] != "folder_root" + or root["parent_id"] is not None + or root["name"] != "/" + ): + raise _inconsistent_workspace_error( + "the catalog is missing its canonical root folder" + ) + + +def validate_workspace_consistency( + catalog_path: str | Path, + summary_path: str | Path, +) -> None: + catalog_path = Path(catalog_path) + validate_catalog_root(catalog_path) + try: + with _readonly_connection(catalog_path) as catalog: + active_file_refs = { + str(row[0]) + for row in catalog.execute( + "SELECT file_ref FROM files WHERE deleted_at IS NULL" + ) + } + with _readonly_connection(Path(summary_path)) as summary: + projected_file_refs = { + str(row[0]) + for row in summary.execute( + "SELECT file_ref FROM semantic_index_docs" + ) + } + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "catalog and Summary Projection references could not be read" + ) from exc + orphaned = sorted(projected_file_refs - active_file_refs) + if orphaned: + shown = ", ".join(orphaned[:5]) + raise _inconsistent_workspace_error( + "Summary Projection file_ref values do not reference active catalog files: " + f"{shown}" + ) + missing = sorted(active_file_refs - projected_file_refs) + if missing: + shown = ", ".join(missing[:5]) + raise _inconsistent_workspace_error( + "active catalog file_ref values are missing from the Summary Projection: " + f"{shown}" + ) + + +def validate_catalog_without_projection(catalog_path: str | Path) -> None: + catalog_path = Path(catalog_path) + validate_catalog_root(catalog_path) + try: + with _readonly_connection(catalog_path) as catalog: + active_file_refs = sorted( + str(row[0]) + for row in catalog.execute( + "SELECT file_ref FROM files WHERE deleted_at IS NULL" + ) + ) + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "active catalog references could not be read" + ) from exc + if active_file_refs: + shown = ", ".join(active_file_refs[:5]) + raise _inconsistent_workspace_error( + "active catalog files require a complete Summary Projection: " + f"{shown}" + ) diff --git a/pageindex/filesystem/agent.py b/pageindex/filesystem/agent.py index f782ecb2b..fd4176d6b 100644 --- a/pageindex/filesystem/agent.py +++ b/pageindex/filesystem/agent.py @@ -35,9 +35,9 @@ If the user asks what tools or capabilities you have, describe only the PIFS virtual shell capabilities available inside this workspace: tree, browse, -stat, cat, grep, and ls as an alias for tree -L 1. Do not mention host runtime -tools, SDK internals, or orchestration helpers that are not part of the PIFS -shell. +stat, cat, and grep. Describe PIFS Listing as tree -L 1. Do not mention +host runtime tools, SDK internals, or orchestration helpers that are not part of +the PIFS shell. If the user asks a workspace-related topic question without naming a specific file, treat it as a retrieval task. Start with tree to understand folder @@ -52,41 +52,41 @@ BASH_TOOL_DESCRIPTION = """ Run a command in the PageIndex FileSystem virtual shell. This is not a real operating-system shell. The tool is read-only and always returns JSON. -Use only: tree, browse, stat, cat, grep, and ls as an exact alias for tree -L 1. +Use only: tree, browse, stat, cat, and grep. Start broad workspace questions with tree. Then choose a physical folder, inspect metadata-derived virtual axes such as @year or @ticker with tree, and -copy returned scope paths into later commands. Run browse "" for document +select exact metadata values through paths such as @field/value. Copy returned +scope paths into later commands. Run browse "" for document discovery. Use browse -R when low-signal folder or virtual-node labels do not help prune the query, or after scoped browse misses/rephrasing. Browse returns document candidates only, not final evidence. Use stat only for identity, status, or metadata. Use cat --structure before any cat --page N[-M] for that file. Use grep only as a single-document lexical fallback. -Do not use find, recursive grep, folder grep, pipes, stat schema/field modes, -browse spaces, implicit cat reads, or general knowledge when workspace evidence -is not found. +Other shell commands, pipes, stat modes, folder-wide lexical searches, and +implicit cat reads are unsupported. """ AGENT_TOOL_POLICY = """ Tool policy: - The bash tool is a PageIndex virtual shell, not an operating-system shell. - The default agent tool surface is read-only. -- Use only tree, browse, stat, cat, grep, and ls as an alias for tree -L 1. +- Use only tree, browse, stat, cat, and grep. PIFS Listing is tree -L 1. - Folder paths such as /documents are positional command targets; never put folder paths in --where. - Start with tree to understand workspace and folder structure before document retrieval. -- After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect metadata-derived virtual value nodes, and copy returned scope paths instead of handwriting encoded values. +- After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect @field/value nodes, and copy returned scope paths instead of handwriting encoded values. - After choosing a folder or scope, use browse "" for relevance-ranked document candidates; quote multi-word queries, for example browse /documents "Federal Reserve". - Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query. - If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder. - Only after that persistence protocol may you say the workspace lacks evidence. -- browse uses summary retrieval only; do not use browse --space. +- browse always uses summary retrieval. - Use metadata scope paths for exact filters. Keep --where as a fallback for OR, IN, or contains only when tree paths cannot express the predicate. - browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims. - For financial statement, ratio, or calculation questions, prefer citing a primary statement, reconciliation table, or official table when available; use summaries, MD&A, or footnotes as supporting evidence. -- Use grep for a selected single file only. -- Do not use find, recursive grep, folder grep, pipes, schema browsing, batch metadata commands, browse spaces, implicit cat reads, or general-knowledge fallback. +- Use grep for a selected single file only. +- Other shell commands, pipes, schema browsing, batch metadata commands, folder-wide lexical searches, implicit cat reads, and general-knowledge fallback are unsupported. - Command errors are JSON; recover by trying an available command. - Use cat or grep to gather evidence before making source-backed claims. -- Do not reconstruct a file path from a title. Use exact paths returned by PIFS commands, or use file_ref/document_id when available; quote paths that contain spaces. +- Do not reconstruct a file path from a title. Use the exact path returned by PIFS commands and quote paths that contain spaces. - For broad topic, method, or "what solution" questions that are likely about the workspace, browse for candidate documents before asking the user to choose a document. - Use stat only for identity, metadata, or status questions. Do not run stat merely to understand what a document says. - Prefer target-first cat syntax with stable targets: cat --structure, cat --page 31-59. @@ -100,7 +100,7 @@ - Avoid fetching a broad page span unless page-level citation or verification is required. - Do not call cat --page ; if you need a page span, use cat --page -. - For metadata or summary-field questions, run stat for relevant files before answering. -- Distinguish default/register metadata from caller-provided custom metadata when the evidence supports it. +- Distinguish generated metadata from caller-provided custom metadata when the evidence supports it. - Do a final consistency check before answering: the headline number, calculation, unit, sign, and yes/no polarity must agree with each other. """ diff --git a/pageindex/filesystem/cli.py b/pageindex/filesystem/cli.py index 89ac1a390..8b03a65ba 100644 --- a/pageindex/filesystem/cli.py +++ b/pageindex/filesystem/cli.py @@ -26,6 +26,14 @@ ANSI_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|.)") PIFS_CONFIG_FILE_ENV = "PIFS_CONFIG_FILE" PIFS_WORKSPACE_ENV = "PIFS_WORKSPACE" +PERSISTED_CONFIG_KEYS = { + "workspace", + "embedding_api_key", + "embedding_base_url", + "embedding_model", + "embedding_dimensions", + "embedding_timeout", +} def _config_path() -> Path: @@ -45,14 +53,23 @@ def _read_config() -> dict[str, str]: payload = json.load(handle) if not isinstance(payload, dict): raise ValueError(f"invalid PIFS config file: {path}") - return {str(key): str(value) for key, value in payload.items() if value is not None} + return { + str(key): str(value) + for key, value in payload.items() + if key in PERSISTED_CONFIG_KEYS and value is not None + } def _write_config(config: dict[str, str]) -> Path: path = _config_path() path.parent.mkdir(parents=True, exist_ok=True) + persisted = { + key: value + for key, value in config.items() + if key in PERSISTED_CONFIG_KEYS and value is not None + } with path.open("w", encoding="utf-8") as handle: - json.dump(config, handle, indent=2, sort_keys=True) + json.dump(persisted, handle, indent=2, sort_keys=True) handle.write("\n") return path @@ -182,8 +199,11 @@ def _filesystem_embedding_config() -> dict[str, object]: config_values = _read_config() config: dict[str, object] = {} base_url = config_values.get("embedding_base_url") - api_key = config_values.get("embedding_api_key") - provider = config_values.get("embedding_provider") + api_key = ( + os.environ.get("PIFS_EMBEDDING_API_KEY") + or config_values.get("embedding_api_key") + or os.environ.get("OPENAI_API_KEY") + ) model = config_values.get("embedding_model") dimensions = _optional_int( "embedding_dimensions", @@ -193,8 +213,6 @@ def _filesystem_embedding_config() -> dict[str, object]: "embedding_timeout", config_values.get("embedding_timeout"), ) - if provider and provider.strip(): - config["summary_projection_embedding_provider"] = provider.strip() if model and model.strip(): config["summary_projection_embedding_model"] = model.strip() if dimensions is not None: @@ -344,7 +362,6 @@ def _run_add(argv: list[str], *, workspace: str) -> int: filesystem = _filesystem_from_workspace(workspace) info = filesystem.add_file(args.physical_path, args.virtual_target) print(f"added: {info.get('path')}") - print(f"file_ref: {info['file_ref']}") return 0 @@ -371,7 +388,15 @@ def _run_setmeta(argv: list[str], *, workspace: str) -> int: filesystem = _filesystem_from_workspace(workspace) info = filesystem.set_metadata(args.target, metadata, clear=args.clear) - print(json.dumps(info, ensure_ascii=False, sort_keys=True)) + document = { + "path": info.get("path"), + "document_id": info.get("external_id"), + "title": info.get("title"), + "status": info.get("pageindex_tree_status"), + "metadata": info.get("metadata", {}), + "metadata_status": info.get("metadata_status", {}), + } + print(json.dumps(document, ensure_ascii=False, sort_keys=True)) return 0 diff --git a/pageindex/filesystem/commands.py b/pageindex/filesystem/commands.py index 5b31d5c18..22cfdd15e 100644 --- a/pageindex/filesystem/commands.py +++ b/pageindex/filesystem/commands.py @@ -13,7 +13,7 @@ class PIFSCommandError(ValueError): class PIFSCommandExecutor: - COMMAND_NAMES = {"ls", "tree", "browse", "stat", "cat", "grep"} + COMMAND_NAMES = {"tree", "browse", "stat", "cat", "grep"} FORBIDDEN_SUBSTRINGS = (";", "`", "$(", "||", "&&", "\n", "\r") FORBIDDEN_TOKENS = {"|", ">", "<", ">>", "<<", "&"} BROWSE_PAGE_SIZE = 10 @@ -33,9 +33,8 @@ def describe_available_command_surfaces(self) -> str: [ "Available PIFS BashLike commands:", "- tree [-L depth] [--page N]: folder and metadata-scope orientation", - "- ls : exact alias for tree -L 1", '- browse "" [--page N] [--where JSON] [-R]: summary-ranked document discovery', - "- stat : single document identity or scope metadata", + "- stat : single document identity, status, and metadata", "- cat --structure | --page N[-M]: structure-first document reads", "- grep : single-document lexical evidence fallback", ] @@ -68,14 +67,6 @@ def _execute(self, command: str) -> tuple[dict[str, Any], list[str]]: raise PIFSCommandError(f"Unsupported command: {name}") return getattr(self, f"_cmd_{name}")(args) - def _cmd_ls(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: - if len(args) > 1: - raise PIFSCommandError("ls accepts exactly one optional folder target") - if args and args[0].startswith("-"): - raise PIFSCommandError("ls is only an alias for tree -L 1") - path = args[0] if args else "/" - return self._cmd_tree([path, "-L", "1"]) - def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: path = "/" depth = 10 @@ -104,9 +95,10 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: root = self._scope_root(scope, file_count=self.filesystem.scope_file_count(scope)) root["folders"] = [ { - "path": self._join_scope_path( - scope.path, - self.filesystem.encode_scope_segment(row["value"]), + "path": self._metadata_value_path( + scope.path.rsplit("/", 1)[0] or "/", + scope.metadata_axis, + row["value"], ), "name": str(row["value"]), "type": "metadata_value", @@ -129,7 +121,9 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: "next_page": page + 1 if has_more else None, }, } - next_steps = [f'browse {shlex.quote(scope.path)}/ ""'] + next_steps = [ + f'browse {shlex.quote(scope.path)}/ ""' + ] return data, next_steps page_size = self.TREE_VALUE_PAGE_SIZE needed = page * page_size + 1 @@ -138,7 +132,10 @@ def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: max_depth=depth, limit=max(self.MAX_TREE_FOLDERS, needed), ) - files = self.filesystem.scope_files(scope, limit=needed) + files = [ + self._tree_file(row) + for row in self.filesystem.scope_files(scope, limit=needed) + ] axes = self.filesystem.scope_metadata_axes(scope) tree, has_more = self._folder_tree( scope, @@ -202,18 +199,11 @@ def _cmd_browse(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: ) merged_filter = self.filesystem.merge_scope_filter(scope, where) effective_recursive = recursive or bool(scope.metadata_filter) - if not self.filesystem.has_semantic_channel("summary"): - self.filesystem.configure_existing_projection_retrieval() - if not self.filesystem.has_semantic_channel("summary"): - raise PIFSCommandError("browse summary retrieval is not available") payload = self.filesystem.browse_semantic_files( scope.path, query, - retrieval_query=query, recursive=effective_recursive, - space="summary", page=page, - page_size=self.BROWSE_PAGE_SIZE, metadata_filter=where, ) documents = [self._document_hit(row) for row in payload.get("data", [])] @@ -365,41 +355,65 @@ def _folder_tree( def _document_hit(self, row: dict[str, Any]) -> dict[str, Any]: return { "path": row.get("path"), + "document_id": row.get("external_id"), "title": row.get("title"), + "status": row.get("pageindex_tree_status"), + "rank": row.get("rank"), + "similarity": row.get("similarity"), "summary": row.get("summary", ""), "metadata": row.get("metadata", {}), + "folder_path": row.get("folder_path"), + "folder_paths": row.get("folder_paths", []), + } + + @staticmethod + def _tree_file(row: dict[str, Any]) -> dict[str, Any]: + return { + "path": row["path"], + "document_id": row.get("external_id"), + "title": row["title"], + "status": row["pageindex_tree_status"], + "type": "file", + "metadata": row.get("metadata", {}), } def _document_stat(self, target: str) -> dict[str, Any]: - info = dict(self.filesystem._stat(target)) + file_ref = self.filesystem._resolve_target(target) + entry = self.filesystem.store.get_file(file_ref) + folder_paths = [ + folder["path"] + for folder in self.filesystem.store.folder_memberships(file_ref) + ] + folder_path = self.filesystem._preferred_folder_path( + folder_paths, + entry.folder_path, + entry.folder_path, + ) + path = ( + target + if str(target).startswith("/") + else self.filesystem._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + ) return { - "path": target if str(target).startswith("/") else info.get("path"), - "file_ref": info.get("file_ref"), - "document_id": info.get("external_id") or info.get("document_id"), - "status": info.get("pageindex_tree_status") or info.get("status"), - "page_count": info.get("pageNum"), - "folders": info.get("folders", []), - "metadata": info.get("metadata", {}), - "metadata_status": info.get("metadata_status", {}), - "pageindex_doc_id": info.get("pageindex_doc_id"), - "content_type": info.get("content_type"), - "title": info.get("title") or info.get("name"), + "path": path, + "document_id": entry.external_id, + "title": entry.title, + "status": entry.pageindex_tree_status, + "content_type": entry.content_type, + "metadata": entry.metadata, + "metadata_status": entry.metadata_status, + "folder_paths": folder_paths, } def _document_from_structural_payload(self, payload: dict[str, Any], target: str) -> dict[str, Any]: - document = { - "file_ref": payload.get("file_ref"), - "document_id": payload.get("external_id"), - "status": payload.get("status"), - "pageindex_doc_id": payload.get("pageindex_doc_id"), - "available": bool(payload.get("available", True)), - } + document = self._document_stat(target) + document["available"] = bool(payload.get("available", True)) if payload.get("message"): document["message"] = payload.get("message") - try: - document.update({k: v for k, v in self._document_stat(target).items() if v is not None}) - except (KeyError, ValueError): - pass return document def _page_next_steps(self, target: str, payload: dict[str, Any]) -> list[str]: @@ -528,16 +542,19 @@ def _scope_root(self, scope: Any, *, file_count: int) -> dict[str, Any]: def _scoped_folder_path(self, scope: Any, folder_path: str) -> str: path = self._normalize_folder_path(folder_path) for field, value in getattr(scope, "metadata_filter", {}).items(): - path = self._join_scope_path( - path, - f"@{self.filesystem.encode_scope_segment(field)}", - ) - path = self._join_scope_path( - path, - self.filesystem.encode_scope_segment(value), - ) + path = self._metadata_value_path(path, field, value) return path + def _metadata_value_path(self, base: str, field: object, value: object) -> str: + axis_path = self._join_scope_path( + base, + f"@{self.filesystem.encode_scope_segment(field)}", + ) + return self._join_scope_path( + axis_path, + self.filesystem.encode_scope_segment(value), + ) + @staticmethod def _join_scope_path(base: str, segment: str) -> str: base_path = str(base or "/").rstrip("/") diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py index 9f5767ed5..fb2c48be3 100644 --- a/pageindex/filesystem/core.py +++ b/pageindex/filesystem/core.py @@ -4,22 +4,28 @@ import os import shutil import tempfile +from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import quote, unquote, urlparse +from ._projection_topology import ( + projection_database_pair, + projection_database_path_present, + projection_database_paths, +) from .metadata import MetadataQueryEngine from .store import ( SQLiteFileSystemStore, fingerprint, make_file_ref, - metadata_text, normalize_path, ) -from .types import OpenResult, PIFSQueryScope, SearchResult +from .types import PIFSQueryScope if TYPE_CHECKING: from ..client import PageIndexClient + from .semantic_projection import _EmbeddingCacheKey PROJECTION_INDEX_STATUSES = { "not_indexed", @@ -30,10 +36,6 @@ } DEFAULT_EMBEDDING_DIMENSIONS = 1024 -SEMANTIC_RETRIEVAL_CHANNELS = ("summary",) -SEMANTIC_PROJECTION_INDEX_NAMES = { - "summary": "summary", -} PAGEINDEX_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown"} PAGEINDEX_DOCUMENT_CONTENT_TYPES = { "application/pdf", @@ -48,6 +50,24 @@ } +@dataclass +class _RegistrationRollbackSnapshot: + preexisting_pageindex_doc_ids: set[str] + artifact_baselines: dict[Path, bytes | None] = field(default_factory=dict) + records: list[dict[str, Any]] = field(default_factory=list) + new_records: list[dict[str, Any]] = field(default_factory=list) + created_folder_paths: list[str] = field(default_factory=list) + new_metadata_fields: set[str] = field(default_factory=set) + new_cache_keys: set[_EmbeddingCacheKey] = field(default_factory=set) + catalog_rows: dict[str, dict[str, Any]] = field(default_factory=dict) + membership_rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + metadata_value_rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + projection_rows: dict[ + str, + tuple[dict[str, Any], dict[str, Any]], + ] = field(default_factory=dict) + + def strip_pageindex_text_fields(value: Any) -> Any: if isinstance(value, list): return [strip_pageindex_text_fields(item) for item in value] @@ -66,7 +86,6 @@ def __init__( workspace: Union[str, Path], *, summary_projection_index_dir: Union[str, Path, None] = None, - summary_projection_embedding_provider: str = "openai", summary_projection_embedding_model: str = "text-embedding-3-small", summary_projection_embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, summary_projection_embedding_timeout: float = 60, @@ -74,16 +93,37 @@ def __init__( summary_projection_embedding_base_url: str | None = None, ): self.workspace = Path(workspace).expanduser() - self.store = SQLiteFileSystemStore(self.workspace) - self.metadata = MetadataQueryEngine(self.store) - self.semantic_retrieval_backend: Any | None = None - self.summary_projection_indexer: Any | None = None self.summary_projection_index_dir = ( Path(summary_projection_index_dir).expanduser() if summary_projection_index_dir is not None else self.workspace / "artifacts" / "projection_indexes" ) - self.summary_projection_embedding_provider = summary_projection_embedding_provider + summary_path, cache_path = projection_database_paths( + self.summary_projection_index_dir + ) + catalog_path = self.workspace / "filesystem.sqlite" + catalog_present = catalog_path.exists() or catalog_path.is_symlink() + summary_present = projection_database_path_present(summary_path) + cache_present = projection_database_path_present(cache_path) + if catalog_present or summary_present or cache_present: + SQLiteFileSystemStore.validate_existing_database(catalog_path) + database_pair = projection_database_pair(self.summary_projection_index_dir) + if database_pair is not None: + from .semantic_projection import validate_projection_topology + from ._workspace_consistency import validate_workspace_consistency + + validate_projection_topology(self.summary_projection_index_dir) + validate_workspace_consistency( + catalog_path, + database_pair[0], + ) + elif catalog_present: + from ._workspace_consistency import validate_catalog_without_projection + + validate_catalog_without_projection(catalog_path) + self.store = SQLiteFileSystemStore(self.workspace) + self.metadata = MetadataQueryEngine(self.store) + self.summary_projection: Any | None = None self.summary_projection_embedding_model = summary_projection_embedding_model self.summary_projection_embedding_dimensions = summary_projection_embedding_dimensions self.summary_projection_embedding_timeout = summary_projection_embedding_timeout @@ -117,10 +157,6 @@ def register_file( ] )[0] - def register(self, **kwargs: Any) -> str: - self._ensure_register_completion_defaults() - return self.register_file(**kwargs) - def add_file( self, physical_path: Union[str, Path], @@ -144,7 +180,7 @@ def add_file( ) if self.store.file_basename_exists_in_folder(folder_path, filename): raise FileExistsError(f"File already exists at {virtual_path}") - self._ensure_add_completion_defaults() + projection = self._ensure_summary_projection() add_created_folder_paths = self._add_created_folder_paths(folder_path) file_ref = make_file_ref(virtual_path.strip("/")) uploads_dir = self.workspace / "artifacts" / "uploads" @@ -153,6 +189,8 @@ def add_file( final_dir_created = False catalog_inserted = False records: list[dict[str, Any]] = [] + cache_keys: set[_EmbeddingCacheKey] = set() + preexisting_cache_keys: set[_EmbeddingCacheKey] = set() preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() uploads_dir.mkdir(parents=True, exist_ok=True) @@ -180,6 +218,8 @@ def add_file( } ) records = [record] + cache_keys = projection.cache_keys_for_records(records) + preexisting_cache_keys = projection.existing_cache_keys(cache_keys) self._require_add_pageindex_ready(record) self._register_custom_metadata_fields(records) self.store.insert_files(records) @@ -190,16 +230,19 @@ def add_file( metadata=record["metadata"], metadata_status=record["metadata_status"], ) - self._require_add_summary_projection_ready(record) + self._require_summary_projection_ready(record, operation="add") self._sync_owned_raw_artifact(record) self._ensure_add_semantic_retrieval_ready() except Exception: if catalog_inserted: - self._cleanup_add_catalog_record(file_ref) - self._cleanup_add_summary_projection(records) + self._cleanup_catalog_record(file_ref) + self._cleanup_summary_projection_records(records) + self._cleanup_summary_projection_cache( + cache_keys - preexisting_cache_keys + ) self._cleanup_failed_register_artifacts(records) - self._cleanup_add_pageindex_cache(records, preexisting_pageindex_doc_ids) - self._cleanup_add_created_folders(add_created_folder_paths) + self._cleanup_pageindex_cache(records, preexisting_pageindex_doc_ids) + self._cleanup_created_folders(add_created_folder_paths) if final_dir_created: shutil.rmtree(final_dir, ignore_errors=True) raise @@ -209,15 +252,52 @@ def add_file( return info def register_files(self, files: list[dict[str, Any]]) -> list[str]: - records = [self._prepare_file_record(file) for file in files] - preexisting_file_refs = self._existing_file_refs(records) - new_records = [ - record for record in records if record["file_ref"] not in preexisting_file_refs + files = [ + { + **file, + "metadata": self._validated_register_metadata(file.get("metadata")), + } + for file in files ] + rollback = _RegistrationRollbackSnapshot( + preexisting_pageindex_doc_ids=self._pageindex_cache_doc_ids() + ) try: - self._register_custom_metadata_fields(records) - self.store.insert_files(records) - for record in records: + for file in files: + rollback.records.append( + self._prepare_file_record( + file, + artifact_baselines=rollback.artifact_baselines, + ) + ) + projection = self._ensure_summary_projection() if rollback.records else None + self._capture_existing_registration_rows(rollback, projection) + rollback.new_records = [ + record + for record in rollback.records + if record["file_ref"] not in rollback.catalog_rows + ] + rollback.created_folder_paths = sorted( + { + path + for record in rollback.records + for path in self._add_created_folder_paths(record["folder_path"]) + }, + key=lambda path: (path.count("/"), path), + ) + rollback.new_metadata_fields = { + name + for name in self._custom_metadata_field_names(rollback.records) + if not self.store.metadata_field_exists(name) + } + if projection is not None: + batch_cache_keys = projection.cache_keys_for_records(rollback.records) + rollback.new_cache_keys = ( + batch_cache_keys - projection.existing_cache_keys(batch_cache_keys) + ) + self._register_custom_metadata_fields(rollback.records) + self.store.insert_files(rollback.records) + for record in rollback.records: try: if self._complete_summary_projection_index(record): self.store.update_file_metadata_status( @@ -225,113 +305,61 @@ def register_files(self, files: list[dict[str, Any]]) -> list[str]: metadata=record["metadata"], metadata_status=record["metadata_status"], ) + self._require_summary_projection_ready( + record, + operation="registration", + ) self._sync_owned_raw_artifact(record) except KeyError: continue except Exception: - self._cleanup_add_summary_projection(new_records) - for record in new_records: - self._cleanup_add_catalog_record(str(record["file_ref"])) - self._cleanup_failed_register_artifacts(records) + self._cleanup_summary_projection_records(rollback.new_records) + self._restore_existing_registration_projection(rollback) + self._cleanup_summary_projection_cache(rollback.new_cache_keys) + for record in rollback.new_records: + self._cleanup_catalog_record(str(record["file_ref"])) + self._restore_existing_registration_catalog(rollback) + self._cleanup_created_folders(rollback.created_folder_paths) + self._cleanup_new_metadata_fields(rollback.new_metadata_fields) + self._cleanup_pageindex_cache( + rollback.records, + rollback.preexisting_pageindex_doc_ids, + ) + self._restore_registration_artifact_baselines( + rollback.artifact_baselines + ) raise - return [record["file_ref"] for record in records] + return [record["file_ref"] for record in rollback.records] - def _ensure_register_completion_defaults(self) -> None: - if self.summary_projection_indexer is None: - from .semantic_projection import SummaryProjectionIndexer + def _ensure_summary_projection(self) -> Any: + return self._open_summary_projection(create=True) - self.summary_projection_indexer = SummaryProjectionIndexer.from_provider( - self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - if self.semantic_retrieval_backend is None: - self.configure_semantic_projection_retrieval( - self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) + def _ensure_add_semantic_retrieval_ready(self) -> None: + projection = self._open_summary_projection(create=False) + if not projection.available: + raise RuntimeError("pifs add failed to make the Summary Projection available") + + def _summary_embedding_profile(self) -> Any: + from .semantic_projection import SummaryEmbeddingProfile + + return SummaryEmbeddingProfile( + base_url=self.summary_projection_embedding_base_url, + model=self.summary_projection_embedding_model, + dimensions=self.summary_projection_embedding_dimensions, + timeout=self.summary_projection_embedding_timeout, + api_key=self.summary_projection_embedding_api_key, + ) - def _ensure_add_completion_defaults(self) -> None: - if self.summary_projection_indexer is None: - from .semantic_projection import SummaryProjectionIndexer + def _open_summary_projection(self, *, create: bool) -> Any: + if self.summary_projection is None: + from .semantic_projection import SummaryProjection - self.summary_projection_indexer = SummaryProjectionIndexer.from_provider( + self.summary_projection = SummaryProjection( self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - - def _ensure_add_semantic_retrieval_ready(self) -> None: - indexer = self.summary_projection_indexer - if indexer is None: - raise RuntimeError("pifs add requires a summary projection indexer") - from .semantic_projection import SemanticProjectionSearchBackend - - index_dir = Path(getattr(indexer, "index_dir", self.summary_projection_index_dir)) - embedder = getattr(indexer, "embedder", None) - if embedder is None: - self.configure_semantic_projection_retrieval( - index_dir, - embedding_provider=str( - getattr( - indexer, - "embedding_provider", - self.summary_projection_embedding_provider, - ) - ), - embedding_model=str( - getattr(indexer, "embedding_model", self.summary_projection_embedding_model) - ), - embedding_dimensions=int( - getattr( - indexer, - "embedding_dimensions", - self.summary_projection_embedding_dimensions, - ) - ), - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - else: - embedding_cache = getattr(indexer, "embedding_cache", None) - self.semantic_retrieval_backend = SemanticProjectionSearchBackend( - index_dir, - embedder=embedder, - embedding_provider=str( - getattr( - indexer, - "embedding_provider", - self.summary_projection_embedding_provider, - ) - ), - embedding_model=str( - getattr(indexer, "embedding_model", self.summary_projection_embedding_model) - ), - embedding_dimensions=int( - getattr( - indexer, - "embedding_dimensions", - self.summary_projection_embedding_dimensions, - ) - ), - embedding_cache_path=getattr(embedding_cache, "db_path", None), + profile=self._summary_embedding_profile(), + create=create, ) - if "summary" not in self.semantic_retrieval_channels(): - raise RuntimeError("pifs add failed to configure summary semantic retrieval") + return self.summary_projection def _add_created_folder_paths(self, folder_path: str) -> list[str]: paths = self._folder_ancestor_paths(folder_path) @@ -348,75 +376,6 @@ def _folder_ancestor_paths(folder_path: str) -> list[str]: paths.append("/" + "/".join(segments[:index])) return paths - def configure_existing_projection_retrieval(self) -> bool: - """Attach semantic retrieval to already-built projection indexes. - - Register-time generation owns building the index files. Opening an - existing workspace should still expose semantic retrieval when the - configured embedding dimensions match the existing index. - """ - if self.semantic_retrieval_backend is not None: - return bool(self.semantic_retrieval_channels()) - index_config = self._existing_projection_index_config() - if index_config is None: - return False - existing_dimension = int(index_config.get("dimension") or 0) - if existing_dimension != self.summary_projection_embedding_dimensions: - raise RuntimeError( - "summary projection index dimension mismatch: " - f"{index_config.get('db_path') or self.summary_projection_index_dir} " - f"was built with dimension {existing_dimension}, but configured " - "summary_projection_embedding_dimensions is " - f"{self.summary_projection_embedding_dimensions}. Rebuild the " - "projection index or use a matching embedding configuration." - ) - self.configure_semantic_projection_retrieval( - self.summary_projection_index_dir, - embedding_provider=self.summary_projection_embedding_provider, - embedding_model=self.summary_projection_embedding_model, - embedding_dimensions=self.summary_projection_embedding_dimensions, - embedding_timeout=self.summary_projection_embedding_timeout, - embedding_api_key=self.summary_projection_embedding_api_key, - embedding_base_url=self.summary_projection_embedding_base_url, - ) - return bool(self.semantic_retrieval_channels()) - - def _existing_projection_index_config(self) -> dict[str, Any] | None: - for channel in SEMANTIC_RETRIEVAL_CHANNELS: - index_name = SEMANTIC_PROJECTION_INDEX_NAMES.get(channel) - if not index_name: - continue - index_path = self.summary_projection_index_dir / f"{index_name}.sqlite" - if not index_path.exists(): - continue - from .semantic_index import SQLiteVecSemanticIndex - - try: - info = SQLiteVecSemanticIndex(index_path).info() - except Exception: - continue - if int(info.get("document_count") or 0) <= 0: - continue - metadata = dict(info.get("metadata") or {}) - if metadata.get("channel") and metadata.get("channel") != channel: - continue - return info - return None - - def browse( - self, - path: str = "/", - recursive: bool = False, - limit: int = 100, - max_depth: int | None = None, - ) -> dict[str, list[dict[str, Any]]]: - return self.store.list_folder( - path, - recursive=recursive, - limit=limit, - max_depth=max_depth, - ) - def resolve_query_scope(self, path: str) -> PIFSQueryScope: normalized = normalize_path(path) if self._folder_exists(normalized): @@ -444,10 +403,16 @@ def resolve_query_scope(self, path: str) -> PIFSQueryScope: raise KeyError(f"Unknown folder path: {normalized}") raise ValueError( "Metadata axes must come after the physical folder prefix; " - "inspect the physical folder first, then append @field/value buckets. " + "inspect the physical folder first, then append @field/value segments. " "Use the path returned by tree for values containing '/'." ) - field = unquote(segment[1:]) + axis_segment = segment[1:] + if "=" in axis_segment: + raise ValueError( + "Metadata virtual paths use @field/value; run tree /@field " + "and copy the returned path." + ) + field = unquote(axis_segment) self.metadata.validate_field_name(field) if not self.store.metadata_field_exists(field): raise ValueError("Unknown metadata axis; run tree to inspect available @field axes.") @@ -456,19 +421,21 @@ def resolve_query_scope(self, path: str) -> PIFSQueryScope: "A metadata field can appear only once in a scope path; " "choose one value or use browse --where for advanced predicates." ) - if index + 1 >= len(remainder): + value_index = index + 1 + if value_index == len(remainder): return PIFSQueryScope( path=normalized, folder_path=folder_path, metadata_filter=metadata_filter, metadata_axis=field, ) - value_segment = remainder[index + 1] - if value_segment.startswith("@"): + encoded_value = remainder[value_index] + if encoded_value.startswith("@"): raise ValueError( - "Metadata axis paths require @field/value; run tree /@field to inspect values." + "Metadata axis inspection must be the final path segment; " + "choose a value with @field/value before appending another axis." ) - metadata_filter[field] = unquote(value_segment) + metadata_filter[field] = unquote(encoded_value) index += 2 return PIFSQueryScope( @@ -555,15 +522,23 @@ def scope_files(self, scope: PIFSQueryScope, *, limit: int) -> list[dict[str, An files.append( { "path": self._scope_file_locator(scope, locator_leaf), - "name": locator_leaf, + "locator_leaf": locator_leaf, "type": "file", "file_ref": row["file_ref"], - "document_id": row["document_id"], + "external_id": row["external_id"], "title": leaf, + "pageindex_tree_status": row["pageindex_tree_status"], "metadata": row["metadata"], } ) - files = sorted(files, key=lambda item: (str(item["name"]).lower(), item["path"], item["file_ref"])) + files = sorted( + files, + key=lambda item: ( + str(item["title"]).lower(), + item["path"], + item["file_ref"], + ), + ) return files[:limit] def scope_file_locator(self, scope: PIFSQueryScope, file_ref: str, leaf: str) -> str: @@ -575,8 +550,7 @@ def scope_file_locator(self, scope: PIFSQueryScope, file_ref: str, leaf: str) -> def _scope_file_leaf_items(self, scope: PIFSQueryScope, *, limit: int) -> list[tuple[dict[str, Any], str]]: recursive = bool(scope.metadata_filter) - rows = self.store.search_files( - None, + rows = self.store.list_files( scope={"folder_path": scope.folder_path, "recursive": recursive}, metadata_filter=scope.metadata_filter or None, limit=limit, @@ -603,24 +577,23 @@ def _scope_leaf_counts(leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str counts[leaf] = counts.get(leaf, 0) + 1 return counts - @staticmethod - def _disambiguated_scope_leaf(leaf: str, file_ref: str, leaf_counts: dict[str, int]) -> str: - if leaf_counts.get(leaf, 0) <= 1: - return leaf - return f"{leaf}~{file_ref}" - @classmethod def _scope_locator_leaf_by_file_ref(cls, leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str, str]: leaf_counts = cls._scope_leaf_counts(leaf_items) + reserved = {leaf for leaf, count in leaf_counts.items() if count == 1} used: set[str] = set() + next_suffix: dict[str, int] = {} locator_leaf_by_file_ref: dict[str, str] = {} for row, leaf in sorted(leaf_items, key=lambda item: (item[1].lower(), item[1], item[0]["file_ref"])): - base = cls._disambiguated_scope_leaf(leaf, row["file_ref"], leaf_counts) - locator_leaf = base - suffix = 2 - while locator_leaf in used: - locator_leaf = f"{base}~{suffix}" - suffix += 1 + if leaf_counts[leaf] == 1: + locator_leaf = leaf + else: + suffix = next_suffix.get(leaf, 1) + locator_leaf = f"{leaf}~{suffix}" + while locator_leaf in used or locator_leaf in reserved: + suffix += 1 + locator_leaf = f"{leaf}~{suffix}" + next_suffix[leaf] = suffix + 1 used.add(locator_leaf) locator_leaf_by_file_ref[row["file_ref"]] = locator_leaf return locator_leaf_by_file_ref @@ -640,47 +613,36 @@ def scope_stat(self, path: str) -> dict[str, Any]: @staticmethod def encode_scope_segment(segment: Any) -> str: - return quote(str(segment), safe="") + value = str(segment) + if value in {".", ".."}: + return value.replace(".", "%2E") + return quote(value, safe="") def browse_semantic_files( self, path: str, query: str, *, - retrieval_query: str | None = None, recursive: bool = False, - space: str = "summary", page: int = 1, - page_size: int = 10, metadata_filter: Optional[dict[str, Any] | str] = None, ) -> dict[str, Any]: + page_size = 10 path = normalize_path(path) query_scope = self.resolve_query_scope(path) self.store.folder_info(query_scope.folder_path) - query_text = self._query_text(retrieval_query or query).strip() + query_text = self._query_text(query).strip() if not query_text: raise ValueError("browse requires a query") if page < 1: raise ValueError("browse --page must be at least 1") - if page_size < 1: - raise ValueError("browse page_size must be at least 1") - if space not in SEMANTIC_RETRIEVAL_CHANNELS: - raise ValueError( - "Unsupported browse --space: " - f"{space}. Supported spaces: {', '.join(SEMANTIC_RETRIEVAL_CHANNELS)}" - ) - available_spaces = self.semantic_retrieval_channels() - if space not in available_spaces: - available = ", ".join(available_spaces) if available_spaces else "none" - raise ValueError( - f"browse --space {space} is not available; available spaces: {available}" - ) - search_channel = getattr(self.semantic_retrieval_backend, "search_channel", None) - if search_channel is None: - available = ", ".join(available_spaces) if available_spaces else "none" - raise ValueError( - f"browse --space {space} is not available; available spaces: {available}" - ) + if self.summary_projection is None: + index_path = self.summary_projection_index_dir / "summary.sqlite" + if not index_path.exists(): + raise ValueError("browse Summary Projection is not available") + projection = self._open_summary_projection(create=False) + if not projection.available: + raise ValueError("browse Summary Projection is not available") parsed_filter = self.merge_scope_filter(query_scope, metadata_filter) effective_recursive = recursive or bool(query_scope.metadata_filter) scope = {"folder_path": query_scope.folder_path, "recursive": effective_recursive} @@ -690,13 +652,11 @@ def browse_semantic_files( ) offset = (page - 1) * page_size needed = offset + page_size + 1 - semantic_filters = {"file_ref": scope_file_refs} candidates = ( - search_channel( - space, + projection.search( query_text, limit=needed, - filters=semantic_filters, + file_refs=scope_file_refs, ) if scope_file_refs else [] @@ -705,10 +665,7 @@ def browse_semantic_files( rows: list[dict[str, Any]] = [] seen: set[str] = set() for candidate in candidates: - try: - file_ref = self.store.resolve_file_ref(candidate.document_id) - except KeyError: - continue + file_ref = candidate.file_ref if file_ref in seen: continue if file_ref not in scope_file_ref_set: @@ -746,21 +703,16 @@ def browse_semantic_files( rows.append( { "rank": rank, - "similarity": self._semantic_candidate_similarity(candidate), - "score": self._semantic_candidate_score(candidate), + "similarity": candidate.similarity, "path": stable_path, "file_ref": file_ref, - "document_id": entry.external_id, "external_id": entry.external_id, "title": display_title, - "original_title": entry.title, + "pageindex_tree_status": entry.pageindex_tree_status, "folder_path": folder_path, "folder_paths": folder_paths, "summary": str((entry.metadata or {}).get("summary") or ""), - "snippet": str(getattr(candidate, "snippet", "") or entry.descriptor), "metadata": entry.metadata, - "metadata_status": entry.metadata_status, - "sources": list(getattr(candidate, "sources", []) or []), } ) if len(rows) >= needed: @@ -768,12 +720,10 @@ def browse_semantic_files( page_rows = rows[offset : offset + page_size] payload = { "mode": "files", - "retrieval": f"{space}_vector", + "retrieval": "summary", "query": query, "scope": query_scope.path, "recursive": effective_recursive, - "space": space, - "available_spaces": list(available_spaces), "page": page, "page_size": page_size, "has_more": len(rows) > offset + page_size, @@ -786,23 +736,6 @@ def browse_semantic_files( def folder_info(self, path: str = "/") -> dict[str, Any]: return self.store.folder_info(path) - def find_folders( - self, - path: str = "/", - metadata_filter: Optional[dict[str, Any] | str] = None, - limit: int = 100, - max_depth: int | None = None, - include_self: bool = False, - ) -> list[dict[str, Any]]: - parsed_filter = self.metadata.parse_filter(metadata_filter) - return self.store.find_folders( - path, - metadata_filter=parsed_filter, - limit=limit, - max_depth=max_depth, - include_self=include_self, - ) - def create_folder( self, path: str, @@ -854,135 +787,20 @@ def set_metadata( metadata=replacement, metadata_status=dict(info.get("metadata_status") or {}), ) - return self.store.file_info(file_ref) - - def search( - self, - query: Union[str, list[str], None] = None, - scope: Optional[dict[str, Any]] = None, - metadata_filter: Optional[dict[str, Any] | str] = None, - limit: int = 10, - ) -> list[SearchResult]: - parsed_filter = self.metadata.parse_filter(metadata_filter) - rows = self.store.search_files( - query, - scope=scope, - metadata_filter=parsed_filter, - limit=limit, + updated = self.store.file_info(file_ref) + entry = self.store.get_file(file_ref) + folder_paths = [folder["path"] for folder in updated.get("folders", [])] + folder_path = self._preferred_folder_path( + folder_paths, + entry.folder_path, + entry.folder_path, ) - results = [] - scope_path = self._scope_folder_path(scope) - for row in rows: - folder_paths = [ - folder["path"] - for folder in self.store.folder_memberships(row["file_ref"]) - ] - folder_path = self._preferred_folder_path(folder_paths, scope_path, row["folder_path"]) - display_title = self.store.membership_display_name(row["file_ref"], folder_path) or row["title"] - results.append( - SearchResult( - file_ref=row["file_ref"], - external_id=row["external_id"], - title=display_title, - snippet=row["snippet"], - folder_path=folder_path, - folder_paths=folder_paths, - metadata=row["metadata"], - metadata_status=row["metadata_status"], - id=row["id"], - document_id=row["document_id"], - name=display_title, - description=row["description"], - status=row["status"], - pageNum=row["pageNum"], - createdAt=row["createdAt"], - folderId=row["folderId"], - ) - ) - return results - - def configure_semantic_projection_retrieval( - self, - index_dir: Union[str, Path], - *, - embedding_provider: str = "openai", - embedding_model: str = "text-embedding-3-small", - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_timeout: float = 60, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - fetch_multiplier: int = 100, - ) -> Any: - from .semantic_projection import SemanticProjectionSearchBackend - - self.semantic_retrieval_backend = SemanticProjectionSearchBackend.from_provider( - index_dir, - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - embedding_timeout=embedding_timeout, - embedding_api_key=embedding_api_key, - embedding_base_url=embedding_base_url, - fetch_multiplier=fetch_multiplier, + updated["path"] = self._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, ) - return self.semantic_retrieval_backend - - @property - def has_semantic_retrieval_backend(self) -> bool: - return self.semantic_retrieval_backend is not None - - def semantic_retrieval_channels(self) -> tuple[str, ...]: - backend = self.semantic_retrieval_backend - if backend is None: - return () - available_channels = getattr(backend, "available_channels", None) - if callable(available_channels): - raw_channels = available_channels() - else: - raw_channels = getattr(backend, "semantic_tool_channels", ()) - available = set(raw_channels or ()) - return tuple(channel for channel in SEMANTIC_RETRIEVAL_CHANNELS if channel in available) - - def has_semantic_channel(self, channel: str) -> bool: - return channel in self.semantic_retrieval_channels() - - def retrieval_capabilities(self) -> dict[str, Any]: - semantic_channels = ["summary"] if self.has_semantic_channel("summary") else [] - semantic_commands = ["browse"] if semantic_channels else [] - return { - "lexical": { - "grep_recursive": False, - "grep_recursive_semantic_prefilter": False, - "find_maxdepth": False, - }, - "semantic": { - "backend_configured": self.semantic_retrieval_backend is not None, - "channels": semantic_channels, - "commands": semantic_commands, - }, - } - - def open(self, target: str, location: str = "all") -> OpenResult: - file_ref = self._resolve_target(target) - entry = self.store.get_file(file_ref) - if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: - raise ValueError( - "open() text artifact reads are not supported for PDF/Markdown PageIndex files; " - "use pageindex_structure() or pageindex_pages()." - ) - if str(location).strip().lower() in {"all", "full", "*"}: - return self._open_all(file_ref) - start, end = self._parse_line_range(location) - return self._open_lines(file_ref, start, end) - - def cat_text_artifact(self, target: str, location: str = "all") -> OpenResult: - file_ref = self._resolve_target(target) - entry = self.store.get_file(file_ref) - self._require_text_artifact_file(entry, "cat --all") - if str(location).strip().lower() in {"all", "full", "*"}: - return self._open_all(file_ref) - start, end = self._parse_line_range(location) - return self._open_lines(file_ref, start, end) + return updated def pageindex_structure( self, @@ -1061,20 +879,6 @@ def pageindex_pages(self, target: str, pages: str) -> dict[str, Any]: "text": text, } - def _stat(self, target: str) -> dict[str, Any]: - file_ref = self._resolve_target(target) - return self.store.file_info(file_ref) - - def _require_text_artifact_file(self, entry: Any, command: str) -> None: - if self._file_format(entry) == "text": - return - raise ValueError( - f"{command} is only supported for txt/text files; " - f"got title={entry.title!r}, content_type={entry.content_type!r}. " - "Use cat --structure, " - "or cat --page for PDF/Markdown PageIndex files." - ) - def _require_pageindex_document_file(self, entry: Any, command: str) -> None: if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: return @@ -1117,7 +921,12 @@ def pageindex_client_workspace(self) -> Path: def _pageindex_client(self) -> PageIndexClient: from ..client import PageIndexClient - return PageIndexClient(workspace=str(self.pageindex_client_workspace)) + workspace = self.pageindex_client_workspace + workspace.mkdir(parents=True, exist_ok=True) + metadata_index = workspace / "_meta.json" + if not metadata_index.exists(): + metadata_index.write_text("{}\n", encoding="utf-8") + return PageIndexClient(workspace=str(workspace)) def _pageindex_client_doc_for_entry(self, entry: Any) -> tuple[PageIndexClient, str | None]: client = self._pageindex_client() @@ -1224,9 +1033,6 @@ def _client_json(payload: str) -> Any: except json.JSONDecodeError: return {"error": f"Invalid PageIndexClient JSON response: {payload}"} - def _create_folder(self, path: str) -> str: - return self.create_folder(path) - @classmethod def _resolve_add_target( cls, @@ -1288,22 +1094,31 @@ def _require_add_pageindex_ready(self, record: dict[str, Any]) -> None: ) raise RuntimeError(f"pifs add failed to build PageIndex tree: {message}") - def _require_add_summary_projection_ready(self, record: dict[str, Any]) -> None: + def _require_summary_projection_ready( + self, + record: dict[str, Any], + *, + operation: str, + ) -> None: summary_projection = (record.get("metadata_status") or {}).get("summary_projection") if not summary_projection or not summary_projection.get("requested"): - raise RuntimeError("pifs add requires a requested summary projection index") + raise RuntimeError( + f"PIFS {operation} requires a requested summary projection index" + ) if summary_projection.get("status") != "ready": detail = summary_projection.get("error") or summary_projection.get("status") raise RuntimeError( - f"pifs add failed to build summary projection index: {detail}" + f"PIFS {operation} failed to build summary projection index: {detail}" ) - def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: + def _prepare_file_record( + self, + file: dict[str, Any], + *, + artifact_baselines: dict[Path, bytes | None] | None = None, + ) -> dict[str, Any]: storage_uri = file["storage_uri"] - metadata = file.get("metadata") or {} - if not isinstance(metadata, dict): - raise ValueError("metadata must be a JSON object") - self._validate_register_metadata(metadata) + metadata = self._validated_register_metadata(file.get("metadata")) external_id = file.get("external_id") content = file.get("content") or "" folder_path = normalize_path(file.get("folder_path") or "/") @@ -1325,6 +1140,12 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: file_ref = make_file_ref( str(external_id or self._join_virtual_file_path(folder_path, title).strip("/")) ) + if artifact_baselines is not None: + self._capture_registration_artifact_baselines( + file_ref, + file, + artifact_baselines, + ) ( pageindex_doc_id, pageindex_tree_status, @@ -1350,12 +1171,10 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: pageindex_tree_status=pageindex_tree_status, fallback_content=content, ) - fts_content = file.get("fts_content", artifact_content) source_type = file.get("source_type") metadata_status = self._metadata_status_state(metadata=metadata) self._attach_pageindex_tree_failure(metadata_status, pageindex_tree_failure) indexed_metadata = SQLiteFileSystemStore.indexed_metadata_values(metadata) - searchable_metadata = indexed_metadata text_artifact_path = file.get("text_artifact_path") owns_text_artifact = text_artifact_path is None if text_artifact_path is None: @@ -1365,13 +1184,12 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: if raw_artifact_path is None and file.get("write_raw_artifact", True): raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" owns_raw_artifact = True - descriptor = self._build_descriptor(title, metadata) return { "file_ref": file_ref, "external_id": external_id, "storage_uri": storage_uri, "title": title, - "descriptor": descriptor, + "descriptor": title, "content_type": content_type, "source_type": source_type, "fingerprint": fingerprint(artifact_content), @@ -1384,10 +1202,7 @@ def _prepare_file_record(self, file: dict[str, Any]) -> dict[str, Any]: "metadata_status": metadata_status, "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), "indexed_metadata": indexed_metadata, - "metadata_text": metadata_text(searchable_metadata), "folder_path": folder_path, - "content": fts_content, - "skip_fts": bool(file.get("skip_fts", False)), "_pifs_owned_text_artifact": owns_text_artifact, "_pifs_owned_raw_artifact": owns_raw_artifact, } @@ -1405,7 +1220,7 @@ def _registration_text_artifact_content( return fallback_content if pageindex_tree_status != "built" or not pageindex_doc_id: return fallback_content - return self._pageindex_extracted_text(pageindex_doc_id) + return self._pageindex_extracted_text(pageindex_doc_id) or fallback_content def _pageindex_extracted_text(self, doc_id: str) -> str: client = self._pageindex_client() @@ -1451,12 +1266,10 @@ def _raw_artifact_payload( def _sync_owned_raw_artifact(self, record: dict[str, Any]) -> None: raw_artifact_path = record.get("raw_artifact_path") - if not raw_artifact_path: - return - default_raw_artifact_path = self.store.raw_dir / f"{record['file_ref']}.json" - if Path(raw_artifact_path).expanduser().resolve(strict=False) != ( - default_raw_artifact_path.resolve(strict=False) - ): + if self._managed_raw_artifact_path( + str(record["file_ref"]), + raw_artifact_path, + ) is None: return record["raw_artifact_path"] = str( self.store.write_raw_artifact( @@ -1470,7 +1283,6 @@ def _sync_owned_raw_artifact(self, record: dict[str, Any]) -> None: ) def _record_from_file_entry(self, entry: Any) -> dict[str, Any]: - content = self.store.read_text(entry.file_ref) metadata_status = self._metadata_status_state(metadata=entry.metadata) self._attach_pageindex_tree_failure( metadata_status, @@ -1494,12 +1306,7 @@ def _record_from_file_entry(self, entry: Any) -> dict[str, Any]: "metadata_status": metadata_status, "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), "indexed_metadata": SQLiteFileSystemStore.indexed_metadata_values(entry.metadata), - "metadata_text": metadata_text( - SQLiteFileSystemStore.indexed_metadata_values(entry.metadata) - ), "folder_path": entry.folder_path, - "content": content, - "skip_fts": False, } def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: @@ -1510,20 +1317,19 @@ def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: summary = str(record.get("metadata", {}).get("summary") or "").strip() if not summary: return False - if self.summary_projection_indexer is None: - self._refresh_record_metadata_status(record) - return True + if self.summary_projection is None: + raise RuntimeError("PIFS Summary Projection is not open") try: - result = self.summary_projection_indexer.upsert_summary(record) + result = self.summary_projection.upsert_summary(record) except Exception as exc: summary_index["status"] = "failed" summary_index["error"] = str(exc) self._refresh_record_metadata_status(record) - return True + raise RuntimeError( + f"PIFS failed to build summary projection index: {exc}" + ) from exc summary_index.clear() summary_index.update({"requested": True, **result}) - if summary_index.get("status") != "ready": - summary_index["status"] = "ready" self._refresh_record_metadata_status(record) return True @@ -1541,52 +1347,258 @@ def _cleanup_failed_register_artifacts(self, records: list[dict[str, Any]]) -> N if record.get("_pifs_owned_raw_artifact") and record.get("raw_artifact_path"): self._unlink_artifact(record["raw_artifact_path"]) - def _cleanup_add_catalog_record(self, file_ref: str) -> None: + def _capture_registration_artifact_baselines( + self, + file_ref: str, + file: dict[str, Any], + baselines: dict[Path, bytes | None], + ) -> None: + paths = [] + if file.get("text_artifact_path") is None: + paths.append(self.store.text_dir / f"{file_ref}.txt") + raw_artifact_path = file.get("raw_artifact_path") + if raw_artifact_path is None and file.get("write_raw_artifact", True): + raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" + managed_raw_path = self._managed_raw_artifact_path(file_ref, raw_artifact_path) + if managed_raw_path is not None: + paths.append(managed_raw_path) + try: + existing = self.store.get_file(file_ref) + except KeyError: + existing = None + if existing is not None and existing.pageindex_doc_id: + paths.extend( + [ + self.pageindex_client_workspace / "_meta.json", + self.pageindex_client_workspace + / f"{existing.pageindex_doc_id}.json", + ] + ) + for path in paths: + if path not in baselines: + baselines[path] = path.read_bytes() if path.is_file() else None + + def _managed_raw_artifact_path( + self, + file_ref: str, + raw_artifact_path: Any, + ) -> Path | None: + if not raw_artifact_path: + return None + default_path = self.store.raw_dir / f"{file_ref}.json" + if Path(raw_artifact_path).expanduser().resolve(strict=False) != ( + default_path.resolve(strict=False) + ): + return None + return default_path + + def _restore_registration_artifact_baselines( + self, + baselines: dict[Path, bytes | None], + ) -> None: + for path, content in baselines.items(): + if content is None: + self._unlink_artifact(path) + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + def _cleanup_catalog_record(self, file_ref: str) -> None: try: self.store.delete_file(file_ref) except Exception: return - def _existing_file_refs(self, records: list[dict[str, Any]]) -> set[str]: - existing: set[str] = set() - for record in records: - file_ref = str(record.get("file_ref") or "") - if not file_ref: - continue - try: - self.store.get_file(file_ref) - except KeyError: - continue - existing.add(file_ref) - return existing + def _capture_existing_registration_rows( + self, + snapshot: _RegistrationRollbackSnapshot, + projection: Any | None, + ) -> None: + file_refs = sorted( + { + str(record.get("file_ref") or "") + for record in snapshot.records + if str(record.get("file_ref") or "") + } + ) + with self.store.connect() as connection: + for file_ref in file_refs: + file_row = connection.execute( + "SELECT * FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + if file_row is None: + continue + snapshot.catalog_rows[file_ref] = dict(file_row) + snapshot.membership_rows[file_ref] = [ + dict(row) + for row in connection.execute( + "SELECT * FROM file_folders WHERE file_ref = ? ORDER BY folder_id", + (file_ref,), + ) + ] + snapshot.metadata_value_rows[file_ref] = [ + dict(row) + for row in connection.execute( + "SELECT * FROM metadata_values WHERE file_ref = ? " + "ORDER BY field_id, value_text, created_at", + (file_ref,), + ) + ] + if not snapshot.catalog_rows: + return + if projection is None: + raise RuntimeError( + "PIFS registration cannot snapshot existing files without a Summary Projection" + ) + with projection.index.connect(read_only=True) as connection: + for file_ref in sorted(snapshot.catalog_rows): + doc = connection.execute( + "SELECT * FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchone() + vector = ( + None + if doc is None + else connection.execute( + "SELECT rowid, source_type, embedding " + "FROM semantic_index_vec WHERE rowid = ?", + (doc["rowid"],), + ).fetchone() + ) + if doc is None or vector is None: + raise RuntimeError( + "PIFS registration found an incomplete existing Summary Projection; " + "migrate this workspace before retrying." + ) + vector_row = dict(vector) + vector_row["embedding"] = bytes(vector_row["embedding"]) + snapshot.projection_rows[file_ref] = (dict(doc), vector_row) - def _cleanup_add_summary_projection(self, records: list[dict[str, Any]]) -> None: - indexer = self.summary_projection_indexer - if indexer is None: + def _restore_existing_registration_catalog( + self, + snapshot: _RegistrationRollbackSnapshot, + ) -> None: + if not snapshot.catalog_rows: + return + with self.store.connect() as connection: + for file_ref in sorted(snapshot.catalog_rows): + connection.execute( + "DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,) + ) + connection.execute( + "DELETE FROM file_folders WHERE file_ref = ?", (file_ref,) + ) + connection.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) + self._insert_registration_snapshot_row( + connection, + "files", + snapshot.catalog_rows[file_ref], + ) + for row in snapshot.membership_rows[file_ref]: + self._insert_registration_snapshot_row( + connection, + "file_folders", + row, + ) + for row in snapshot.metadata_value_rows[file_ref]: + self._insert_registration_snapshot_row( + connection, + "metadata_values", + row, + ) + + def _restore_existing_registration_projection( + self, + snapshot: _RegistrationRollbackSnapshot, + ) -> None: + if not snapshot.projection_rows: + return + if self.summary_projection is None: + raise RuntimeError( + "PIFS registration cannot restore an unopened Summary Projection" + ) + with self.summary_projection.index.connect() as connection: + for file_ref in sorted(snapshot.projection_rows): + current = connection.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchall() + for row in current: + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", + (row["rowid"],), + ) + connection.execute( + "DELETE FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ) + doc, vector = snapshot.projection_rows[file_ref] + self._insert_registration_snapshot_row( + connection, + "semantic_index_docs", + doc, + ) + self._insert_registration_snapshot_row( + connection, + "semantic_index_vec", + vector, + ) + + @staticmethod + def _insert_registration_snapshot_row( + connection: Any, + table: str, + row: dict[str, Any], + ) -> None: + columns = list(row) + placeholders = ", ".join("?" for _ in columns) + connection.execute( + f"INSERT INTO {table}({', '.join(columns)}) VALUES ({placeholders})", + [row[column] for column in columns], + ) + + def _cleanup_summary_projection_records( + self, + records: list[dict[str, Any]], + ) -> None: + projection = self.summary_projection + if projection is None: return - delete_summary = getattr(indexer, "delete_summary", None) for record in records: file_ref = str(record.get("file_ref") or "") if not file_ref: continue try: - if callable(delete_summary): - delete_summary(file_ref) - continue - index = getattr(indexer, "index", None) - delete_file_refs = getattr(index, "delete_file_refs", None) - if callable(delete_file_refs): - delete_file_refs([file_ref]) + projection.delete_summary(file_ref) except Exception: continue - def _cleanup_add_created_folders(self, folder_paths: list[str]) -> None: + def _cleanup_summary_projection_cache( + self, + keys: set[_EmbeddingCacheKey], + ) -> None: + if not keys or self.summary_projection is None: + return + try: + self.summary_projection.delete_cache_keys(keys) + except Exception: + return + + def _cleanup_created_folders(self, folder_paths: list[str]) -> None: for folder_path in reversed(folder_paths): try: self.store.delete_empty_folder(folder_path) except Exception: continue + def _cleanup_new_metadata_fields(self, names: set[str]) -> None: + for name in sorted(names): + try: + self.store.delete_metadata_field_if_unreferenced(name) + except Exception: + continue + def _pageindex_cache_doc_ids(self) -> set[str]: workspace = self.pageindex_client_workspace doc_ids = {path.stem for path in workspace.glob("*.json") if path.name != "_meta.json"} @@ -1601,7 +1613,7 @@ def _pageindex_cache_doc_ids(self) -> set[str]: doc_ids.update(str(doc_id) for doc_id in payload) return doc_ids - def _cleanup_add_pageindex_cache( + def _cleanup_pageindex_cache( self, records: list[dict[str, Any]], preexisting_doc_ids: set[str], @@ -1658,35 +1670,6 @@ def _refresh_record_metadata_status( record["metadata_json"] = json.dumps(record["metadata"], ensure_ascii=False) record["metadata_status_json"] = json.dumps(metadata_status, ensure_ascii=False) record["indexed_metadata"] = SQLiteFileSystemStore.indexed_metadata_values(record["metadata"]) - record["metadata_text"] = metadata_text(record["indexed_metadata"]) - - def _open_lines(self, file_ref: str, start: int, end: int) -> OpenResult: - entry = self.store.get_file(file_ref) - lines = self.store.read_text(file_ref).splitlines() - start = max(1, start) - end = min(max(start, end), len(lines)) - text = "\n".join(lines[start - 1:end]) - return OpenResult( - file_ref=file_ref, - start_line=start, - end_line=end, - text=text, - external_id=entry.external_id, - folder_path=entry.folder_path, - ) - - def _open_all(self, file_ref: str) -> OpenResult: - entry = self.store.get_file(file_ref) - text = self.store.read_text(file_ref) - line_count = len(text.splitlines()) - return OpenResult( - file_ref=file_ref, - start_line=1, - end_line=line_count, - text=text, - external_id=entry.external_id, - folder_path=entry.folder_path, - ) @classmethod def _structural_unavailable( @@ -1769,7 +1752,7 @@ def _resolve_scope_file_locator(self, target: str) -> str: matches = [ item for item in self.scope_files(scope, limit=self.scope_file_count(scope) + 1) - if item["name"] == leaf + if item["locator_leaf"] == leaf ] if not matches: raise KeyError(f"Unknown file target: {target}") @@ -1787,31 +1770,6 @@ def _scope_file_required_message(path: str) -> str: f'browse {path} "" to select a file leaf.' ) - @staticmethod - def _semantic_candidate_score(candidate: Any) -> float | None: - try: - return float(getattr(candidate, "score")) - except (AttributeError, TypeError, ValueError): - return None - - @classmethod - def _semantic_candidate_similarity(cls, candidate: Any) -> float: - distances: list[float] = [] - for source in getattr(candidate, "sources", []) or []: - if not isinstance(source, dict) or source.get("distance") is None: - continue - try: - distances.append(float(source["distance"])) - except (TypeError, ValueError): - continue - if distances: - distance = max(min(distances), 0.0) - return round(max(0.0, min(1.0, 1.0 / (1.0 + distance))), 4) - score = cls._semantic_candidate_score(candidate) - if score is None: - return 0.0 - return round(max(0.0, min(1.0, score)), 4) - @staticmethod def _metadata_filter_payload(metadata_filter: Any) -> str: if isinstance(metadata_filter, str): @@ -1838,19 +1796,11 @@ def _stable_file_locator( ).strip() if not title: raise RuntimeError(f"browse cannot build a virtual path for {file_ref}: missing title") - target = self._join_virtual_file_path(folder_path, title.strip("/")) - try: - resolved_file_ref = self.store.resolve_file_ref(target) - except KeyError as exc: - raise RuntimeError( - f"browse produced an unresolved virtual path for {file_ref}: {target}" - ) from exc - if resolved_file_ref != file_ref: - raise RuntimeError( - "browse produced a non-idempotent virtual path: " - f"{target} resolved to {resolved_file_ref}, expected {file_ref}" - ) - return target + return self.scope_file_locator( + self.resolve_query_scope(folder_path), + file_ref, + title, + ) @staticmethod def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: @@ -1860,25 +1810,39 @@ def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: ) @staticmethod - def _build_descriptor(title: str, metadata: dict[str, Any]) -> str: - source = metadata.get("source_type") or metadata.get("repo") or metadata.get("channel") - return f"{title} ({source})" if source else title - - @staticmethod - def _validate_register_metadata(metadata: dict[str, Any]) -> None: - if "summary" in metadata: + def _validated_register_metadata(metadata: Any) -> dict[str, Any]: + if metadata is None: + validated = {} + elif not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + else: + validated = dict(metadata) + try: + json.dumps(validated, ensure_ascii=False) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON serializable") from exc + if "summary" in validated: raise ValueError("summary is managed by PageIndex doc_description") + return validated def _register_custom_metadata_fields(self, records: list[dict[str, Any]]) -> None: - fields = {} + fields = { + name: {} + for name in self._custom_metadata_field_names(records) + if not self.store.metadata_field_exists(name) + } + if fields: + self.metadata.register_schema({"fields": fields}, source="user") + + def _custom_metadata_field_names(self, records: list[dict[str, Any]]) -> set[str]: + fields = set() for record in records: for name in SQLiteFileSystemStore.indexed_metadata_values( record.get("metadata", {}) ): if self.metadata.FIELD_RE.match(str(name)): - fields[str(name)] = {} - if fields: - self.metadata.register_schema({"fields": fields}, source="user") + fields.add(str(name)) + return fields @staticmethod def _metadata_status_state(*, metadata: dict[str, Any]) -> dict[str, Any]: @@ -1907,13 +1871,6 @@ def _refresh_summary_projection_status( if summary_index.get("status", "not_indexed") == "not_indexed": summary_index["status"] = "pending_index" - @staticmethod - def _scope_folder_path(scope: Optional[dict[str, Any]]) -> Optional[str]: - if not scope: - return None - path = scope.get("folder_path") or scope.get("path") - return normalize_path(path) if path else None - def _folder_exists(self, path: str) -> bool: try: self.store.folder_info(path) @@ -1947,15 +1904,3 @@ def _preferred_folder_path( if non_root: return sorted(non_root, key=lambda item: (len(item), item))[0] return fallback - - @staticmethod - def _parse_line_range(location: str) -> tuple[int, int]: - value = str(location).strip() - if "-" in value: - left, right = value.split("-", 1) - start, end = int(left), int(right) - else: - start = end = int(value) - if start < 1 or end < start: - raise ValueError(f"Invalid line range: {location}") - return start, end diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py index cc01d820a..420793e4b 100644 --- a/pageindex/filesystem/semantic_index.py +++ b/pageindex/filesystem/semantic_index.py @@ -2,18 +2,42 @@ import hashlib import json +import re import sqlite3 from dataclasses import dataclass from pathlib import Path -from typing import Any, Protocol +from typing import Any import sqlite_vec +from ._embedding_identity import normalize_base_url, normalize_model +from ._sqlite_schema import sqlite_schema_signature + class SemanticIndexError(RuntimeError): pass +SCHEMA_VERSION = 2 +SUMMARY_TABLES = { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", +} +SUMMARY_CONFIG_KEYS = {"adapter", "adapter_version", "dimension", "metadata"} +VEC0_DECLARATION_RE = re.compile( + r""" + ^\s*CREATE\s+VIRTUAL\s+TABLE\s+[\"`\[]?semantic_index_vec[\"`\]]? + \s+USING\s+vec0\s*\(\s* + source_type\s+TEXT\s+PARTITION\s+KEY\s*,\s* + embedding\s+FLOAT\s*\[\s*(\d+)\s*\]\s* + (?:distance_metric\s*=\s*([A-Za-z0-9_-]+)\s*)? + \)\s*$ + """, + re.IGNORECASE | re.VERBOSE, +) + + @dataclass(frozen=True) class SemanticIndexRecord: file_ref: str @@ -36,27 +60,6 @@ class SemanticSearchResult: metadata: dict[str, Any] -class RebuildableSemanticIndex(Protocol): - def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: - ... - - def upsert_many(self, records: list[SemanticIndexRecord]) -> int: - ... - - def search( - self, - vector: list[float], - *, - limit: int = 10, - filters: dict[str, Any] | None = None, - fetch_multiplier: int = 20, - ) -> list[SemanticSearchResult]: - ... - - def info(self) -> dict[str, Any]: - ... - - class SQLiteVecSemanticIndex: """Rebuildable local semantic index backed by sqlite-vec. @@ -66,56 +69,77 @@ class SQLiteVecSemanticIndex: def __init__(self, db_path: str | Path): self.db_path = Path(db_path).expanduser() - self.db_path.parent.mkdir(parents=True, exist_ok=True) def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: if dimension <= 0: raise SemanticIndexError("semantic index dimension must be positive") + self.db_path.parent.mkdir(parents=True, exist_ok=True) with self.connect() as conn: - conn.executescript( - """ - DROP TABLE IF EXISTS semantic_index_vec; - DROP TABLE IF EXISTS semantic_index_docs; - DROP TABLE IF EXISTS semantic_index_config; - CREATE TABLE semantic_index_config ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - CREATE TABLE semantic_index_docs ( - rowid INTEGER PRIMARY KEY, - file_ref TEXT NOT NULL UNIQUE, - external_id TEXT, - source_type TEXT NOT NULL DEFAULT '', - title TEXT NOT NULL DEFAULT '', - text_hash TEXT NOT NULL, - text_chars INTEGER NOT NULL DEFAULT 0, - metadata_json TEXT NOT NULL DEFAULT '{}', - created_at TEXT DEFAULT CURRENT_TIMESTAMP, - updated_at TEXT DEFAULT CURRENT_TIMESTAMP - ); - CREATE INDEX idx_semantic_index_docs_file_ref - ON semantic_index_docs(file_ref); - CREATE INDEX idx_semantic_index_docs_external_id - ON semantic_index_docs(external_id); - CREATE INDEX idx_semantic_index_docs_source_type - ON semantic_index_docs(source_type); - """ - ) - conn.execute( - "CREATE VIRTUAL TABLE semantic_index_vec USING " - f"vec0(source_type TEXT partition key, embedding float[{dimension}])" + self._create_schema(conn, dimension=dimension, metadata=metadata or {}) + + def validate(self, expected_identity: dict[str, Any]) -> dict[str, Any]: + info = self.validate_schema() + if info["metadata"] != expected_identity: + raise SemanticIndexError( + "Incompatible PIFS Summary Embedding Profile; migrate the workspace or " + "use the base URL, model, and dimensions that created this projection." ) - config = { - "dimension": str(dimension), - "adapter": "sqlite-vec", - "adapter_version": sqlite_vec.__version__, - "metadata": json.dumps(metadata or {}, ensure_ascii=False, sort_keys=True), - } - conn.executemany( - "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", - sorted(config.items()), + if int(info["dimension"]) != int(expected_identity["dimensions"]): + raise SemanticIndexError( + "Incompatible PIFS Summary Projection dimensions; migrate the workspace " + "or use the matching embedding profile." ) - conn.commit() + return info + + def validate_schema(self) -> dict[str, Any]: + try: + with self.connect(read_only=True) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + tables = self._logical_tables(conn) + config = self._config(conn) + info = self._info_from_connection(conn, config) + actual = self._schema_signature(conn, tables) + dimension = int(info["dimension"]) + self._validate_projection_rows(conn, dimension=dimension) + metadata = info["metadata"] + if ( + version != SCHEMA_VERSION + or tables != SUMMARY_TABLES + or set(config) != SUMMARY_CONFIG_KEYS + or config["adapter"] != "sqlite-vec" + or not config["adapter_version"] + or dimension <= 0 + or set(metadata) != {"base_url", "model", "dimensions"} + or not isinstance(metadata["base_url"], str) + or not metadata["base_url"].strip() + or normalize_base_url(metadata["base_url"]) != metadata["base_url"] + or not isinstance(metadata["model"], str) + or not metadata["model"].strip() + or normalize_model(metadata["model"]) != metadata["model"] + or isinstance(metadata["dimensions"], bool) + or int(metadata["dimensions"]) != dimension + or actual["vec0"] is None + or actual["vec0"]["dimension"] != dimension + or actual["vec0"]["distance_metric"] != "l2" + ): + raise self._incompatible_schema_error() + with self._memory_connection() as expected_conn: + self._create_schema( + expected_conn, + dimension=dimension, + metadata=metadata, + ) + expected = self._schema_signature( + expected_conn, + self._logical_tables(expected_conn), + ) + if actual != expected: + raise self._incompatible_schema_error() + except SemanticIndexError: + raise + except (json.JSONDecodeError, sqlite3.Error, TypeError, ValueError) as exc: + raise self._incompatible_schema_error() from exc + return info def upsert_many(self, records: list[SemanticIndexRecord]) -> int: if not records: @@ -298,27 +322,8 @@ def _search_fetch_k( return min(4096, max(limit, limit * max(fetch_multiplier, 1))) def info(self) -> dict[str, Any]: - with self.connect() as conn: - config = { - row["key"]: row["value"] - for row in conn.execute( - "SELECT key, value FROM semantic_index_config ORDER BY key" - ).fetchall() - } - count = conn.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] - parsed_metadata: dict[str, Any] - try: - parsed_metadata = json.loads(config.get("metadata", "{}")) - except json.JSONDecodeError: - parsed_metadata = {} - return { - "db_path": str(self.db_path), - "adapter": config.get("adapter", "sqlite-vec"), - "adapter_version": config.get("adapter_version", ""), - "dimension": int(config.get("dimension", "0") or 0), - "document_count": count, - "metadata": parsed_metadata, - } + with self.connect(read_only=True) as conn: + return self._info_from_connection(conn, self._config(conn)) def dimension(self) -> int: with self.connect() as conn: @@ -331,14 +336,175 @@ def dimension(self) -> int: ) return int(row["value"]) - def connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) + def connect(self, *, read_only: bool = False) -> sqlite3.Connection: + if read_only: + conn = sqlite3.connect( + f"{self.db_path.resolve().as_uri()}?mode=ro", + uri=True, + ) + else: + conn = sqlite3.connect(self.db_path) conn.row_factory = sqlite3.Row + self._load_extension(conn) + return conn + + @staticmethod + def _load_extension(conn: sqlite3.Connection) -> None: conn.enable_load_extension(True) sqlite_vec.load(conn) conn.enable_load_extension(False) + + @classmethod + def _memory_connection(cls) -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + cls._load_extension(conn) return conn + @staticmethod + def _create_schema( + conn: sqlite3.Connection, + *, + dimension: int, + metadata: dict[str, Any], + ) -> None: + conn.executescript( + """ + DROP TABLE IF EXISTS semantic_index_vec; + DROP TABLE IF EXISTS semantic_index_docs; + DROP TABLE IF EXISTS semantic_index_config; + CREATE TABLE semantic_index_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE semantic_index_docs ( + rowid INTEGER PRIMARY KEY, + file_ref TEXT NOT NULL UNIQUE, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + conn.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + f"vec0(source_type TEXT partition key, embedding float[{dimension}])" + ) + config = { + "dimension": str(dimension), + "adapter": "sqlite-vec", + "adapter_version": sqlite_vec.__version__, + "metadata": json.dumps(metadata, ensure_ascii=False, sort_keys=True), + } + conn.executemany( + "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", + sorted(config.items()), + ) + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + conn.commit() + + @staticmethod + def _logical_tables(conn: sqlite3.Connection) -> set[str]: + return { + str(row[1]) + for row in conn.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} + and not str(row[1]).startswith("sqlite_") + and not str(row[1]).startswith("semantic_index_vec_") + } + + @staticmethod + def _config(conn: sqlite3.Connection) -> dict[str, str]: + return { + str(row["key"]): str(row["value"]) + for row in conn.execute( + "SELECT key, value FROM semantic_index_config ORDER BY key" + ) + } + + def _info_from_connection( + self, + conn: sqlite3.Connection, + config: dict[str, str], + ) -> dict[str, Any]: + metadata = json.loads(config.get("metadata", "{}")) + if not isinstance(metadata, dict): + raise self._incompatible_schema_error() + return { + "db_path": str(self.db_path), + "adapter": config.get("adapter", ""), + "adapter_version": config.get("adapter_version", ""), + "dimension": int(config.get("dimension", "0") or 0), + "document_count": int( + conn.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] + ), + "metadata": metadata, + } + + @staticmethod + def _schema_signature( + conn: sqlite3.Connection, + tables: set[str], + ) -> dict[str, Any]: + signature = sqlite_schema_signature(conn, tables) + vec_row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' " + "AND name = 'semantic_index_vec'" + ).fetchone() + match = VEC0_DECLARATION_RE.fullmatch(str(vec_row[0] if vec_row else "")) + signature["vec0"] = ( + { + "partition_key": ("source_type", "text"), + "embedding_type": "float", + "dimension": int(match.group(1)), + "distance_metric": str(match.group(2) or "l2").lower(), + } + if match + else None + ) + return signature + + @classmethod + def _validate_projection_rows( + cls, + conn: sqlite3.Connection, + *, + dimension: int, + ) -> None: + docs = { + int(row["rowid"]): str(row["source_type"]) + for row in conn.execute( + "SELECT rowid, source_type FROM semantic_index_docs" + ) + } + vectors: dict[int, str] = {} + expected_bytes = dimension * 4 + for row in conn.execute( + "SELECT rowid, source_type, embedding FROM semantic_index_vec" + ): + rowid = int(row["rowid"]) + if len(bytes(row["embedding"])) != expected_bytes: + raise cls._incompatible_schema_error() + vectors[rowid] = str(row["source_type"]) + if docs != vectors: + raise cls._incompatible_schema_error() + + @staticmethod + def _incompatible_schema_error() -> SemanticIndexError: + return SemanticIndexError( + "Incompatible PIFS Summary Projection schema; migrate this workspace " + "with pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + @staticmethod def text_hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py index 5eea7f5c9..4347469f6 100644 --- a/pageindex/filesystem/semantic_projection.py +++ b/pageindex/filesystem/semantic_projection.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import sqlite3 import struct import time @@ -8,224 +7,115 @@ from pathlib import Path from typing import Any +from ._embedding_identity import ( + DEFAULT_OPENAI_BASE_URL, + normalize_base_url, + normalize_model, +) +from ._projection_topology import projection_database_pair, projection_database_paths +from ._sqlite_schema import ( + normalized_table_sql, + regular_table_names, + sqlite_schema_signature, +) from .core import DEFAULT_EMBEDDING_DIMENSIONS from .semantic_index import ( + SCHEMA_VERSION, SQLiteVecSemanticIndex, - SemanticIndexError, SemanticIndexRecord, - SemanticSearchResult, ) -SUMMARY_CHANNEL = "summary" SUMMARY_INDEX_NAME = "summary" +_EmbeddingCacheKey = tuple[str, str, int, str] + + +@dataclass(frozen=True) +class SummaryEmbeddingProfile: + base_url: str | None = DEFAULT_OPENAI_BASE_URL + model: str = "text-embedding-3-small" + dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS + timeout: float = 60 + api_key: str | None = None + + def __post_init__(self) -> None: + model = normalize_model(self.model) + if int(self.dimensions) <= 0: + raise ValueError("embedding dimensions must be positive") + if float(self.timeout) <= 0: + raise ValueError("embedding timeout must be positive") + object.__setattr__(self, "base_url", normalize_base_url(self.base_url)) + object.__setattr__(self, "model", model) + object.__setattr__(self, "dimensions", int(self.dimensions)) + object.__setattr__(self, "timeout", float(self.timeout)) + + @property + def identity(self) -> dict[str, Any]: + return { + "base_url": self.base_url, + "model": self.model, + "dimensions": self.dimensions, + } @dataclass(frozen=True) -class SemanticProjectionCandidate: - document_id: str - score: float - sources: list[dict[str, Any]] +class SummaryProjectionCandidate: + file_ref: str + distance: float source_type: str title: str metadata: dict[str, Any] - snippet: str + @property + def similarity(self) -> float: + return 1.0 / (1.0 + max(0.0, self.distance)) -class SemanticProjectionSearchBackend: - """Semantic channel retrieval over rebuildable projection indexes. - The SQLite catalog remains the source of truth. This backend only reads - external sqlite-vec projection indexes and returns candidate document ids - for the catalog to resolve and filter. - """ +class SummaryProjection: + """Own the complete PIFS Summary Projection lifecycle.""" def __init__( self, index_dir: str | Path, *, - embedder: Any, - embedding_provider: str, - embedding_model: str, - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_cache_path: str | Path | None = None, + profile: SummaryEmbeddingProfile, + embedder: Any | None = None, + create: bool = False, fetch_multiplier: int = 100, ) -> None: self.index_dir = Path(index_dir).expanduser() - self.embedder = embedder - self.embedding_provider = embedding_provider - self.embedding_model = embedding_model - self.embedding_dimensions = embedding_dimensions - self.cache_model = embedding_cache_model_key(embedding_model, embedding_dimensions) - self.embedding_cache = EmbeddingCache( - Path(embedding_cache_path).expanduser() - if embedding_cache_path is not None - else self.index_dir / "embedding_cache.sqlite" - ) + self.profile = profile + self._embedder_instance = embedder self.fetch_multiplier = fetch_multiplier - self.summary_index = SQLiteVecSemanticIndex( - self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" - ) - - @classmethod - def from_provider( - cls, - index_dir: str | Path, - *, - embedding_provider: str = "openai", - embedding_model: str = "text-embedding-3-small", - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_timeout: float = 60, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - **kwargs: Any, - ) -> "SemanticProjectionSearchBackend": - return cls( - index_dir, - embedder=make_embedder( - embedding_provider, - embedding_model, - dimensions=embedding_dimensions, - timeout=embedding_timeout, - api_key=embedding_api_key, - base_url=embedding_base_url, - ), - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - **kwargs, - ) - - def search_channel( - self, - channel: str, - query: str, - *, - limit: int = 10, - filters: dict[str, Any] | None = None, - ) -> list[SemanticProjectionCandidate]: - if channel != SUMMARY_CHANNEL: - raise ValueError(f"unsupported semantic channel: {channel}") - if channel not in self.available_channels(): - return [] - query = normalize_text(query) - if not query: - return [] - vector = self.embedding_cache.embed_texts( - [query], - provider=self.embedding_provider, - model=self.cache_model, - embedder=self.embedder, - batch_size=1, - )[0] - results = self.summary_index.search( - vector, - limit=limit, - filters=filters, - fetch_multiplier=self.fetch_multiplier, - ) - return rank_single_semantic_channel(SUMMARY_CHANNEL, results) - - def available_channels(self) -> tuple[str, ...]: - return (SUMMARY_CHANNEL,) if self._summary_document_count() > 0 else () - - def info(self) -> dict[str, Any]: - return { - "index_dir": str(self.index_dir), - "embedding_provider": self.embedding_provider, - "embedding_model": self.embedding_model, - "embedding_dimensions": self.embedding_dimensions, - "strategy": "semantic_channel_vector", - "available_channels": list(self.available_channels()), - "channels": {SUMMARY_CHANNEL: self._safe_summary_info()}, - } - - def _summary_document_count(self) -> int: - info = self._safe_summary_info() - if not info.get("available"): - return 0 - return int(info.get("document_count") or 0) - - def _safe_summary_info(self) -> dict[str, Any]: - index = self.summary_index - if not index.db_path.exists(): - return { - "db_path": str(index.db_path), - "available": False, - "document_count": 0, - "error": "index file is missing", - } - try: - info = index.info() - except (OSError, sqlite3.Error, SemanticIndexError) as exc: - return { - "db_path": str(index.db_path), - "available": False, - "document_count": 0, - "error": str(exc), - } - return {**info, "available": int(info.get("document_count") or 0) > 0} - - -class SummaryProjectionIndexer: - """Synchronous register-time summary projection indexer.""" - - def __init__( - self, - index_dir: str | Path, - *, - embedder: Any, - embedding_provider: str, - embedding_model: str, - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_cache_path: str | Path | None = None, - ) -> None: - self.index_dir = Path(index_dir).expanduser() - self.index_dir.mkdir(parents=True, exist_ok=True) - self.embedder = embedder - self.embedding_provider = embedding_provider - self.embedding_model = embedding_model - self.embedding_dimensions = embedding_dimensions - self.cache_model = embedding_cache_model_key(embedding_model, embedding_dimensions) - self.embedding_cache = EmbeddingCache( - Path(embedding_cache_path).expanduser() - if embedding_cache_path is not None - else self.index_dir / "embedding_cache.sqlite" - ) self.index = SQLiteVecSemanticIndex( self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" ) - self._ensure_index() + cache_path = self.index_dir / "embedding_cache.sqlite" + database_pair = projection_database_pair(self.index_dir) + if database_pair is not None: + self.index.validate(self.profile.identity) + self.embedding_cache = EmbeddingCache(cache_path, create=False) + elif create: + self.index_dir.mkdir(parents=True, exist_ok=True) + try: + self.index.reset( + dimension=self.profile.dimensions, + metadata=self.profile.identity, + ) + self.embedding_cache = EmbeddingCache(cache_path, create=True) + except Exception: + self._cleanup_failed_create() + raise + else: + raise RuntimeError("PIFS Summary Projection is not available") - @classmethod - def from_provider( - cls, - index_dir: str | Path, - *, - embedding_provider: str = "openai", - embedding_model: str = "text-embedding-3-small", - embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, - embedding_timeout: float = 60, - embedding_api_key: str | None = None, - embedding_base_url: str | None = None, - **kwargs: Any, - ) -> "SummaryProjectionIndexer": - cls._validate_existing_index_dimension(index_dir, embedding_dimensions) - return cls( - index_dir, - embedder=make_embedder( - embedding_provider, - embedding_model, - dimensions=embedding_dimensions, - timeout=embedding_timeout, - api_key=embedding_api_key, - base_url=embedding_base_url, - ), - embedding_provider=embedding_provider, - embedding_model=embedding_model, - embedding_dimensions=embedding_dimensions, - **kwargs, - ) + def _cleanup_failed_create(self) -> None: + for path in projection_database_paths(self.index_dir): + for suffix in ("", "-journal", "-shm", "-wal"): + try: + Path(f"{path}{suffix}").unlink() + except FileNotFoundError: + continue def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: summary = str((record.get("metadata") or {}).get("summary") or "").strip() @@ -233,12 +123,10 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: return {"status": "skipped", "reason": "missing_summary"} vector = self.embedding_cache.embed_texts( [summary], - provider=self.embedding_provider, - model=self.cache_model, - embedder=self.embedder, + profile=self.profile, + embedder=self._embedder(), batch_size=1, )[0] - metadata = dict(record.get("metadata") or {}) count = self.index.upsert_many( [ SemanticIndexRecord( @@ -248,7 +136,7 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: external_id=record.get("external_id"), source_type=str(record.get("source_type") or ""), title=str(record.get("title") or ""), - metadata=metadata, + metadata=dict(record.get("metadata") or {}), ) ] ) @@ -256,135 +144,140 @@ def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: "status": "ready", "indexed_rows": count, "index_path": str(self.index.db_path), - "embedding_provider": self.embedding_provider, - "embedding_model": self.embedding_model, - "embedding_dimensions": self.embedding_dimensions, + **self.profile.identity, } def delete_summary(self, file_ref: str) -> int: return self.index.delete_file_refs([file_ref]) - def _ensure_index(self) -> None: - if not self.index.db_path.exists(): - self.index.reset( - dimension=self.embedding_dimensions, - metadata=self._index_metadata(), + def cache_keys_for_records( + self, + records: list[dict[str, Any]], + ) -> set[_EmbeddingCacheKey]: + summaries = [ + str((record.get("metadata") or {}).get("summary") or "").strip() + for record in records + ] + return self.embedding_cache.keys_for_texts( + [summary for summary in summaries if summary], + profile=self.profile, + ) + + def existing_cache_keys( + self, + keys: set[_EmbeddingCacheKey], + ) -> set[_EmbeddingCacheKey]: + return self.embedding_cache.existing_keys(keys) + + def delete_cache_keys(self, keys: set[_EmbeddingCacheKey]) -> int: + return self.embedding_cache.delete_keys(keys) + + def search( + self, + query: str, + *, + limit: int = 10, + file_refs: list[str] | None = None, + ) -> list[SummaryProjectionCandidate]: + query = normalize_text(query) + if not query or not self.available: + return [] + vector = self.embedding_cache.embed_texts( + [query], + profile=self.profile, + embedder=self._embedder(), + batch_size=1, + )[0] + filters = {"file_ref": file_refs} if file_refs is not None else None + return [ + SummaryProjectionCandidate( + file_ref=result.file_ref, + distance=result.distance, + source_type=result.source_type, + title=result.title, + metadata=result.metadata, ) - return - try: - existing_dimension = self.index.dimension() - except Exception as exc: - raise RuntimeError( - "could not validate existing summary projection index config; " - f"refusing to reset {self.index.db_path}. Move the existing index " - "aside or rebuild it intentionally before changing embedding config." - ) from exc - if existing_dimension != self.embedding_dimensions: - raise self._dimension_mismatch_error( - self.index.db_path, - existing_dimension, - self.embedding_dimensions, + for result in self.index.search( + vector, + limit=limit, + filters=filters, + fetch_multiplier=self.fetch_multiplier, ) + ] - def _index_metadata(self) -> dict[str, Any]: + @property + def available(self) -> bool: + return int(self.index.info().get("document_count") or 0) > 0 + + def info(self) -> dict[str, Any]: return { - "channel": "summary", - "embedding_provider": self.embedding_provider, - "embedding_model": self.embedding_model, - "embedding_dimensions": self.embedding_dimensions, + **self.index.info(), + "embedding_identity": self.profile.identity, + "available": self.available, } - @classmethod - def _validate_existing_index_dimension( - cls, - index_dir: str | Path, - embedding_dimensions: int, - ) -> None: - index_path = ( - Path(index_dir).expanduser() / f"{SUMMARY_INDEX_NAME}.sqlite" - ) - if not index_path.exists(): - return - index = SQLiteVecSemanticIndex(index_path) - try: - existing_dimension = index.dimension() - except Exception as exc: - raise RuntimeError( - "could not validate existing summary projection index config; " - f"refusing to reset {index_path}. Move the existing index " - "aside or rebuild it intentionally before changing embedding config." - ) from exc - if existing_dimension != embedding_dimensions: - raise cls._dimension_mismatch_error( - index_path, - existing_dimension, - embedding_dimensions, - ) - - @staticmethod - def _dimension_mismatch_error( - index_path: Path, - existing_dimension: int, - embedding_dimensions: int, - ) -> RuntimeError: - return RuntimeError( - "summary projection index dimension mismatch: " - f"{index_path} was built with dimension {existing_dimension}, " - f"but configured embedding_dimensions is {embedding_dimensions}. " - "Use the matching embedding config, or rebuild the projection index " - "at a new path after preserving the existing data." - ) + def _embedder(self) -> Any: + if self._embedder_instance is None: + self._embedder_instance = EmbeddingClient(self.profile) + return self._embedder_instance class EmbeddingCache: - def __init__(self, db_path: Path): + def __init__(self, db_path: Path, *, create: bool) -> None: self.db_path = db_path - self.db_path.parent.mkdir(parents=True, exist_ok=True) - with self.connect() as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS embedding_cache ( - provider TEXT NOT NULL, - model TEXT NOT NULL, - text_hash TEXT NOT NULL, - dimension INTEGER NOT NULL, - vector_blob BLOB, - vector_json TEXT, - created_at TEXT DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY(provider, model, text_hash) - ) - """ + exists = self.db_path.exists() + if exists: + self._validate() + elif create: + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as connection: + self._create_schema(connection) + else: + raise RuntimeError( + "PIFS embedding cache is missing; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." ) - conn.commit() - def connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path) - conn.row_factory = sqlite3.Row - return conn + def connect(self, *, read_only: bool = False) -> sqlite3.Connection: + if read_only: + connection = sqlite3.connect( + f"{self.db_path.resolve().as_uri()}?mode=ro", + uri=True, + ) + else: + connection = sqlite3.connect(self.db_path) + connection.row_factory = sqlite3.Row + return connection def embed_texts( self, texts: list[str], *, - provider: str, - model: str, + profile: SummaryEmbeddingProfile, embedder: Any, batch_size: int, ) -> list[list[float]]: hashes = [SQLiteVecSemanticIndex.text_hash(text) for text in texts] cached: dict[str, list[float]] = {} - with self.connect() as conn: + with self.connect() as connection: for text_hash in sorted(set(hashes)): - row = conn.execute( + row = connection.execute( """ - SELECT vector_blob, vector_json + SELECT vector_blob FROM embedding_cache - WHERE provider = ? AND model = ? AND text_hash = ? + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? """, - (provider, model, text_hash), + ( + profile.base_url, + profile.model, + profile.dimensions, + text_hash, + ), ).fetchone() if row is not None: - cached[text_hash] = decode_vector(row["vector_blob"], row["vector_json"]) + cached[text_hash] = decode_vector( + bytes(row["vector_blob"]), profile.dimensions + ) missing_positions = [ index for index, text_hash in enumerate(hashes) if text_hash not in cached ] @@ -397,117 +290,196 @@ def embed_texts( "embedding response length mismatch: " f"requested {len(positions)}, received {len(vectors)}" ) - with self.connect() as conn: - conn.executemany( + for vector in vectors: + if len(vector) != profile.dimensions: + raise ValueError( + "embedding dimension mismatch: " + f"expected {profile.dimensions}, received {len(vector)}" + ) + with self.connect() as connection: + connection.executemany( """ INSERT OR REPLACE INTO embedding_cache( - provider, model, text_hash, dimension, vector_blob, vector_json - ) - VALUES (?, ?, ?, ?, ?, '') + base_url, model, dimensions, text_hash, vector_blob + ) VALUES (?, ?, ?, ?, ?) """, [ ( - provider, - model, + profile.base_url, + profile.model, + profile.dimensions, hashes[index], - len(vector), encode_vector(vector), ) for index, vector in zip(positions, vectors) ], ) - conn.commit() for index, vector in zip(positions, vectors): cached[hashes[index]] = vector return [cached[text_hash] for text_hash in hashes] + @staticmethod + def keys_for_texts( + texts: list[str], + *, + profile: SummaryEmbeddingProfile, + ) -> set[_EmbeddingCacheKey]: + return { + ( + str(profile.base_url), + profile.model, + profile.dimensions, + SQLiteVecSemanticIndex.text_hash(text), + ) + for text in texts + } -class EmbeddingClient: - def __init__( + def existing_keys( self, - *, - provider: str, - model: str, - dimensions: int, - timeout: float, - api_key: str | None = None, - base_url: str | None = None, - ): - self.provider = provider.lower() - self.model = model - self.dimensions = dimensions - if self.provider != "openai": - raise ValueError(f"unknown embedding provider: {provider}") + keys: set[_EmbeddingCacheKey], + ) -> set[_EmbeddingCacheKey]: + existing: set[_EmbeddingCacheKey] = set() + with self.connect(read_only=True) as connection: + for key in sorted(keys): + if connection.execute( + """ + SELECT 1 + FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + key, + ).fetchone() is not None: + existing.add(key) + return existing + + def delete_keys(self, keys: set[_EmbeddingCacheKey]) -> int: + if not keys: + return 0 + with self.connect() as connection: + before = connection.total_changes + connection.executemany( + """ + DELETE FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + sorted(keys), + ) + return connection.total_changes - before + + def _validate(self) -> None: + try: + with self.connect(read_only=True) as connection: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + actual = self._schema_signature(connection) + invalid_rows = int( + connection.execute( + "SELECT COUNT(*) FROM embedding_cache " + "WHERE dimensions <= 0 OR length(vector_blob) != dimensions * 4 " + "OR trim(base_url) = '' OR trim(model) = '' OR trim(text_hash) = ''" + ).fetchone()[0] + ) + identities = connection.execute( + "SELECT DISTINCT base_url, model FROM embedding_cache" + ).fetchall() + with sqlite3.connect(":memory:") as expected_connection: + expected_connection.row_factory = sqlite3.Row + self._create_schema(expected_connection) + expected = self._schema_signature(expected_connection) + invalid_identity = any( + normalize_base_url(row["base_url"]) != row["base_url"] + or normalize_model(row["model"]) != row["model"] + for row in identities + ) + except (sqlite3.Error, ValueError) as exc: + raise self._incompatible_schema_error() from exc + if ( + version != SCHEMA_VERSION + or actual != expected + or invalid_rows + or invalid_identity + ): + raise self._incompatible_schema_error() + + @staticmethod + def _create_schema(connection: sqlite3.Connection) -> None: + connection.executescript( + f""" + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(base_url, model, dimensions, text_hash) + ); + PRAGMA user_version = {SCHEMA_VERSION}; + """ + ) + + @staticmethod + def _schema_signature(connection: sqlite3.Connection) -> dict[str, Any]: + tables = regular_table_names(connection) + if tables != {"embedding_cache"}: + return {"tables": tables} + signature = sqlite_schema_signature(connection, tables) + signature["sql"] = normalized_table_sql(connection, "embedding_cache") + return signature + + @staticmethod + def _incompatible_schema_error() -> RuntimeError: + return RuntimeError( + "Incompatible PIFS embedding cache schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + +def validate_projection_topology(index_dir: str | Path) -> bool: + index_dir = Path(index_dir).expanduser() + database_pair = projection_database_pair(index_dir) + if database_pair is None: + return False + summary_path, cache_path = database_pair + SQLiteVecSemanticIndex(summary_path).validate_schema() + EmbeddingCache(cache_path, create=False) + return True + + +class EmbeddingClient: + def __init__(self, profile: SummaryEmbeddingProfile) -> None: from openai import OpenAI - if not api_key: + if not profile.api_key: raise ValueError("embedding_api_key is required for PIFS embeddings") + self.profile = profile self.client = OpenAI( - api_key=api_key, - base_url=base_url or None, - timeout=timeout, + api_key=profile.api_key, + base_url=profile.base_url, + timeout=profile.timeout, ) def embed(self, texts: list[str]) -> list[list[float]]: - kwargs: dict[str, Any] = {"model": self.model, "input": texts} - if self.dimensions > 0: - kwargs["dimensions"] = self.dimensions - response = self.client.embeddings.create(**kwargs) - return [list(item.embedding) for item in sorted(response.data, key=lambda item: item.index)] - - -def make_embedder( - provider: str, - model: str, - *, - dimensions: int, - timeout: float, - api_key: str | None = None, - base_url: str | None = None, -) -> Any: - return EmbeddingClient( - provider=provider, - model=model, - dimensions=dimensions, - timeout=timeout, - api_key=api_key, - base_url=base_url, - ) - - -def rank_single_semantic_channel( - channel: str, - results: list[SemanticSearchResult], -) -> list[SemanticProjectionCandidate]: - rows: list[SemanticProjectionCandidate] = [] - seen: set[str] = set() - for rank, result in enumerate(results, 1): - doc_id = str(result.external_id or result.file_ref) - if doc_id in seen: - continue - seen.add(doc_id) - rows.append( - SemanticProjectionCandidate( - document_id=doc_id, - score=1 / (60 + rank), - sources=[{"channel": channel, "rank": rank, "distance": result.distance}], - source_type=result.source_type, - title=result.title, - metadata=result.metadata, - snippet=f"{channel}_vector rank={rank}", - ) + response = self.client.embeddings.create( + model=self.profile.model, + input=texts, + dimensions=self.profile.dimensions, ) - return rows + return [ + list(item.embedding) + for item in sorted(response.data, key=lambda item: item.index) + ] + def normalize_text(text: str) -> str: return " ".join(str(text or "").split()) -def embedding_cache_model_key(model: str, dimensions: int) -> str: - return f"{model}:dimensions={dimensions}" if dimensions > 0 else model - - -def embed_with_retry(embedder: Any, texts: list[str], *, max_attempts: int = 8) -> list[list[float]]: +def embed_with_retry( + embedder: Any, + texts: list[str], + *, + max_attempts: int = 8, +) -> list[list[float]]: for attempt in range(1, max_attempts + 1): try: return embedder.embed(texts) @@ -528,9 +500,7 @@ def is_retryable_embedding_error(exc: Exception) -> bool: except (TypeError, ValueError): status = None if status is not None: - if status in {408, 409, 429}: - return True - if status >= 500: + if status in {408, 409, 429} or status >= 500: return True if 400 <= status < 500: return False @@ -542,13 +512,9 @@ def encode_vector(vector: list[float]) -> bytes: return struct.pack(f"<{len(vector)}f", *vector) -def decode_vector(blob: bytes | None, vector_json: str | None) -> list[float]: - if blob: - if len(blob) % 4 != 0: - raise ValueError("invalid cached vector blob length") - return list(struct.unpack(f"<{len(blob) // 4}f", blob)) - if vector_json: - value = json.loads(vector_json) - if isinstance(value, list): - return [float(item) for item in value] - raise ValueError("cached embedding row does not contain a vector") +def decode_vector(blob: bytes, dimensions: int) -> list[float]: + if len(blob) != dimensions * 4: + raise ValueError( + f"cached embedding has {len(blob) // 4} dimensions, expected {dimensions}" + ) + return list(struct.unpack(f"<{dimensions}f", blob)) diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py index a714c3979..31450e67d 100644 --- a/pageindex/filesystem/store.py +++ b/pageindex/filesystem/store.py @@ -7,9 +7,17 @@ from pathlib import Path from typing import Any, Iterable, Optional +from ._sqlite_schema import regular_table_names, sqlite_schema_signature from .types import FileEntry, MetadataField -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 +CATALOG_TABLES = { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", +} class SQLiteFileSystemStore: @@ -20,9 +28,9 @@ def __init__(self, workspace: str | Path): self.text_dir = self.workspace / "artifacts" / "text" self.raw_dir = self.workspace / "artifacts" / "raw" self.pageindex_client_dir = self.workspace / "artifacts" / "pageindex_client" + self.initialize_schema() for path in (self.text_dir, self.raw_dir, self.pageindex_client_dir): path.mkdir(parents=True, exist_ok=True) - self.initialize_schema() def connect(self) -> sqlite3.Connection: conn = sqlite3.connect(self.db_path) @@ -31,12 +39,37 @@ def connect(self) -> sqlite3.Connection: return conn def initialize_schema(self) -> None: + if self.db_path.exists() or self.db_path.is_symlink(): + self.validate_existing_database(self.db_path) + return with self.connect() as conn: self._create_current_schema(conn) self.ensure_folder(conn, "/") conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") - def _create_current_schema(self, conn: sqlite3.Connection) -> None: + @classmethod + def validate_existing_database(cls, db_path: str | Path) -> None: + db_path = Path(db_path) + if not db_path.is_file() or db_path.stat().st_size <= 0: + raise cls._incompatible_schema_error() + try: + with cls._readonly_connection(db_path) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + actual = cls._schema_signature(conn) + with sqlite3.connect(":memory:") as expected_conn: + expected_conn.row_factory = sqlite3.Row + cls._create_current_schema(expected_conn) + expected = cls._schema_signature(expected_conn) + except sqlite3.Error as exc: + raise cls._incompatible_schema_error() from exc + if version != SCHEMA_VERSION or actual != expected: + raise cls._incompatible_schema_error() + from ._workspace_consistency import validate_catalog_root + + validate_catalog_root(db_path) + + @staticmethod + def _create_current_schema(conn: sqlite3.Connection) -> None: conn.executescript( """ CREATE TABLE IF NOT EXISTS files ( @@ -101,19 +134,38 @@ def _create_current_schema(self, conn: sqlite3.Connection) -> None: FOREIGN KEY(field_id) REFERENCES metadata_fields(field_id) ON DELETE CASCADE ); - CREATE VIRTUAL TABLE IF NOT EXISTS file_fts - USING fts5(file_ref UNINDEXED, title, body, metadata_text); - CREATE INDEX IF NOT EXISTS idx_files_external_id ON files(external_id); CREATE INDEX IF NOT EXISTS idx_files_source_type ON files(source_type); - CREATE INDEX IF NOT EXISTS idx_folders_path ON folders(path); CREATE INDEX IF NOT EXISTS idx_folders_parent_id ON folders(parent_id); CREATE INDEX IF NOT EXISTS idx_file_folders_folder ON file_folders(folder_id); - CREATE INDEX IF NOT EXISTS idx_metadata_fields_name ON metadata_fields(name); CREATE INDEX IF NOT EXISTS idx_metadata_values_field_text ON metadata_values(field_id, value_text); """ ) + @staticmethod + def _readonly_connection(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect( + f"{path.resolve().as_uri()}?mode=ro", + uri=True, + ) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + return connection + + @staticmethod + def _schema_signature(conn: sqlite3.Connection) -> dict[str, Any]: + tables = regular_table_names(conn) + return sqlite_schema_signature(conn, tables) + + @staticmethod + def _incompatible_schema_error() -> RuntimeError: + return RuntimeError( + "Incompatible PIFS catalog schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it " + f"(expected exact schema version {SCHEMA_VERSION} with tables " + f"{sorted(CATALOG_TABLES)})." + ) + @staticmethod def _json_object(value: Any) -> dict[str, Any]: try: @@ -138,8 +190,6 @@ def insert_files(self, records: list[dict[str, Any]]) -> None: file_rows = [] membership_rows = [] file_ref_rows = [] - fts_file_ref_rows = [] - fts_rows = [] metadata_rows = [] pending_folder_titles: dict[tuple[str, str], str] = {} metadata_field_ids = { @@ -179,16 +229,6 @@ def insert_files(self, records: list[dict[str, Any]]) -> None: ) ) file_ref_rows.append((record["file_ref"],)) - if not record.get("skip_fts", False): - fts_file_ref_rows.append((record["file_ref"],)) - fts_rows.append( - ( - record["file_ref"], - record["title"], - record["content"], - record["metadata_text"], - ) - ) metadata_rows.extend( self._metadata_insert_values( record["file_ref"], @@ -214,15 +254,6 @@ def insert_files(self, records: list[dict[str, Any]]) -> None: """, metadata_rows, ) - if fts_file_ref_rows: - conn.executemany("DELETE FROM file_fts WHERE file_ref = ?", fts_file_ref_rows) - conn.executemany( - """ - INSERT INTO file_fts(file_ref, title, body, metadata_text) - VALUES (?, ?, ?, ?) - """, - fts_rows, - ) @staticmethod def _file_insert_sql() -> str: @@ -482,21 +513,6 @@ def _registered_field_id(conn: sqlite3.Connection, name: str) -> str | None: ).fetchone() return None if row is None else row["field_id"] - def replace_fts(self, conn: sqlite3.Connection, record: dict[str, Any]) -> None: - conn.execute("DELETE FROM file_fts WHERE file_ref = ?", (record["file_ref"],)) - conn.execute( - """ - INSERT INTO file_fts(file_ref, title, body, metadata_text) - VALUES (?, ?, ?, ?) - """, - ( - record["file_ref"], - record["title"], - record["content"], - record["metadata_text"], - ), - ) - def upsert_metadata_fields( self, fields: Iterable[MetadataField], @@ -539,6 +555,26 @@ def metadata_field_exists(self, name: str) -> bool: ).fetchone() return row is not None + def delete_metadata_field_if_unreferenced(self, name: str) -> bool: + with self.connect() as conn: + row = conn.execute( + "SELECT field_id FROM metadata_fields WHERE name = ?", + (name,), + ).fetchone() + if row is None: + return False + referenced = conn.execute( + "SELECT 1 FROM metadata_values WHERE field_id = ? LIMIT 1", + (row["field_id"],), + ).fetchone() + if referenced is not None: + return False + cursor = conn.execute( + "DELETE FROM metadata_fields WHERE field_id = ?", + (row["field_id"],), + ) + return cursor.rowcount > 0 + def list_metadata_fields(self) -> list[MetadataField]: with self.connect() as conn: rows = conn.execute( @@ -787,48 +823,13 @@ def find_folders( rows = conn.execute(sql, params).fetchall() return [self._folder_row_to_dict(row) for row in rows] - def search_files( + def list_files( self, - query: str | list[str] | None, *, scope: Optional[dict[str, Any]] = None, metadata_filter: Optional[dict[str, Any]] = None, - limit: int = 10, + limit: int = 100, ) -> list[dict[str, Any]]: - query_text = self._query_text(query) - match_queries = self._fts_match_queries(query_text) if query_text else [None] - if query_text and not match_queries: - match_queries = [None] - results: list[dict[str, Any]] = [] - seen: set[str] = set() - for match_query in match_queries: - if query_text and match_query is None: - rows = self._search_literal_once( - query_text, - scope, - metadata_filter, - max(limit * 25, limit), - ) - else: - rows = self._search_once(match_query, scope, metadata_filter, max(limit * 25, limit)) - for row in rows: - if row["file_ref"] in seen: - continue - seen.add(row["file_ref"]) - results.append(self._search_row_to_dict(row)) - if len(results) >= limit: - return results - if results: - return results - return results - - def _search_literal_once( - self, - query_text: str, - scope: Optional[dict[str, Any]], - metadata_filter: Optional[dict[str, Any]], - limit: int, - ) -> list[sqlite3.Row]: selects = [ "f.file_ref", "f.external_id", @@ -860,21 +861,9 @@ def _search_literal_once( LIMIT 1 ) AS folder_path """, - "f.descriptor AS snippet", - "0 AS rank", - ] - where = [ - "f.deleted_at IS NULL", - """ - ( - lower(file_fts.title) LIKE lower(?) ESCAPE '\\' - OR lower(file_fts.body) LIKE lower(?) ESCAPE '\\' - OR lower(file_fts.metadata_text) LIKE lower(?) ESCAPE '\\' - ) - """, ] - literal_like = self._contains_like(query_text) - params: list[Any] = [literal_like, literal_like, literal_like] + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] scope_sql, scope_params = self._scope_sql(scope) if scope_sql: where.append(scope_sql) @@ -885,14 +874,14 @@ def _search_literal_once( sql = f""" SELECT {", ".join(selects)} FROM files f - JOIN file_fts ON file_fts.file_ref = f.file_ref WHERE {" AND ".join(where)} - ORDER BY f.created_at DESC, f.title + ORDER BY lower(f.title), f.title, f.file_ref LIMIT ? """ params.append(limit) with self.connect() as conn: - return conn.execute(sql, params).fetchall() + rows = conn.execute(sql, params).fetchall() + return [self._file_summary(row) for row in rows] def file_refs_for_scope( self, @@ -1027,78 +1016,6 @@ def list_metadata_values( for row in rows ] - def _search_once( - self, - match_query: str | None, - scope: Optional[dict[str, Any]], - metadata_filter: Optional[dict[str, Any]], - limit: int, - ) -> list[sqlite3.Row]: - joins = [] - selects = [ - "f.file_ref", - "f.external_id", - "f.title", - "f.descriptor", - "f.pageindex_tree_status", - "f.metadata_json", - "f.metadata_status_json", - "f.created_at", - """ - ( - SELECT display_folder.folder_id - FROM file_folders display_ff - JOIN folders display_folder - ON display_folder.folder_id = display_ff.folder_id - WHERE display_ff.file_ref = f.file_ref - ORDER BY display_folder.path - LIMIT 1 - ) AS folder_id - """, - """ - ( - SELECT display_folder.path - FROM file_folders display_ff - JOIN folders display_folder - ON display_folder.folder_id = display_ff.folder_id - WHERE display_ff.file_ref = f.file_ref - ORDER BY display_folder.path - LIMIT 1 - ) AS folder_path - """, - ] - where = ["f.deleted_at IS NULL"] - params: list[Any] = [] - if match_query: - joins.append("JOIN file_fts ON file_fts.file_ref = f.file_ref") - selects.append("snippet(file_fts, 2, '', '', '...', 16) AS snippet") - selects.append("bm25(file_fts) AS rank") - where.append("file_fts MATCH ?") - params.append(match_query) - order_by = "rank" - else: - selects.append("f.descriptor AS snippet") - selects.append("0 AS rank") - order_by = "f.created_at DESC, f.title" - scope_sql, scope_params = self._scope_sql(scope) - if scope_sql: - where.append(scope_sql) - params.extend(scope_params) - metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) - where.extend(metadata_sql) - params.extend(metadata_params) - sql = f""" - SELECT {", ".join(selects)} - FROM files f - {" ".join(joins)} - WHERE {" AND ".join(where)} - ORDER BY {order_by} - LIMIT ? - """ - params.append(limit) - with self.connect() as conn: - return conn.execute(sql, params).fetchall() - def _metadata_filter_sql(self, metadata_filter: Optional[dict[str, Any]]) -> tuple[list[str], list[Any]]: if not metadata_filter: return [], [] @@ -1205,7 +1122,6 @@ def update_file_metadata_status( if row is None: raise KeyError(f"Unknown file_ref: {file_ref}") indexed_metadata = self.indexed_metadata_values(metadata) - metadata_text_value = metadata_text(indexed_metadata) conn.execute( """ UPDATE files @@ -1225,19 +1141,10 @@ def update_file_metadata_status( file_ref, indexed_metadata, ) - conn.execute( - """ - UPDATE file_fts - SET metadata_text = ? - WHERE file_ref = ? - """, - (metadata_text_value, file_ref), - ) def delete_file(self, target: str) -> None: with self.connect() as conn: file_ref = self._resolve_file_ref(conn, target) - conn.execute("DELETE FROM file_fts WHERE file_ref = ?", (file_ref,)) conn.execute("DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,)) conn.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) @@ -1573,9 +1480,7 @@ def folder_memberships(self, file_ref: str) -> list[dict[str, Any]]: return [ { "folder_id": row["folder_id"], - "id": row["folder_id"], "parent_id": row["parent_id"], - "parent_folder_id": row["parent_id"], "name": row["name"], "path": row["path"], "kind": row["kind"], @@ -1939,9 +1844,7 @@ def _folder_depth_sql(path_expr: str) -> str: def _folder_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: return { "folder_id": row["folder_id"], - "id": row["folder_id"], "parent_id": row["parent_id"], - "parent_folder_id": row["parent_id"], "name": row["name"], "description": cls._row_value(row, "description", ""), "path": row["path"], @@ -1956,44 +1859,17 @@ def _folder_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: @classmethod def _file_summary(cls, row: sqlite3.Row) -> dict[str, Any]: - external_id = row["external_id"] display_title = cls._row_value(row, "display_title", row["title"]) return { "file_ref": row["file_ref"], - "id": external_id or row["file_ref"], - "document_id": external_id, - "external_id": external_id, - "name": display_title, + "external_id": row["external_id"], "title": display_title, - "original_title": row["title"], - "description": cls._row_value(row, "descriptor", row["title"]), - "status": cls._row_value(row, "pageindex_tree_status", "not_built"), - "pageNum": None, - "createdAt": cls._row_value(row, "created_at"), - "folderId": cls._row_value(row, "folder_id"), - "folder_path": row["folder_path"], - "metadata": json.loads(row["metadata_json"] or "{}"), - "metadata_status": json.loads( - cls._row_value(row, "metadata_status_json", "{}") or "{}" + "descriptor": cls._row_value(row, "descriptor", row["title"]), + "pageindex_tree_status": cls._row_value( + row, "pageindex_tree_status", "not_built" ), - } - - @classmethod - def _search_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: - external_id = row["external_id"] - return { - "file_ref": row["file_ref"], - "id": external_id or row["file_ref"], - "document_id": external_id, - "external_id": external_id, - "name": row["title"], - "title": row["title"], - "description": cls._row_value(row, "descriptor", row["title"]), - "status": cls._row_value(row, "pageindex_tree_status", "not_built"), - "pageNum": None, - "createdAt": cls._row_value(row, "created_at"), - "folderId": cls._row_value(row, "folder_id"), - "snippet": row["snippet"] or row["title"], + "created_at": cls._row_value(row, "created_at"), + "folder_id": cls._row_value(row, "folder_id"), "folder_path": row["folder_path"], "metadata": json.loads(row["metadata_json"] or "{}"), "metadata_status": json.loads( @@ -2031,15 +1907,8 @@ def _file_entry(row: sqlite3.Row) -> FileEntry: def _file_entry_to_dict(cls, entry: FileEntry) -> dict[str, Any]: return { "file_ref": entry.file_ref, - "id": entry.external_id or entry.file_ref, - "document_id": entry.external_id, "external_id": entry.external_id, - "name": entry.title, - "path": cls._virtual_file_path(entry.folder_path, entry.title), "title": entry.title, - "description": entry.descriptor, - "status": entry.pageindex_tree_status, - "pageNum": None, "descriptor": entry.descriptor, "content_type": entry.content_type, "source_type": entry.source_type, @@ -2051,75 +1920,6 @@ def _file_entry_to_dict(cls, entry: FileEntry) -> dict[str, Any]: "folder_path": entry.folder_path, } - @staticmethod - def _virtual_file_path(folder_path: str, title: str) -> str: - folder_path = normalize_path(folder_path) - return f"/{title}" if folder_path == "/" else f"{folder_path}/{title}" - - @staticmethod - def _query_text(query: str | list[str] | None) -> str: - if query is None: - return "" - if isinstance(query, list): - return " ".join(str(item) for item in query) - return str(query) - - @classmethod - def _fts_match_queries(cls, query: str) -> list[str]: - terms = cls._fts_terms(query) - if not terms: - return [] - queries = [" ".join(terms)] - if len(terms) > 1: - queries.append(" OR ".join(terms)) - return queries - - @staticmethod - def _fts_terms(query: str) -> list[str]: - stopwords = { - "a", - "an", - "and", - "are", - "as", - "at", - "be", - "by", - "did", - "do", - "does", - "for", - "from", - "how", - "in", - "is", - "it", - "of", - "on", - "or", - "that", - "the", - "to", - "was", - "were", - "what", - "when", - "where", - "which", - "who", - "why", - "with", - } - terms = re.findall(r"[A-Za-z0-9_]+", query.lower()) - unique_terms = [] - seen = set() - for term in terms: - if term in stopwords or term in seen: - continue - seen.add(term) - unique_terms.append(term) - return unique_terms - @staticmethod def _metadata_value_items(value: Any) -> list[str]: if value is None: @@ -2179,15 +1979,3 @@ def make_file_ref(seed: str) -> str: def fingerprint(content: str) -> str: return hashlib.sha256(content.encode("utf-8")).hexdigest() - - -def metadata_text(metadata: dict[str, Any]) -> str: - values = [] - for value in metadata.values(): - if isinstance(value, list): - values.extend(str(item) for item in value) - elif isinstance(value, dict): - values.append(json.dumps(value, ensure_ascii=False, sort_keys=True)) - elif value is not None: - values.append(str(value)) - return " ".join(values) diff --git a/pageindex/filesystem/types.py b/pageindex/filesystem/types.py index 82c04bb54..6f933e5f6 100644 --- a/pageindex/filesystem/types.py +++ b/pageindex/filesystem/types.py @@ -4,45 +4,6 @@ from typing import Any, Optional -@dataclass(frozen=True) -class SearchResult: - file_ref: str - external_id: Optional[str] - title: str - snippet: str - folder_path: str - folder_paths: list[str] - metadata: dict[str, Any] - id: Optional[str] = None - document_id: Optional[str] = None - name: str = "" - description: str = "" - status: str = "" - pageNum: Optional[int] = None - createdAt: Optional[str] = None - folderId: Optional[str] = None - metadata_status: dict[str, Any] = field(default_factory=dict) - - -@dataclass(frozen=True) -class OpenResult: - file_ref: str - start_line: int - end_line: int - text: str - external_id: Optional[str] = None - folder_path: str = "" - - -@dataclass(frozen=True) -class FolderEntry: - folder_id: str - parent_id: Optional[str] - name: str - path: str - kind: str - - @dataclass(frozen=True) class FileEntry: file_ref: str @@ -75,10 +36,3 @@ class PIFSQueryScope: folder_path: str metadata_filter: dict[str, str] = field(default_factory=dict) metadata_axis: Optional[str] = None - - -@dataclass(frozen=True) -class CommandResult: - command: str - data: Any - text: str diff --git a/tests/pifs_markdown_fixture.py b/tests/pifs_markdown_fixture.py index cc7dc3525..deda5bcac 100644 --- a/tests/pifs_markdown_fixture.py +++ b/tests/pifs_markdown_fixture.py @@ -38,6 +38,14 @@ def get_page_content(self, doc_id, pages): return json.dumps(self.documents[doc_id]["pages"]) +class FakeEmbeddingClient: + def __init__(self, dimensions: int): + self.dimensions = dimensions + + def embed(self, texts: list[str]) -> list[list[float]]: + return [[1.0, *([0.0] * (self.dimensions - 1))] for _ in texts] + + def register_markdown( filesystem, tmp_path: Path, @@ -53,6 +61,16 @@ def register_markdown( client = FakePageIndexClient() filesystem._test_pageindex_client = client filesystem._pageindex_client = lambda: client + if filesystem.summary_projection is None: + from pageindex.filesystem.semantic_projection import SummaryProjection + + profile = filesystem._summary_embedding_profile() + filesystem.summary_projection = SummaryProjection( + filesystem.summary_projection_index_dir, + profile=profile, + embedder=FakeEmbeddingClient(profile.dimensions), + create=True, + ) filename = title if title and title.endswith(".md") else f"{external_id}.md" source = tmp_path / filename source.write_text(text or f"{external_id} alpha evidence", encoding="utf-8") diff --git a/tests/test_filesystem_store.py b/tests/test_filesystem_store.py index affeb50cb..d9b726d42 100644 --- a/tests/test_filesystem_store.py +++ b/tests/test_filesystem_store.py @@ -35,9 +35,6 @@ def connect(self): "raw_artifact_path": None, "metadata": {}, "metadata_json": json.dumps({}), - "metadata_text": "", - "content": "", - "skip_fts": True, } ] ) @@ -106,7 +103,7 @@ def test_file_upsert_preserves_soft_delete_tombstone(tmp_path): assert row["storage_uri"].endswith("/doc_report.md") assert row["deleted_at"] == "2001-02-03 04:05:06" - assert filesystem.search(None) == [] + assert filesystem.store.list_files() == [] def test_listing_uses_one_consistent_folder_membership_row(tmp_path): @@ -127,7 +124,7 @@ def test_listing_uses_one_consistent_folder_membership_row(tmp_path): metadata={"display_name": "Alpha"}, ) - listing = filesystem.browse("/", recursive=True, limit=10) + listing = filesystem.store.list_folder("/", recursive=True, limit=10) row = next(item for item in listing["files"] if item["external_id"] == "doc_shared") assert row["folder_path"] == "/a" diff --git a/tests/test_pifs_add_command.py b/tests/test_pifs_add_command.py deleted file mode 100644 index 80670eb28..000000000 --- a/tests/test_pifs_add_command.py +++ /dev/null @@ -1,484 +0,0 @@ -import json -from pathlib import Path - -import pytest - - -class StaticEmbedder: - def embed(self, texts): - return [[1.0, 0.0, 0.0] for _ in texts] - - -def make_summary_indexer(workspace: Path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - return SummaryProjectionIndexer( - workspace / "artifacts" / "projection_indexes", - embedder=StaticEmbedder(), - embedding_provider="test", - embedding_model="static", - embedding_dimensions=3, - ) - - -def make_filesystem(workspace: Path): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem( - workspace=workspace, - summary_projection_embedding_provider="test", - summary_projection_embedding_model="static", - summary_projection_embedding_dimensions=3, - ) - filesystem.summary_projection_indexer = make_summary_indexer(workspace) - return filesystem - - -@pytest.fixture(autouse=True) -def fake_pageindex_index(monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - path = Path(file_path) - doc_id = f"doc_{path.stem}" - text = path.read_text(encoding="utf-8") - doc = { - "id": doc_id, - "type": "md", - "path": str(path.resolve()), - "doc_name": path.name, - "doc_description": f"Summary for {path.name}: {text[:60]}", - "line_count": len(text.splitlines()), - "structure": [ - { - "title": path.stem, - "node_id": "0001", - "line_num": 1, - "text": text, - "nodes": [], - } - ], - "pages": [{"page": 1, "content": text}], - } - write_pageindex_client_doc(self.workspace, doc_id, doc) - self.documents[doc_id] = doc - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - - -def write_pageindex_client_doc(workspace: Path, doc_id: str, doc: dict) -> None: - workspace.mkdir(parents=True, exist_ok=True) - (workspace / f"{doc_id}.json").write_text( - json.dumps(doc, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - meta = { - doc_id: { - "type": doc.get("type", ""), - "doc_name": doc.get("doc_name", ""), - "doc_description": doc.get("doc_description", ""), - "path": doc.get("path", ""), - "line_count": doc.get("line_count"), - } - } - (workspace / "_meta.json").write_text( - json.dumps(meta, ensure_ascii=False, indent=2), - encoding="utf-8", - ) - - -def test_add_text_folder_target_copies_artifact_indexes_summary_and_is_readable(tmp_path): - from pageindex.filesystem import PIFSCommandExecutor - - source = tmp_path / "filing.md" - source.write_text("alpha filing text for pifs add", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - info = filesystem.add_file(str(source), "/documents/reports") - - assert info["path"] == "/documents/reports/filing.md" - assert info["folder_path"] == "/documents/reports" - assert filesystem.folder_info("/documents/reports")["path"] == "/documents/reports" - entry = filesystem.store.get_file(info["file_ref"]) - assert entry.storage_uri != source.as_uri() - assert "/artifacts/uploads/" in entry.storage_uri - copied_path = Path(entry.storage_uri.removeprefix("file://")) - assert copied_path.read_text(encoding="utf-8") == "alpha filing text for pifs add" - assert copied_path.resolve() != source.resolve() - - executor = PIFSCommandExecutor(filesystem) - rendered = json.loads(executor.execute("grep alpha /documents/reports/filing.md")) - - assert rendered["data"]["matches"] == [ - {"line": 1, "text": "alpha filing text for pifs add"} - ] - assert info["metadata"]["summary"].startswith("Summary for filing.md") - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 1 - - -def test_add_rejects_same_folder_same_basename_without_overwrite(tmp_path): - from pageindex.filesystem import PIFSCommandExecutor - - source = tmp_path / "conflict.md" - source.write_text("first body", encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - filesystem.add_file(source, "/documents") - source.write_text("second body must not overwrite", encoding="utf-8") - - with pytest.raises(FileExistsError, match="already exists"): - filesystem.add_file(source, "/documents") - - executor = PIFSCommandExecutor(filesystem) - rendered = json.loads(executor.execute("grep first /documents/conflict.md")) - assert rendered["data"]["matches"] == [{"line": 1, "text": "first body"}] - - -def test_add_rejects_unsupported_type_before_registration(tmp_path): - source = tmp_path / "payload.json" - source.write_text('{"unsupported": true}', encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - with pytest.raises(ValueError, match="Unsupported file type"): - filesystem.add_file(source, "/documents") - - assert filesystem.browse("/", recursive=True)["files"] == [] - assert not list((tmp_path / "workspace" / "artifacts" / "uploads").glob("**/*")) - - -def test_add_configures_semantic_retrieval_in_same_filesystem_instance(tmp_path): - source = tmp_path / "semantic.md" - source.write_text("alpha semantic recall text", encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - assert filesystem.semantic_retrieval_channels() == () - - filesystem.add_file(source, "/documents") - - assert filesystem.semantic_retrieval_channels() == ("summary",) - results = filesystem.browse_semantic_files( - "/documents", - "semantic recall", - recursive=True, - page_size=5, - ) - assert [item["path"] for item in results["data"]] == ["/documents/semantic.md"] - - -def test_add_markdown_builds_pageindex_tree_from_copied_artifact(tmp_path, monkeypatch): - from pageindex import PageIndexClient - from pageindex.filesystem import PIFSCommandExecutor - - indexed_paths = [] - - def fake_index(self, file_path, mode="auto"): - indexed_paths.append(Path(file_path)) - doc_id = "doc_added_md" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "notes.md", - "doc_description": "summary", - "line_count": 3, - "structure": [ - { - "title": "Notes", - "node_id": "0001", - "line_num": 1, - "text": "# Notes\n\ncopied markdown body", - "nodes": [], - } - ], - } - write_pageindex_client_doc(self.workspace, doc_id, doc) - self.documents[doc_id] = doc - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "notes.md" - source.write_text("# Notes\n\ncopied markdown body", encoding="utf-8") - filesystem = make_filesystem(tmp_path / "workspace") - - info = filesystem.add_file(source, "/documents") - executor = PIFSCommandExecutor(filesystem) - structure = json.loads(executor.execute("cat /documents/notes.md --structure")) - entry = filesystem.store.get_file(info["file_ref"]) - - assert structure["data"]["document"]["available"] is True - assert structure["data"]["structure"][0]["title"] == "Notes" - assert indexed_paths == [Path(entry.storage_uri.removeprefix("file://"))] - assert indexed_paths[0].resolve() != source.resolve() - - -def test_add_failure_does_not_leave_visible_catalog_or_artifacts(tmp_path, monkeypatch): - source = tmp_path / "atomic.md" - source.write_text("atomic body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - filesystem.add_file(source, "/documents") - - assert filesystem.browse("/", recursive=True)["files"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_markdown_insert_failure_removes_pageindex_cache(tmp_path, monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = "doc_failed_add_md" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "failed.md", - "doc_description": "summary", - "line_count": 3, - "structure": [ - { - "title": "Failed", - "node_id": "0001", - "line_num": 1, - "text": "# Failed\n\nbody", - "nodes": [], - } - ], - } - write_pageindex_client_doc(self.workspace, doc_id, doc) - self.documents[doc_id] = doc - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "failed.md" - source.write_text("# Failed\n\nbody", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - filesystem.add_file(source, "/documents/reports") - - pageindex_workspace = workspace / "artifacts" / "pageindex_client" - assert not (pageindex_workspace / "doc_failed_add_md.json").exists() - meta_path = pageindex_workspace / "_meta.json" - if meta_path.exists(): - meta = json.loads(meta_path.read_text(encoding="utf-8")) - assert "doc_failed_add_md" not in meta - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert listing["folders"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_markdown_index_failure_removes_pageindex_cache_delta(tmp_path, monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = "doc_partial_before_raise" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "partial.md", - "doc_description": "summary", - "line_count": 3, - "structure": [{"title": "Partial", "node_id": "0001", "nodes": []}], - } - self.documents[doc_id] = doc - self._save_doc(doc_id) - raise RuntimeError("index failed after cache write") - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "partial.md" - source.write_text("# Partial\n\nbody", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - pageindex_workspace = workspace / "artifacts" / "pageindex_client" - - with pytest.raises(RuntimeError, match="requires PageIndex extraction"): - filesystem.add_file(source, "/documents/reports") - - assert not (pageindex_workspace / "doc_partial_before_raise.json").exists() - meta_path = pageindex_workspace / "_meta.json" - if meta_path.exists(): - meta = json.loads(meta_path.read_text(encoding="utf-8")) - assert "doc_partial_before_raise" not in meta - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert listing["folders"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_markdown_failure_preserves_unrelated_pageindex_cache(tmp_path, monkeypatch): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = "doc_failed_add_md" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": "failed.md", - "doc_description": "summary", - "line_count": 3, - "structure": [{"title": "Failed", "node_id": "0001", "nodes": []}], - } - self.documents[doc_id] = doc - self._save_doc(doc_id) - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - source = tmp_path / "failed.md" - source.write_text("# Failed\n\nbody", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - pageindex_workspace = workspace / "artifacts" / "pageindex_client" - write_pageindex_client_doc( - pageindex_workspace, - "doc_unrelated", - { - "id": "doc_unrelated", - "type": "md", - "path": str((tmp_path / "unrelated.md").resolve()), - "doc_name": "unrelated.md", - "doc_description": "summary", - "line_count": 1, - "structure": [{"title": "Unrelated", "node_id": "0001", "nodes": []}], - }, - ) - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - filesystem.add_file(source, "/documents") - - assert not (pageindex_workspace / "doc_failed_add_md.json").exists() - assert (pageindex_workspace / "doc_unrelated.json").exists() - meta = json.loads((pageindex_workspace / "_meta.json").read_text(encoding="utf-8")) - assert "doc_failed_add_md" not in meta - assert "doc_unrelated" in meta - - -def test_add_failure_after_summary_vector_rolls_back_catalog_and_vector( - tmp_path, monkeypatch -): - source = tmp_path / "post_vector.md" - source.write_text("post vector rollback body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_status_update(*args, **kwargs): - raise RuntimeError("metadata status update failed") - - monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) - - with pytest.raises(RuntimeError, match="metadata status update failed"): - filesystem.add_file(source, "/documents") - - assert filesystem.browse("/", recursive=True)["files"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_failure_removes_nested_folders_created_only_for_add(tmp_path, monkeypatch): - source = tmp_path / "nested.md" - source.write_text("nested rollback body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - - def fail_status_update(*args, **kwargs): - raise RuntimeError("metadata status update failed") - - monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) - - with pytest.raises(RuntimeError, match="metadata status update failed"): - filesystem.add_file(source, "/documents/reports") - - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert listing["folders"] == [] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - assert not list((workspace / "artifacts" / "uploads").glob("**/*")) - assert not list((workspace / "artifacts" / "text").glob("*.txt")) - assert not list((workspace / "artifacts" / "raw").glob("*.json")) - - -def test_add_failure_preserves_preexisting_parent_folder(tmp_path, monkeypatch): - source = tmp_path / "nested.md" - source.write_text("nested rollback body", encoding="utf-8") - workspace = tmp_path / "workspace" - filesystem = make_filesystem(workspace) - filesystem.create_folder("/documents") - - def fail_status_update(*args, **kwargs): - raise RuntimeError("metadata status update failed") - - monkeypatch.setattr(filesystem.store, "update_file_metadata_status", fail_status_update) - - with pytest.raises(RuntimeError, match="metadata status update failed"): - filesystem.add_file(source, "/documents/reports") - - listing = filesystem.browse("/", recursive=True) - assert listing["files"] == [] - assert [folder["path"] for folder in listing["folders"]] == ["/documents"] - assert filesystem.summary_projection_indexer.index.info()["document_count"] == 0 - - -def test_cli_add_uses_workspace_and_prints_added_file(monkeypatch, capsys, tmp_path): - from pageindex.filesystem import cli - - source = tmp_path / "cli.md" - source.write_text("cli body", encoding="utf-8") - calls = [] - - class FakeAddFileSystem: - def __init__(self, workspace, **_kwargs): - self.workspace = Path(workspace) - - def configure_existing_projection_retrieval(self): - return False - - def add_file(self, physical_path, virtual_target): - calls.append((self.workspace, physical_path, virtual_target)) - return { - "file_ref": "file_cli", - "path": "/documents/cli.md", - } - - monkeypatch.setattr(cli, "PageIndexFileSystem", FakeAddFileSystem) - - status = cli.main(["--workspace", str(tmp_path / "workspace"), "add", str(source), "/documents"]) - - assert status == 0 - assert calls == [(tmp_path / "workspace", str(source), "/documents")] - assert capsys.readouterr().out == ( - "added: /documents/cli.md\n" - "file_ref: file_cli\n" - ) diff --git a/tests/test_pifs_agent_stream.py b/tests/test_pifs_agent_stream.py index 5e1d0e684..244e8249b 100644 --- a/tests/test_pifs_agent_stream.py +++ b/tests/test_pifs_agent_stream.py @@ -1,10 +1,8 @@ -import ast import io import os import tempfile import threading import unittest -from pathlib import Path from unittest.mock import patch from types import SimpleNamespace @@ -30,21 +28,6 @@ from pageindex.filesystem import PageIndexFileSystem -def load_demo_agent_prompt() -> str: - demo_path = Path(__file__).resolve().parents[1] / "examples" / "pifs_demo.py" - module = ast.parse(demo_path.read_text(encoding="utf-8")) - for node in module.body: - if isinstance(node, ast.Assign): - names = [ - target.id - for target in node.targets - if isinstance(target, ast.Name) - ] - if "PIFS_DEMO_AGENT_PROMPT" in names and isinstance(node.value, ast.Constant): - return str(node.value.value) - raise AssertionError("PIFS_DEMO_AGENT_PROMPT not found") - - class StructuredAnswer(BaseModel): model_config = ConfigDict(extra="forbid") @@ -88,20 +71,25 @@ def test_tools_mode_does_not_print_model_text(self): observer = PIFSAgentStreamObserver("tools", stream_log=stream_log, output=output) observer.handle_event(self.raw_event("response.output_text.delta", "hidden from tools mode")) - observer.handle_event(self.raw_event("response.function_call_arguments.delta", '{"command":"ls /"}')) - observer.emit_tool_call("ls /") + observer.handle_event(self.raw_event("response.function_call_arguments.delta", '{"command":"tree / -L 1"}')) + observer.emit_tool_call("tree / -L 1") observer.emit_tool_result(ok=True, output='{"ok": true}', seconds=0.001) observer.finish() printed = output.getvalue() self.assertNotIn("hidden from tools mode", printed) self.assertIn("[llm -> pifs command]", printed) - self.assertIn("ls /", printed) + self.assertIn("tree / -L 1", printed) self.assertIn("[pifs -> llm result preview]", printed) self.assertIn('{"ok": true}', printed) - self.assertEqual(stream_log[0], {"kind": "tool_call", "command": "ls /"}) + self.assertEqual( + stream_log[0], {"kind": "tool_call", "command": "tree / -L 1"} + ) self.assertEqual(stream_log[1]["kind"], "tool_result") - self.assertEqual(stream_log[2], {"kind": "tool_args", "text": '{"command":"ls /"}'}) + self.assertEqual( + stream_log[2], + {"kind": "tool_args", "text": '{"command":"tree / -L 1"}'}, + ) def test_empty_tool_command_is_not_printed_or_logged(self): output = io.StringIO() @@ -258,7 +246,7 @@ def test_prompt_routes_topic_retrieval_through_browse_after_folder_exploration(s self.assertIn("verify the relevant facts with cat or grep", AGENT_TOOL_POLICY) self.assertIn("cat --structure", AGENT_TOOL_POLICY) self.assertIn("cat --page", AGENT_TOOL_POLICY) - self.assertIn("Use grep for a selected single file", AGENT_TOOL_POLICY) + self.assertIn("Use grep for a selected single file", AGENT_TOOL_POLICY) self.assertIn("Use grep only as a single-document lexical fallback", BASH_TOOL_DESCRIPTION) def test_prompt_allows_recursive_browse_for_low_signal_tree_labels(self): @@ -283,20 +271,9 @@ def test_default_agent_prompts_do_not_suggest_legacy_semantic_commands(self): ): self.assertNotIn(old_command, prompt_surface) - def test_demo_prompt_uses_browse_strategy_and_not_old_vector_commands(self): - demo_prompt = load_demo_agent_prompt() - - self.assertIn("Start with tree", demo_prompt) - self.assertIn('browse /documents "Federal Reserve supervision regulation"', demo_prompt) - self.assertIn('browse -R /documents "Federal Reserve supervision regulation"', demo_prompt) - self.assertIn("verify", demo_prompt) - self.assertIn("cat --structure", demo_prompt) - self.assertIn("Use grep only for one selected file", demo_prompt) - self.assertIn("Do not guess cat --page ranges from grep line numbers", demo_prompt) - self.assertNotIn("search-summary", demo_prompt) - def test_prompt_rejects_find_grep_as_exhaustive_search(self): - self.assertIn("Do not use find, recursive grep, folder grep, pipes", AGENT_TOOL_POLICY) + self.assertIn("folder-wide lexical searches", AGENT_TOOL_POLICY) + self.assertIn("Other shell commands, pipes", AGENT_TOOL_POLICY) self.assertIn("Only after that persistence protocol may you say the workspace lacks evidence", AGENT_TOOL_POLICY) def test_system_prompt_sets_workspace_identity_and_scope(self): @@ -304,6 +281,10 @@ def test_system_prompt_sets_workspace_identity_and_scope(self): self.assertIn("VectifyAI Team", AGENT_SYSTEM_PROMPT) self.assertIn("current PageIndex FileSystem\nworkspace", AGENT_SYSTEM_PROMPT) self.assertIn("unrelated to the current workspace", AGENT_SYSTEM_PROMPT) + self.assertIn("@field/value", AGENT_TOOL_POLICY) + self.assertIn("@field/value", BASH_TOOL_DESCRIPTION) + self.assertNotIn("@field=value", AGENT_TOOL_POLICY) + self.assertNotIn("@field=value", BASH_TOOL_DESCRIPTION) self.assertIn("do not answer it as\na general-purpose assistant", AGENT_SYSTEM_PROMPT) self.assertIn("workspace-related topic question", AGENT_SYSTEM_PROMPT) self.assertIn("clarify only after the persistence protocol", AGENT_SYSTEM_PROMPT) diff --git a/tests/test_pifs_cli.py b/tests/test_pifs_cli.py index d6f5b0a5d..7c2ae441a 100644 --- a/tests/test_pifs_cli.py +++ b/tests/test_pifs_cli.py @@ -37,16 +37,12 @@ def test_pifs_config_file_overrides_default_location(monkeypatch, tmp_path): class FakeFileSystem: - def __init__(self, workspace): + def __init__(self, workspace, **kwargs): self.workspace = Path(workspace) - self.projection_retrieval_configured = False + self.kwargs = kwargs - def configure_existing_projection_retrieval(self): - self.projection_retrieval_configured = True - return True - -def test_cli_workspace_does_not_eagerly_configure_projection_retrieval(monkeypatch, tmp_path): +def test_cli_workspace_does_not_eagerly_open_projection(monkeypatch, tmp_path): from pageindex.filesystem import cli workspace = tmp_path / "workspace" @@ -56,7 +52,7 @@ def test_cli_workspace_does_not_eagerly_configure_projection_retrieval(monkeypat filesystem = cli._filesystem_from_workspace(str(workspace)) assert filesystem.workspace == workspace - assert filesystem.projection_retrieval_configured is False + assert filesystem.kwargs.get("summary_projection_embedding_model") is None def test_cli_workspace_without_projection_index_does_not_require_sqlite_vec( @@ -81,7 +77,6 @@ def block_sqlite_vec(name, globals=None, locals=None, fromlist=(), level=0): filesystem = cli._filesystem_from_workspace(str(workspace)) assert filesystem.workspace == workspace - assert filesystem.semantic_retrieval_channels() == () def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): @@ -94,12 +89,11 @@ def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): json.dumps( { "workspace": str(workspace), - "embedding_provider": "openai", "embedding_model": "gemini-embedding-2-preview", "embedding_dimensions": 3072, "embedding_timeout": 12.5, "embedding_base_url": "https://example.invalid/openai/", - "embedding_api_key": "test-gemini-key", + "embedding_api_key": "config-key", } ), encoding="utf-8", @@ -111,6 +105,7 @@ def __init__(self, workspace, **kwargs): self.kwargs = kwargs monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") + monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") monkeypatch.setattr(cli, "PageIndexFileSystem", ConfiguredFileSystem) @@ -118,57 +113,49 @@ def __init__(self, workspace, **kwargs): assert filesystem.workspace == workspace assert filesystem.kwargs == { - "summary_projection_embedding_provider": "openai", "summary_projection_embedding_model": "gemini-embedding-2-preview", "summary_projection_embedding_dimensions": 3072, "summary_projection_embedding_timeout": 12.5, "summary_projection_embedding_base_url": "https://example.invalid/openai/", - "summary_projection_embedding_api_key": "test-gemini-key", + "summary_projection_embedding_api_key": "ignored-env-key", } assert os.environ["PIFS_EMBEDDING_API_KEY"] == "ignored-env-key" assert os.environ["PIFS_EMBEDDING_BASE_URL"] == "https://ignored.invalid/" -def test_browse_surfaces_projection_dimension_mismatch_lazily(tmp_path): +@pytest.mark.parametrize( + ("config_key", "pifs_key", "openai_key", "expected"), + [ + ("config-key", None, None, "config-key"), + ("config-key", "pifs-key", "openai-key", "pifs-key"), + ("config-key", None, "openai-key", "config-key"), + (None, None, "openai-key", "openai-key"), + ], +) +def test_cli_embedding_api_key_precedence( + config_key, pifs_key, openai_key, expected, monkeypatch, tmp_path +): from pageindex.filesystem import cli - from pageindex.filesystem.commands import PIFSCommandExecutor - from pageindex.filesystem.semantic_index import SemanticIndexRecord, SQLiteVecSemanticIndex - - workspace = tmp_path / "workspace" - index_dir = workspace / "artifacts" / "projection_indexes" - summary_index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - summary_index.reset( - dimension=3, - metadata={ - "channel": "summary", - "embedding_provider": "test", - "embedding_model": "fake", - "embedding_dimensions": 3, - }, - ) - summary_index.upsert_many( - [ - SemanticIndexRecord( - file_ref="file_a", - external_id="doc_a", - source_type="documents", - title="A", - text="summary", - vector=[1.0, 0.0, 0.0], - ) - ] - ) - filesystem = cli._filesystem_from_workspace(str(workspace)) + config_path = pifs_config_path(tmp_path) + config_path.parent.mkdir(parents=True, exist_ok=True) + config = {"workspace": str(tmp_path / "workspace")} + if config_key is not None: + config["embedding_api_key"] = config_key + config_path.write_text(json.dumps(config), encoding="utf-8") + if pifs_key is None: + monkeypatch.delenv("PIFS_EMBEDDING_API_KEY", raising=False) + else: + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", pifs_key) + if openai_key is None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + else: + monkeypatch.setenv("OPENAI_API_KEY", openai_key) + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) - assert filesystem.semantic_retrieval_channels() == () - result = json.loads(PIFSCommandExecutor(filesystem).execute('browse / "summary"')) + filesystem = cli._filesystem_from_workspace(str(tmp_path / "workspace")) - assert result["success"] is False - assert result["error"]["code"] == "invalid_command" - assert "summary projection index dimension mismatch" in result["error"]["message"] - assert "dimension 3" in result["error"]["message"] - assert "summary_projection_embedding_dimensions is 1024" in result["error"]["message"] + assert filesystem.kwargs["summary_projection_embedding_api_key"] == expected def test_cli_passthrough_invokes_pifs_command_executor(monkeypatch, capsys, tmp_path): @@ -190,13 +177,13 @@ def execute(self, command): monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) - status = cli.main(["--workspace", str(workspace), "ls", "/documents"]) + status = cli.main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) assert status == 0 - assert capsys.readouterr().out == "executed:ls /documents\n" + assert capsys.readouterr().out == "executed:tree /documents -L 1\n" assert len(executor_instances) == 1 assert executor_instances[0].filesystem.workspace == workspace - assert executor_instances[0].commands == ["ls /documents"] + assert executor_instances[0].commands == ["tree /documents -L 1"] def test_cli_passthrough_returns_nonzero_for_failed_json_envelope(monkeypatch, capsys, tmp_path): @@ -271,9 +258,12 @@ def set_metadata(self, target, metadata, *, clear=False): (workspace, "/documents/report.md", {"ticker": "AAPL"}, False) ] assert json.loads(capsys.readouterr().out) == { - "file_ref": "file_report", + "document_id": None, "metadata": {"ticker": "AAPL"}, + "metadata_status": {}, "path": "/documents/report.md", + "status": None, + "title": None, } @@ -286,7 +276,11 @@ def test_cli_setmeta_clear_uses_empty_object(monkeypatch, capsys, tmp_path): class FakeSetMetaFileSystem(FakeFileSystem): def set_metadata(self, target, metadata, *, clear=False): calls.append((self.workspace, target, metadata, clear)) - return {"file_ref": "file_report", "metadata": metadata} + return { + "path": "/documents/report.md", + "file_ref": "file_report", + "metadata": metadata, + } monkeypatch.setattr(cli, "PageIndexFileSystem", FakeSetMetaFileSystem) @@ -297,8 +291,12 @@ def set_metadata(self, target, metadata, *, clear=False): assert status == 0 assert calls == [(workspace, "/documents/report.md", {}, True)] assert json.loads(capsys.readouterr().out) == { - "file_ref": "file_report", + "document_id": None, "metadata": {}, + "metadata_status": {}, + "path": "/documents/report.md", + "status": None, + "title": None, } @@ -326,10 +324,10 @@ def execute(self, command): assert cli.main(["set", "workspace", str(workspace)]) == 0 capsys.readouterr() - status = cli.main(["ls", "/documents"]) + status = cli.main(["tree", "/documents", "-L", "1"]) assert status == 0 - assert capsys.readouterr().out == "executed:ls /documents\n" + assert capsys.readouterr().out == "executed:tree /documents -L 1\n" assert executor_instances[0].filesystem.workspace == workspace diff --git a/tests/test_pifs_core_alignment.py b/tests/test_pifs_core_alignment.py deleted file mode 100644 index 6c22fa92c..000000000 --- a/tests/test_pifs_core_alignment.py +++ /dev/null @@ -1,356 +0,0 @@ -import json -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace - -from tests.pifs_markdown_fixture import register_markdown - - -class BrowseBackend: - def __init__(self, document_ids, channels=("summary",), file_refs_by_document_id=None): - self.document_ids = list(document_ids) - self.channels = channels - self.file_refs_by_document_id = dict(file_refs_by_document_id or {}) - self.calls = [] - - def available_channels(self): - return self.channels - - def search_channel(self, channel, query, *, limit=10, filters=None): - self.calls.append((channel, query, limit, filters)) - file_ref_filter = set() - if isinstance(filters, dict): - raw_file_refs = filters.get("file_ref") or [] - if isinstance(raw_file_refs, str): - file_ref_filter = {raw_file_refs} - else: - file_ref_filter = {str(item) for item in raw_file_refs} - document_ids = self.document_ids - if file_ref_filter and self.file_refs_by_document_id: - document_ids = [ - document_id - for document_id in document_ids - if self.file_refs_by_document_id.get(document_id) in file_ref_filter - ] - return [ - SimpleNamespace( - document_id=document_id, - snippet=f"{channel} candidate {rank}: {query}", - score=1.0 - rank * 0.01, - sources=[{"channel": channel, "rank": rank}], - ) - for rank, document_id in enumerate(document_ids[:limit], 1) - ] - - -def _payload(output): - return json.loads(output) - - -def _register_file(filesystem, root, external_id, folder_path, *, title=None, text=None): - return register_markdown( - filesystem, - root, - external_id, - folder_path, - title=title or f"{external_id}.md", - text=text, - metadata={"department": "finance"}, - ) - - -class PIFSCoreAlignmentTest(unittest.TestCase): - def test_aligned_command_surface_and_json_errors(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file(filesystem, root, "doc_a", "/documents") - executor = PIFSCommandExecutor(filesystem) - - self.assertEqual( - executor.allowed_commands(), - {"browse", "cat", "grep", "ls", "stat", "tree"}, - ) - - for command in ( - "find /documents", - "grep -R alpha /documents", - "grep alpha /documents", - "stat --schema", - "stat --field department doc_a", - "stat doc_a doc_a", - "cat doc_a", - "cat doc_a --all", - "tree /documents | grep doc", - 'browse /documents "alpha" --space entity', - 'browse /documents "alpha" --json', - ): - result = _payload(executor.execute(command)) - self.assertFalse(result["success"]) - self.assertEqual(result["error"]["code"], "invalid_command") - self.assertEqual(result["next_steps"], []) - - def test_ls_is_tree_depth_one_alias(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file(filesystem, root, "doc_root", "/documents") - _register_file(filesystem, root, "doc_child", "/documents/team") - executor = PIFSCommandExecutor(filesystem) - calls = [] - original = executor._cmd_tree - - def record_tree(args): - calls.append(args) - return original(args) - - executor._cmd_tree = record_tree - - ls_result = _payload(executor.execute("ls /documents")) - tree_result = _payload(executor.execute("tree /documents -L 1")) - - self.assertEqual(calls[0], ["/documents", "-L", "1"]) - self.assertEqual(ls_result, tree_result) - self.assertTrue(ls_result["success"]) - self.assertEqual( - list(ls_result["data"]), - ["tree", "total_folders", "depth", "truncated"], - ) - self.assertEqual(ls_result["data"]["depth"], 1) - - def test_tree_defaults_to_chat_aligned_depth_ten_without_clamping_explicit_depth(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file(filesystem, root, "doc_root", "/documents") - executor = PIFSCommandExecutor(filesystem) - - default_tree = _payload(executor.execute("tree /documents")) - explicit_tree = _payload(executor.execute("tree /documents -L 99")) - - self.assertTrue(default_tree["success"]) - self.assertEqual(default_tree["data"]["depth"], 10) - self.assertTrue(explicit_tree["success"]) - self.assertEqual(explicit_tree["data"]["depth"], 99) - - def test_browse_summary_scoped_and_paginated(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_direct": _register_file(filesystem, root, "doc_direct", "/documents"), - "doc_deep": _register_file(filesystem, root, "doc_deep", "/documents/reports"), - "doc_other": _register_file(filesystem, root, "doc_other", "/other"), - } - backend = BrowseBackend( - ["doc_other", "doc_deep", "doc_direct"], - file_refs_by_document_id=refs, - ) - filesystem.semantic_retrieval_backend = backend - executor = PIFSCommandExecutor(filesystem) - - missing_query = _payload(executor.execute("browse /documents")) - self.assertFalse(missing_query["success"]) - self.assertIn("requires a query", missing_query["error"]["message"]) - - removed_space = _payload( - executor.execute('browse /documents "alpha" --space entity') - ) - self.assertFalse(removed_space["success"]) - self.assertIn("--space is removed", removed_space["error"]["message"]) - - direct = _payload(executor.execute('browse /documents "alpha"')) - self.assertTrue(direct["success"]) - self.assertEqual( - set(direct["data"]["documents"][0]), - {"path", "title", "summary", "metadata"}, - ) - self.assertEqual( - [doc["path"] for doc in direct["data"]["documents"]], - ["/documents/doc_direct.md"], - ) - self.assertEqual( - direct["data"]["scope"], - { - "folder": "/documents", - "recursive": False, - "query": "alpha", - "where": None, - "retrieval": "summary", - }, - ) - self.assertEqual(backend.calls[-1][0], "summary") - - recursive = _payload(executor.execute('browse -R /documents "alpha"')) - self.assertEqual( - [doc["path"] for doc in recursive["data"]["documents"]], - ["/documents/reports/doc_deep.md", "/documents/doc_direct.md"], - ) - self.assertTrue(recursive["data"]["scope"]["recursive"]) - - def test_stat_and_grep_are_single_document_only(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - _register_file( - filesystem, - root, - "doc_a", - "/documents", - title="report.md", - text="alpha evidence\nbeta evidence", - ) - executor = PIFSCommandExecutor(filesystem) - - stat = _payload(executor.execute("stat /documents/report.md")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["document_id"], "doc_a") - self.assertTrue(stat["data"]["document"]["file_ref"].startswith("file_")) - - grep = _payload(executor.execute("grep alpha /documents/report.md")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["document_id"], "doc_a") - self.assertEqual( - grep["data"]["matches"], - [{"line": 1, "text": "alpha evidence"}], - ) - - folder_grep = _payload(executor.execute("grep alpha /documents")) - self.assertFalse(folder_grep["success"]) - self.assertIn("not a folder", folder_grep["error"]["message"]) - - def test_cat_structure_and_page_use_aligned_json_shapes(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - class FakePageClient: - documents = { - "pi_doc": { - "doc_description": "Summary for report.md", - "pages": [{"page": 1, "content": "cached text"}], - } - } - - def index(self, file_path, mode="auto"): - return "pi_doc" - - def _ensure_doc_loaded(self, doc_id): - return None - - def get_document_structure(self, doc_id): - assert doc_id == "pi_doc" - return json.dumps( - [{"title": "Risk", "node_id": "0001", "text": "hidden", "nodes": []}] - ) - - def get_page_content(self, doc_id, pages): - assert doc_id == "pi_doc" - assert pages == "1-2" - return json.dumps( - [ - {"page": 1, "content": "page one evidence"}, - {"page": 2, "content": "page two evidence"}, - ] - ) - - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) - source = root / "report.md" - source.write_text("cached text", encoding="utf-8") - filesystem = PageIndexFileSystem(workspace=root / "workspace") - filesystem._pageindex_client = lambda: FakePageClient() - file_ref = filesystem.register_file( - storage_uri=source.as_uri(), - folder_path="/documents", - external_id="doc_pdf", - title="report.md", - content_type="text/markdown", - content="cached text", - ) - filesystem.store.update_pageindex_pointer( - file_ref, - pageindex_doc_id="pi_doc", - pageindex_tree_status="built", - ) - executor = PIFSCommandExecutor(filesystem) - - structure = _payload(executor.execute("cat doc_pdf --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["document_id"], "doc_pdf") - self.assertEqual( - structure["data"]["structure"], - [{"title": "Risk", "node_id": "0001", "nodes": []}], - ) - self.assertEqual(structure["data"]["pagination"], {"available": True}) - - page = _payload(executor.execute("cat doc_pdf --page 1-2")) - self.assertTrue(page["success"]) - self.assertEqual(page["data"]["requested_pages"], "1-2") - self.assertEqual( - [item["page"] for item in page["data"]["returned_pages"]], - [1, 2], - ) - self.assertEqual( - page["data"]["content"], - { - "text": "page one evidence\n\npage two evidence", - "available": True, - }, - ) - - def test_agent_policy_mentions_only_aligned_surface(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - from pageindex.filesystem.agent import build_pifs_agent_instructions - - with tempfile.TemporaryDirectory() as tmp: - filesystem = PageIndexFileSystem(workspace=Path(tmp) / "workspace") - executor = PIFSCommandExecutor(filesystem) - - instructions = build_pifs_agent_instructions(filesystem, executor=executor) - - for expected in ("tree", "browse", "stat", "cat", "grep"): - self.assertIn(expected, instructions) - for removed in ( - "find --", - "grep -R", - "stat --schema", - "stat --field", - "cat --all", - "cat --range", - ): - self.assertNotIn(removed, instructions) - self.assertIn( - "run cat --structure before the first cat --page", - instructions, - ) - self.assertIn("metadata-derived virtual axis", instructions) - self.assertIn("metadata-derived virtual value", instructions) - - def test_projection_surface_is_summary(self): - from pageindex.filesystem.core import ( - SEMANTIC_PROJECTION_INDEX_NAMES, - SEMANTIC_RETRIEVAL_CHANNELS, - ) - semantic_projection_source = Path("pageindex/filesystem/semantic_projection.py").read_text( - encoding="utf-8" - ) - - self.assertEqual(SEMANTIC_RETRIEVAL_CHANNELS, ("summary",)) - self.assertEqual(SEMANTIC_PROJECTION_INDEX_NAMES, {"summary": "summary"}) - self.assertIn('SUMMARY_INDEX_NAME = "summary"', semantic_projection_source) - self.assertNotIn("entity_vectors", semantic_projection_source) - self.assertNotIn("relation_vectors", semantic_projection_source) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_pifs_like_escape.py b/tests/test_pifs_like_escape.py index 304bad646..262306c70 100644 --- a/tests/test_pifs_like_escape.py +++ b/tests/test_pifs_like_escape.py @@ -42,20 +42,16 @@ def test_descendant_folder_filter_treats_underscore_literally(tmp_path): external_id="wildcard_neighbor", ) - recursive = filesystem.browse("/proj_1", recursive=True, limit=10) - folder_id = filesystem.folder_info("/proj_1")["folder_id"] - scoped_results = filesystem.search( - scope={"folder_id": folder_id, "recursive": True}, + recursive = filesystem.store.list_folder("/proj_1", recursive=True, limit=10) + scoped_results = filesystem.store.list_files( + scope={"folder_path": "/proj_1", "recursive": True}, limit=10, ) - ranked_folders = { - folder["path"]: folder - for folder in filesystem.find_folders("/", max_depth=1, limit=10) - } + ranked_folders = {folder["path"]: folder for folder in filesystem.store.find_folders("/", max_depth=1, limit=10)} assert {folder["path"] for folder in recursive["folders"]} == {"/proj_1/docs"} assert {file["external_id"] for file in recursive["files"]} == {"literal_underscore"} - assert {result.external_id for result in scoped_results} == {"literal_underscore"} + assert {result["external_id"] for result in scoped_results} == {"literal_underscore"} assert ranked_folders["/proj_1"]["matched_files"] == 1 assert ranked_folders["/projA1"]["matched_files"] == 1 assert filesystem.store.count_files_in_folder("/proj_1", recursive=True) == 1 @@ -99,14 +95,14 @@ def test_metadata_contains_treats_percent_and_underscore_literally(tmp_path): metadata={"status": "buildXalpha"}, ) - percent_results = filesystem.search( + percent_results = filesystem.store.list_files( metadata_filter={"status": {"$contains": "100% done"}}, limit=10, ) - underscore_results = filesystem.search( + underscore_results = filesystem.store.list_files( metadata_filter={"status": {"$contains": "build_alpha"}}, limit=10, ) - assert {result.external_id for result in percent_results} == {"literal_percent"} - assert {result.external_id for result in underscore_results} == {"literal_underscore"} + assert {result["external_id"] for result in percent_results} == {"literal_percent"} + assert {result["external_id"] for result in underscore_results} == {"literal_underscore"} diff --git a/tests/test_pifs_register_side_effects.py b/tests/test_pifs_register_side_effects.py deleted file mode 100644 index 6d2b8a65b..000000000 --- a/tests/test_pifs_register_side_effects.py +++ /dev/null @@ -1,77 +0,0 @@ -from pathlib import Path - -import pytest - -from tests.pifs_markdown_fixture import register_markdown - - -class RecordingSummaryIndexer: - def __init__(self): - self.upserted = [] - self.deleted = [] - - def upsert_summary(self, record): - self.upserted.append(dict(record)) - return {"status": "ready"} - - def delete_summary(self, file_ref): - self.deleted.append(file_ref) - - -def test_register_insert_failure_cleans_owned_artifacts_and_skips_projection( - tmp_path: Path, monkeypatch -): - from pageindex.filesystem import PageIndexFileSystem - - workspace = tmp_path / "workspace" - indexer = RecordingSummaryIndexer() - filesystem = PageIndexFileSystem(workspace=workspace) - filesystem.summary_projection_indexer = indexer - - def fail_insert(records): - raise RuntimeError("catalog insert failed") - - monkeypatch.setattr(filesystem.store, "insert_files", fail_insert) - - with pytest.raises(RuntimeError, match="catalog insert failed"): - register_markdown( - filesystem, - tmp_path, - "doc_insert_failure", - "/documents", - title="insert_failure.md", - text="Markdown content for registration.", - ) - - assert indexer.upserted == [] - assert list((workspace / "artifacts" / "raw").glob("*.json")) == [] - assert list((workspace / "artifacts" / "text").glob("*.txt")) == [] - - -def test_register_failure_after_catalog_insert_cleans_catalog_and_projection( - tmp_path: Path, monkeypatch -): - from pageindex.filesystem import PageIndexFileSystem - - workspace = tmp_path / "workspace" - indexer = RecordingSummaryIndexer() - filesystem = PageIndexFileSystem(workspace=workspace) - filesystem.summary_projection_indexer = indexer - - def fail_sync(record): - raise RuntimeError("raw sync failed") - - monkeypatch.setattr(filesystem, "_sync_owned_raw_artifact", fail_sync) - - with pytest.raises(RuntimeError, match="raw sync failed"): - register_markdown( - filesystem, - tmp_path, - "doc_sync_failure", - "/documents", - title="sync_failure.md", - text="Markdown content for registration.", - ) - - assert filesystem.search(None) == [] - assert indexer.deleted == [indexer.upserted[0]["file_ref"]] diff --git a/tests/test_pifs_scope_paths.py b/tests/test_pifs_scope_paths.py deleted file mode 100644 index 0e021908a..000000000 --- a/tests/test_pifs_scope_paths.py +++ /dev/null @@ -1,711 +0,0 @@ -import json -import tempfile -import unittest -from types import SimpleNamespace - -from tests.pifs_markdown_fixture import register_markdown - - -class BrowseBackend: - def __init__(self, document_ids, *, file_refs_by_document_id=None): - self.document_ids = list(document_ids) - self.file_refs_by_document_id = dict(file_refs_by_document_id or {}) - - def available_channels(self): - return ("summary",) - - def search_channel(self, channel, query, *, limit=10, filters=None): - file_ref_filter = set((filters or {}).get("file_ref") or []) - document_ids = self.document_ids - if file_ref_filter: - document_ids = [ - document_id - for document_id in document_ids - if self.file_refs_by_document_id.get(document_id) in file_ref_filter - ] - return [ - SimpleNamespace( - document_id=document_id, - snippet=f"{channel} candidate {rank}: {query}", - score=1.0 - rank * 0.01, - sources=[{"channel": channel, "rank": rank}], - ) - for rank, document_id in enumerate(document_ids[:limit], 1) - ] - - -def _payload(output): - return json.loads(output) - - -class PIFSScopePathTest(unittest.TestCase): - def test_tree_browse_and_stat_accept_metadata_scope_paths(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_aapl_2024": register_markdown( - filesystem, - root, - "doc_aapl_2024", - "/documents/sec-filings", - title="aapl-2024.md", - metadata={"year": 2024, "ticker": "AAPL", "doc_type": "10-K"}, - ), - "doc_aapl_2023": register_markdown( - filesystem, - root, - "doc_aapl_2023", - "/documents/sec-filings", - title="aapl-2023.md", - metadata={"year": 2023, "ticker": "AAPL", "doc_type": "10-K"}, - ), - "doc_msft_2024": register_markdown( - filesystem, - root, - "doc_msft_2024", - "/documents/sec-filings", - title="msft-2024.md", - metadata={"year": 2024, "ticker": "MSFT", "doc_type": "10-K"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_msft_2024", "doc_aapl_2024", "doc_aapl_2023"], - file_refs_by_document_id=refs, - ) - executor = PIFSCommandExecutor(filesystem) - - tree_root = _payload(executor.execute("tree /documents -L 1")) - self.assertTrue(tree_root["success"]) - - root_folders = tree_root["data"]["tree"]["folders"] - self.assertTrue(any(item["path"] == "/documents/sec-filings" for item in root_folders)) - self.assertEqual( - [item["name"] for item in root_folders if item["type"] == "metadata_axis"], - ["@doc_type", "@ticker", "@year"], - ) - - tree_year = _payload(executor.execute("tree /documents/@year")) - self.assertTrue(tree_year["success"]) - self.assertEqual( - [item["path"] for item in tree_year["data"]["tree"]["folders"]], - ["/documents/@year/2024", "/documents/@year/2023"], - ) - - browse = _payload( - executor.execute('browse /documents/@year/2024/@ticker/AAPL "risk factors"') - ) - self.assertTrue(browse["success"]) - self.assertEqual( - [item["path"] for item in browse["data"]["documents"]], - ["/documents/@year/2024/@ticker/AAPL/aapl-2024.md"], - ) - self.assertEqual(browse["data"]["scope"]["path"], "/documents/@year/2024/@ticker/AAPL") - self.assertEqual(browse["data"]["scope"]["folder_path"], "/documents") - self.assertEqual( - browse["data"]["scope"]["metadata_filter"], - {"year": "2024", "ticker": "AAPL"}, - ) - - locator = browse["data"]["documents"][0]["path"] - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - self.assertEqual(stat["data"]["document"]["title"], "aapl-2024.md") - - def test_metadata_scope_file_locators_round_trip_to_file_commands(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_3m_2018": register_markdown( - filesystem, - root, - "doc_3m_2018", - "/docs/filings", - title="3M_2018_10K.md", - text="cash flow evidence\nnet sales evidence", - metadata={"company": "3M"}, - ), - "doc_other": register_markdown( - filesystem, - root, - "doc_other", - "/docs/filings", - title="Other_2018_10K.md", - text="other cash flow", - metadata={"company": "Other"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_3m_2018", "doc_other"], - file_refs_by_document_id=refs, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - self.assertEqual( - [item["path"] for item in tree["data"]["tree"]["files"]], - ["/docs/@company/3M/3M_2018_10K.md"], - ) - - browse = _payload(executor.execute('browse /docs/@company/3M "cash flow"')) - self.assertTrue(browse["success"]) - locator = browse["data"]["documents"][0]["path"] - self.assertEqual(locator, "/docs/@company/3M/3M_2018_10K.md") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - self.assertEqual(stat["data"]["document"]["title"], "3M_2018_10K.md") - - structure = _payload(executor.execute(f"cat {locator} --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["path"], locator) - self.assertTrue(structure["data"]["structure"]) - - grep = _payload(executor.execute(f"grep cash {locator}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], locator) - self.assertEqual(grep["data"]["matches"][0]["line"], 1) - - scope_only = _payload(executor.execute("cat /docs/@company/3M --structure")) - self.assertFalse(scope_only["success"]) - self.assertIn("scope, not a file locator", scope_only["error"]["message"]) - self.assertIn("tree /docs/@company/3M", scope_only["error"]["message"]) - self.assertIn('browse /docs/@company/3M ""', scope_only["error"]["message"]) - - def test_tree_depth_does_not_bypass_file_pagination(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - for index in range(55): - register_markdown( - filesystem, - root, - f"doc_{index:02d}", - "/docs", - title=f"report-{index:02d}.md", - metadata={"company": "3M"}, - ) - executor = PIFSCommandExecutor(filesystem) - - first_page = _payload(executor.execute("tree /docs/@company/3M -L 100")) - self.assertTrue(first_page["success"]) - self.assertEqual(len(first_page["data"]["tree"]["files"]), 50) - self.assertEqual( - first_page["data"]["pagination"], - {"page": 1, "page_size": 50, "has_more": True, "next_page": 2}, - ) - - second_page = _payload(executor.execute("tree /docs/@company/3M -L 100 --page 2")) - self.assertTrue(second_page["success"]) - self.assertEqual(len(second_page["data"]["tree"]["files"]), 5) - self.assertEqual( - second_page["data"]["pagination"], - {"page": 2, "page_size": 50, "has_more": False, "next_page": None}, - ) - - def test_tree_file_pagination_sorts_full_scope_before_slicing(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - for index in range(55): - file_ref = register_markdown( - filesystem, - root, - f"doc_{index:02d}", - "/source", - title=f"source-{54 - index:02d}.md", - metadata={"company": "3M"}, - ) - filesystem.attach_file_to_folder( - file_ref, - "/docs", - metadata={"display_name": f"report-{index:02d}.md"}, - ) - executor = PIFSCommandExecutor(filesystem) - - first_page = _payload(executor.execute("tree /docs/@company/3M --page 1")) - second_page = _payload(executor.execute("tree /docs/@company/3M --page 2")) - - self.assertTrue(first_page["success"]) - self.assertTrue(second_page["success"]) - first_names = [item["name"] for item in first_page["data"]["tree"]["files"]] - second_names = [item["name"] for item in second_page["data"]["tree"]["files"]] - - self.assertEqual(first_names, [f"report-{index:02d}.md" for index in range(50)]) - self.assertEqual(second_names, [f"report-{index:02d}.md" for index in range(50, 55)]) - self.assertEqual(len(set(first_names + second_names)), 55) - - def test_duplicate_file_leaves_in_scope_return_disambiguated_locators(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_a": register_markdown( - filesystem, - root, - "doc_a", - "/docs/a", - title="report.md", - metadata={"company": "3M"}, - ), - "doc_b": register_markdown( - filesystem, - root, - "doc_b", - "/docs/b", - title="report.md", - metadata={"company": "3M"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_a", "doc_b"], - file_refs_by_document_id=refs, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - paths = [item["path"] for item in tree["data"]["tree"]["files"]] - self.assertEqual(len(paths), 2) - self.assertEqual(len(set(paths)), 2) - - for path in paths: - stat = _payload(executor.execute(f"stat {path}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], path) - - browse = _payload(executor.execute('browse /docs/@company/3M "report"')) - self.assertTrue(browse["success"]) - browse_paths = [item["path"] for item in browse["data"]["documents"]] - self.assertEqual(len(browse_paths), 2) - self.assertEqual(len(set(browse_paths)), 2) - - def test_disambiguated_file_leaf_does_not_collide_with_real_leaf(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - first_ref = register_markdown( - filesystem, - root, - "doc_first", - "/docs/a", - title="x.md", - text="first collision evidence", - metadata={"company": "3M"}, - ) - register_markdown( - filesystem, - root, - "doc_second", - "/docs/b", - title="x.md", - text="second collision evidence", - metadata={"company": "3M"}, - ) - register_markdown( - filesystem, - root, - "doc_real_suffix", - "/docs/c", - title=f"x.md~{first_ref}", - text="real suffix collision evidence", - metadata={"company": "3M"}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - paths = [item["path"] for item in tree["data"]["tree"]["files"]] - self.assertEqual(len(paths), 3) - self.assertEqual(len(set(paths)), 3) - - for path in paths: - stat = _payload(executor.execute(f"stat {path}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], path) - - grep = _payload(executor.execute(f"grep evidence {path}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], path) - - def test_metadata_scope_file_leaf_can_match_metadata_axis_name(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - register_markdown( - filesystem, - root, - "doc_axis_leaf", - "/docs", - title="@ticker", - text="axis leaf evidence", - metadata={"company": "3M", "ticker": "MMM"}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - locator = tree["data"]["tree"]["files"][0]["path"] - self.assertEqual(locator, "/docs/@company/3M/%40ticker") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - - structure = _payload(executor.execute(f"cat {locator} --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["path"], locator) - - grep = _payload(executor.execute(f"grep evidence {locator}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], locator) - self.assertEqual(grep["data"]["matches"][0]["line"], 1) - - def test_metadata_scope_file_leaf_with_slash_round_trips(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - file_ref = register_markdown( - filesystem, - root, - "doc_slash_leaf", - "/source", - title="source.md", - text="slash leaf evidence", - metadata={"company": "3M"}, - ) - filesystem.attach_file_to_folder( - file_ref, - "/docs", - metadata={"display_name": "2024/report.md"}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /docs/@company/3M")) - self.assertTrue(tree["success"]) - locator = tree["data"]["tree"]["files"][0]["path"] - self.assertEqual(locator, "/docs/@company/3M/2024%2Freport.md") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - - structure = _payload(executor.execute(f"cat {locator} --structure")) - self.assertTrue(structure["success"]) - self.assertEqual(structure["data"]["document"]["path"], locator) - - grep = _payload(executor.execute(f"grep evidence {locator}")) - self.assertTrue(grep["success"]) - self.assertEqual(grep["data"]["document"]["path"], locator) - self.assertEqual(grep["data"]["matches"][0]["line"], 1) - - def test_scope_paths_reject_duplicate_and_unknown_metadata_fields(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - register_markdown( - filesystem, - root, - "doc_aapl_2024", - "/documents", - metadata={"year": 2024, "ticker": "AAPL"}, - ) - executor = PIFSCommandExecutor(filesystem) - - duplicate = _payload(executor.execute("tree /documents/@ticker/AAPL/@ticker/MSFT")) - self.assertFalse(duplicate["success"]) - self.assertIn("can appear only once", duplicate["error"]["message"]) - - unknown = _payload(executor.execute("tree /documents/@sector")) - self.assertFalse(unknown["success"]) - self.assertIn("Unknown metadata axis", unknown["error"]["message"]) - - def test_tree_metadata_value_pagination_uses_page_flag(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - for index in range(55): - ticker = f"T{index:02d}" - register_markdown( - filesystem, - root, - f"doc_{ticker}", - "/documents", - title=f"{ticker}.md", - metadata={"ticker": ticker}, - ) - - executor = PIFSCommandExecutor(filesystem) - - first_page = _payload(executor.execute("tree /documents/@ticker")) - self.assertTrue(first_page["success"]) - self.assertEqual(len(first_page["data"]["tree"]["folders"]), 50) - self.assertEqual( - first_page["data"]["pagination"], - {"page": 1, "page_size": 50, "has_more": True, "next_page": 2}, - ) - self.assertEqual( - [item["value"] for item in first_page["data"]["tree"]["folders"][:3]], - ["T00", "T01", "T02"], - ) - - second_page = _payload(executor.execute("tree /documents/@ticker --page 2")) - self.assertTrue(second_page["success"]) - self.assertEqual( - [item["value"] for item in second_page["data"]["tree"]["folders"]], - ["T50", "T51", "T52", "T53", "T54"], - ) - self.assertEqual( - second_page["data"]["pagination"], - {"page": 2, "page_size": 50, "has_more": False, "next_page": None}, - ) - - def test_tree_child_paths_keep_active_metadata_scope(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - register_markdown( - filesystem, - root, - "doc_sec_2024", - "/documents/sec", - title="sec-2024.md", - metadata={"year": 2024, "ticker": "AAPL"}, - ) - register_markdown( - filesystem, - root, - "doc_sec_2023", - "/documents/sec", - title="sec-2023.md", - metadata={"year": 2023, "ticker": "AAPL"}, - ) - register_markdown( - filesystem, - root, - "doc_other_2024", - "/documents/other", - title="other-2024.md", - metadata={"year": 2024, "ticker": "MSFT"}, - ) - - executor = PIFSCommandExecutor(filesystem) - - tree = _payload(executor.execute("tree /documents/@year/2024 -L 1")) - self.assertTrue(tree["success"]) - sec_node = next( - item - for item in tree["data"]["tree"]["folders"] - if item["name"] == "sec" - ) - self.assertEqual(sec_node["path"], "/documents/sec/@year/2024") - self.assertEqual(sec_node["file_count"], 1) - - scope_stat = _payload(executor.execute("stat /documents/sec/@year/2024")) - self.assertFalse(scope_stat["success"]) - self.assertIn("scope, not a file locator", scope_stat["error"]["message"]) - - scoped_tree = _payload(executor.execute("tree /documents/sec/@year/2024")) - self.assertTrue(scoped_tree["success"]) - locator = scoped_tree["data"]["tree"]["files"][0]["path"] - self.assertEqual(locator, "/documents/sec/@year/2024/sec-2024.md") - - stat = _payload(executor.execute(f"stat {locator}")) - self.assertTrue(stat["success"]) - self.assertEqual(stat["data"]["document"]["path"], locator) - - def test_core_browse_semantic_files_accepts_metadata_scoped_path(self): - from pageindex.filesystem import PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - refs = { - "doc_aapl": register_markdown( - filesystem, - root, - "doc_aapl", - "/documents/sec", - title="aapl.md", - metadata={"ticker": "AAPL"}, - ), - "doc_msft": register_markdown( - filesystem, - root, - "doc_msft_direct", - "/documents", - title="msft-direct.md", - metadata={"ticker": "MSFT"}, - ), - "doc_msft_descendant": register_markdown( - filesystem, - root, - "doc_msft_descendant", - "/documents/sec", - title="msft-descendant.md", - metadata={"ticker": "MSFT"}, - ), - } - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_msft_direct", "doc_msft_descendant", "doc_aapl"], - file_refs_by_document_id=refs, - ) - - payload = filesystem.browse_semantic_files( - "/documents/@ticker/AAPL", - "query", - recursive=False, - ) - - self.assertEqual( - [item["document_id"] for item in payload["data"]], - ["doc_aapl"], - ) - self.assertEqual(payload["scope"], "/documents/@ticker/AAPL") - - def test_core_browse_semantic_files_prefers_in_scope_physical_membership(self): - from pageindex.filesystem import PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - file_ref = register_markdown( - filesystem, - root, - "doc_shared", - "/adocs", - title="shared.md", - metadata={"ticker": "AAPL"}, - ) - filesystem.store.attach_file_to_folder(file_ref, "/zdocs") - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_shared"], - file_refs_by_document_id={"doc_shared": file_ref}, - ) - - payload = filesystem.browse_semantic_files( - "/zdocs/@ticker/AAPL", - "query", - recursive=False, - ) - - self.assertEqual([item["document_id"] for item in payload["data"]], ["doc_shared"]) - self.assertEqual(payload["data"][0]["folder_path"], "/zdocs") - self.assertTrue(payload["data"][0]["path"].startswith("/zdocs/")) - - def test_one_document_is_reachable_through_multiple_metadata_virtual_paths(self): - from pageindex.filesystem import PIFSCommandExecutor, PageIndexFileSystem - - with tempfile.TemporaryDirectory() as tmp: - from pathlib import Path - - root = Path(tmp) - filesystem = PageIndexFileSystem(workspace=root / "workspace") - file_ref = register_markdown( - filesystem, - root, - "doc_contract", - "/documents/contracts", - title="apple-renewal.md", - metadata={"vendor": "Apple", "year": 2024, "doc_type": "contract"}, - ) - filesystem.semantic_retrieval_backend = BrowseBackend( - ["doc_contract"], - file_refs_by_document_id={"doc_contract": file_ref}, - ) - executor = PIFSCommandExecutor(filesystem) - - tree_root = _payload(executor.execute("tree /documents -L 1")) - self.assertTrue(tree_root["success"]) - root_nodes = tree_root["data"]["tree"]["folders"] - self.assertIn( - "/documents/@vendor", - [item["path"] for item in root_nodes if item["type"] == "metadata_axis"], - ) - - tree_vendor = _payload(executor.execute("tree /documents/@vendor")) - self.assertTrue(tree_vendor["success"]) - self.assertEqual( - [ - item["type"] - for item in tree_vendor["data"]["tree"]["folders"] - if item["path"] == "/documents/@vendor/Apple" - ], - ["metadata_value"], - ) - - vendor = _payload( - executor.execute('browse /documents/@vendor/Apple "renewal contract"') - ) - year = _payload(executor.execute('browse /documents/@year/2024 "renewal contract"')) - - self.assertTrue(vendor["success"]) - self.assertTrue(year["success"]) - self.assertEqual( - [item["path"] for item in vendor["data"]["documents"]], - ["/documents/@vendor/Apple/apple-renewal.md"], - ) - self.assertEqual( - [item["path"] for item in year["data"]["documents"]], - ["/documents/@year/2024/apple-renewal.md"], - ) - self.assertEqual( - vendor["data"]["scope"]["metadata_filter"], - {"vendor": "Apple"}, - ) - self.assertEqual( - year["data"]["scope"]["metadata_filter"], - {"year": "2024"}, - ) diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py new file mode 100644 index 000000000..3708f67b2 --- /dev/null +++ b/tests/test_pifs_simplification_acceptance.py @@ -0,0 +1,2839 @@ +import json +import hashlib +import sqlite3 +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +class FakeOpenAI: + def __init__(self, **_kwargs): + self.embeddings = self + + def create(self, *, model, input, dimensions): + assert model == "test-embedding" + assert dimensions == 3 + + def vector(text): + lowered = str(text).lower() + if "alpha" in lowered: + return [1.0, 0.0, 0.0] + if "beta" in lowered: + return [0.0, 1.0, 0.0] + return [0.0, 0.0, 1.0] + + return SimpleNamespace( + data=[ + SimpleNamespace(index=index, embedding=vector(text)) + for index, text in enumerate(input) + ] + ) + + +def install_network_fakes(monkeypatch): + import openai + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + path = Path(file_path) + text = ( + path.read_text(encoding="utf-8") + if path.suffix.lower() in {".md", ".markdown"} + else "pdf alpha evidence" + ) + path_digest = hashlib.sha256(str(path.resolve()).encode("utf-8")).hexdigest() + document_id = f"pageindex_{path_digest[:16]}" + document = { + "id": document_id, + "type": "pdf" if path.suffix.lower() == ".pdf" else "md", + "path": str(path.resolve()), + "doc_name": path.name, + "doc_description": f"Summary for {path.name}: {text}", + "line_count": len(text.splitlines()), + "structure": [ + { + "title": path.stem, + "node_id": "0001", + "line_num": 1, + "text": text, + "nodes": [], + } + ], + "pages": [{"page": 1, "content": text}], + } + self.documents[document_id] = document + if self.workspace: + self._save_doc(document_id) + return document_id + + monkeypatch.setattr(openai, "OpenAI", FakeOpenAI) + monkeypatch.setattr(PageIndexClient, "index", fake_index) + + +def write_embedding_config(tmp_path, monkeypatch): + config = tmp_path / "pifs.json" + config.write_text( + json.dumps( + { + "embedding_base_url": "https://EXAMPLE.invalid/v1/", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + "embedding_timeout": 12, + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("PIFS_CONFIG_FILE", str(config)) + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "runtime-secret") + return config + + +def logical_tables(connection): + return { + row[1] + for row in connection.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} and not row[1].startswith("sqlite_") + and not row[1].startswith("semantic_index_vec_") + } + + +def workspace_state(workspace): + workspace = Path(workspace) + if not workspace.exists(): + return {} + return { + path.relative_to(workspace).as_posix(): ( + f"symlink:{path.readlink()}" + if path.is_symlink() + else "directory" + if path.is_dir() + else hashlib.sha256(path.read_bytes()).hexdigest() + ) + for path in sorted(workspace.rglob("*")) + } + + +def create_projection_v2(workspace): + from pageindex.filesystem.semantic_projection import ( + SummaryEmbeddingProfile, + SummaryProjection, + ) + + projection_dir = Path(workspace) / "artifacts" / "projection_indexes" + SummaryProjection( + projection_dir, + profile=SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="test-embedding", + dimensions=3, + api_key="runtime-only", + ), + create=True, + ) + return projection_dir + + +def connect_summary_database(path): + import sqlite_vec + + connection = sqlite3.connect(path) + connection.enable_load_extension(True) + sqlite_vec.load(connection) + connection.enable_load_extension(False) + return connection + + +def insert_projection_document(summary_path, *, file_ref="orphan_projection_ref"): + import sqlite_vec + + with connect_summary_database(summary_path) as connection: + connection.execute( + """ + INSERT INTO semantic_index_docs( + rowid, file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + ) VALUES (1, ?, 'orphan-doc', 'markdown', 'orphan.md', + 'orphan-hash', 14, '{}') + """, + (file_ref,), + ) + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (1, 'markdown', ?)", + (sqlite_vec.serialize_float32([1.0, 0.0, 0.0]),), + ) + + +def delete_projection_document(summary_path, file_ref): + with connect_summary_database(summary_path) as connection: + row = connection.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchone() + assert row is not None + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", + (row[0],), + ) + connection.execute( + "DELETE FROM semantic_index_docs WHERE rowid = ?", + (row[0],), + ) + + +def catalog_file_count(workspace): + with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: + return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] + + +def registration_logical_state(workspace): + workspace = Path(workspace) + catalog_path = workspace / "filesystem.sqlite" + projection_dir = workspace / "artifacts" / "projection_indexes" + summary_path = projection_dir / "summary.sqlite" + cache_path = projection_dir / "embedding_cache.sqlite" + + with sqlite3.connect(catalog_path) as connection: + catalog = { + "files": connection.execute( + """ + SELECT file_ref, external_id, storage_uri, title, descriptor, + content_type, source_type, fingerprint, text_artifact_path, + raw_artifact_path, pageindex_doc_id, pageindex_tree_status, + metadata_json, metadata_status_json, deleted_at + FROM files + ORDER BY file_ref + """ + ).fetchall(), + "folders": connection.execute( + """ + SELECT folder_id, parent_id, name, path, description, kind, metadata_json + FROM folders + ORDER BY path + """ + ).fetchall(), + "metadata_fields": connection.execute( + """ + SELECT field_id, name, description, source + FROM metadata_fields + ORDER BY name + """ + ).fetchall(), + "metadata_values": connection.execute( + """ + SELECT file_ref, field_id, value_text + FROM metadata_values + ORDER BY file_ref, field_id, value_text + """ + ).fetchall(), + } + + summary = {"docs": [], "vec": []} + if summary_path.is_file(): + with connect_summary_database(summary_path) as connection: + summary = { + "docs": connection.execute( + """ + SELECT rowid, file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + FROM semantic_index_docs + ORDER BY rowid + """ + ).fetchall(), + "vec": connection.execute( + """ + SELECT rowid, source_type, hex(embedding) + FROM semantic_index_vec + ORDER BY rowid + """ + ).fetchall(), + } + + cache = [] + if cache_path.is_file(): + with sqlite3.connect(cache_path) as connection: + cache = connection.execute( + """ + SELECT base_url, model, dimensions, text_hash, hex(vector_blob) + FROM embedding_cache + ORDER BY base_url, model, dimensions, text_hash + """ + ).fetchall() + + artifacts = {} + for artifact_kind in ("text", "raw"): + artifact_dir = workspace / "artifacts" / artifact_kind + if artifact_dir.is_dir(): + artifacts[artifact_kind] = { + path.relative_to(artifact_dir).as_posix(): hashlib.sha256( + path.read_bytes() + ).hexdigest() + for path in sorted(artifact_dir.rglob("*")) + if path.is_file() + } + else: + artifacts[artifact_kind] = {} + + pageindex_dir = workspace / "artifacts" / "pageindex_client" + meta_path = pageindex_dir / "_meta.json" + artifacts["pageindex_client"] = { + "meta": json.loads(meta_path.read_text(encoding="utf-8")) + if meta_path.is_file() + else {}, + "documents": { + path.name: { + "json": json.loads(path.read_text(encoding="utf-8")), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for path in sorted(pageindex_dir.glob("*.json")) + if path.name != "_meta.json" + }, + } + + return { + "catalog": catalog, + "summary": summary, + "cache": cache, + "artifacts": artifacts, + } + + +def open_test_filesystem(workspace): + from pageindex.filesystem import PageIndexFileSystem + + return PageIndexFileSystem( + workspace, + summary_projection_embedding_base_url="https://example.invalid/v1", + summary_projection_embedding_model="test-embedding", + summary_projection_embedding_dimensions=3, + summary_projection_embedding_api_key="runtime-secret", + ) + + +def nested_path_with_length(root, target_length): + path = Path(root) + while len(str(path)) < target_length: + component_length = min(200, target_length - len(str(path)) - 1) + path /= "x" * component_length + assert len(str(path)) == target_length + return path + + +def test_cli_rejects_removed_ls_command_with_structured_error(tmp_path, capsys): + from pageindex.filesystem.cli import main + + status = main(["--workspace", str(tmp_path / "workspace"), "ls", "/"]) + + payload = json.loads(capsys.readouterr().out) + assert status == 2 + assert payload == { + "success": False, + "error": {"code": "invalid_command", "message": "Unsupported command: ls"}, + "next_steps": [], + } + + +def test_fresh_cli_workspace_uses_the_five_table_catalog_schema_v2(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + version = connection.execute("PRAGMA user_version").fetchone()[0] + tables = { + row[0] + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + + assert version == 2 + assert tables == { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", + } + + +def test_cli_rejects_legacy_catalog_without_mutating_it(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + workspace.mkdir() + database = workspace / "filesystem.sqlite" + with sqlite3.connect(database) as connection: + connection.execute("CREATE TABLE legacy_sentinel(value TEXT NOT NULL)") + connection.execute("INSERT INTO legacy_sentinel(value) VALUES ('preserve me')") + connection.execute("PRAGMA user_version = 1") + before = hashlib.sha256(database.read_bytes()).hexdigest() + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert hashlib.sha256(database.read_bytes()).hexdigest() == before + + +@pytest.mark.parametrize( + "mutation", + ["extra_column", "missing_index", "missing_primary_and_foreign_keys"], +) +def test_cli_rejects_pseudo_v2_catalog_without_mutating_workspace( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + database = workspace / "filesystem.sqlite" + with sqlite3.connect(database) as connection: + if mutation == "extra_column": + connection.execute("ALTER TABLE files ADD COLUMN legacy_provider TEXT") + elif mutation == "missing_index": + connection.execute("DROP INDEX idx_files_external_id") + else: + connection.executescript( + """ + ALTER TABLE file_folders RENAME TO legacy_file_folders; + CREATE TABLE file_folders ( + file_ref TEXT NOT NULL, + folder_id TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_file_folders; + CREATE INDEX idx_file_folders_folder ON file_folders(folder_id); + """ + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_rejects_partial_legacy_projection_without_creating_summary( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = workspace / "artifacts" / "projection_indexes" + projection_dir.mkdir(parents=True) + cache_path = projection_dir / "embedding_cache.sqlite" + with sqlite3.connect(cache_path) as connection: + connection.execute( + "CREATE TABLE embedding_cache(provider TEXT, model TEXT, text_hash TEXT)" + ) + connection.execute("PRAGMA user_version = 1") + before = hashlib.sha256(cache_path.read_bytes()).hexdigest() + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert hashlib.sha256(cache_path.read_bytes()).hexdigest() == before + assert not (projection_dir / "summary.sqlite").exists() + assert catalog_file_count(workspace) == 0 + + +def test_cli_preflights_partial_projection_before_creating_catalog(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + projection_dir = workspace / "artifacts" / "projection_indexes" + projection_dir.mkdir(parents=True) + cache_path = projection_dir / "embedding_cache.sqlite" + with sqlite3.connect(cache_path) as connection: + connection.execute( + "CREATE TABLE embedding_cache(provider TEXT, model TEXT, text_hash TEXT)" + ) + connection.execute("PRAGMA user_version = 1") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert not (workspace / "filesystem.sqlite").exists() + + +@pytest.mark.parametrize("catalog_state", ["missing", "zero_byte"]) +def test_cli_rejects_projection_pair_without_valid_catalog_before_mutating_workspace( + catalog_state, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + create_projection_v2(workspace) + if catalog_state == "zero_byte": + (workspace / "filesystem.sqlite").write_bytes(b"") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + if catalog_state == "missing": + assert not (workspace / "filesystem.sqlite").exists() + else: + assert (workspace / "filesystem.sqlite").stat().st_size == 0 + + +@pytest.mark.parametrize( + "projection_state", + ["both_broken", "broken_summary_only", "broken_summary_with_valid_cache"], +) +def test_cli_rejects_broken_projection_symlinks_without_mutating_workspace( + projection_state, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = workspace / "artifacts" / "projection_indexes" + if projection_state == "broken_summary_with_valid_cache": + create_projection_v2(workspace) + (projection_dir / "summary.sqlite").unlink() + else: + projection_dir.mkdir(parents=True) + (projection_dir / "summary.sqlite").symlink_to("missing-summary.sqlite") + if projection_state == "both_broken": + (projection_dir / "embedding_cache.sqlite").symlink_to( + "missing-cache.sqlite" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_rejects_existing_zero_byte_catalog_without_projection_or_mutation( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + workspace.mkdir() + catalog = workspace / "filesystem.sqlite" + catalog.write_bytes(b"") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert catalog.stat().st_size == 0 + + +@pytest.mark.parametrize("mutation", ["orphan_projection", "missing_root"]) +def test_cli_rejects_inconsistent_catalog_projection_relationships_without_mutation( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + insert_projection_document(projection_dir / "summary.sqlite") + if mutation == "missing_root": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute("DELETE FROM folders WHERE path = '/'") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + ("catalog_state", "projection_state", "should_fail"), + [ + ("active", "projected", False), + ("active", "missing", True), + ("deleted", "missing", False), + ("deleted", "projected", True), + ], +) +def test_runtime_requires_complete_projection_for_every_active_catalog_file( + catalog_state, projection_state, should_fail, tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha consistency evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + if catalog_state == "deleted": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE files SET deleted_at = '2026-01-02 03:04:05' " + "WHERE file_ref = ?", + (file_ref,), + ) + if projection_state == "missing": + delete_projection_document( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite", + file_ref, + ) + before = workspace_state(workspace) + + if should_fail: + with pytest.raises(RuntimeError, match="migrate_pifs_workspace.py"): + open_test_filesystem(workspace) + else: + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize(("catalog_state", "should_fail"), [("active", True), ("deleted", False)]) +def test_runtime_requires_projection_pair_when_catalog_only_has_active_files( + catalog_state, should_fail, tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha catalog-only evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + if catalog_state == "deleted": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE files SET deleted_at = '2026-01-02 03:04:05' " + "WHERE file_ref = ?", + (file_ref,), + ) + projection_dir = workspace / "artifacts" / "projection_indexes" + (projection_dir / "summary.sqlite").unlink() + (projection_dir / "embedding_cache.sqlite").unlink() + before = workspace_state(workspace) + + if should_fail: + with pytest.raises(RuntimeError, match="migrate_pifs_workspace.py"): + open_test_filesystem(workspace) + else: + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "mutation", + ["missing_vector", "extra_vector", "source_type_mismatch", "wrong_blob_length"], +) +def test_runtime_rejects_projection_document_vector_mismatches_without_mutation( + mutation, tmp_path, monkeypatch +): + import sqlite_vec + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha vector consistency evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + summary_path = workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + with connect_summary_database(summary_path) as connection: + row = connection.execute( + "SELECT rowid, source_type, embedding FROM semantic_index_vec" + ).fetchone() + assert row is not None + if mutation == "missing_vector": + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", (row[0],) + ) + elif mutation == "extra_vector": + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (999, 'markdown', ?)", + (sqlite_vec.serialize_float32([0.0, 0.0, 1.0]),), + ) + elif mutation == "source_type_mismatch": + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", (row[0],) + ) + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (?, 'pdf', ?)", + (row[0], row[2]), + ) + else: + connection.execute( + "UPDATE semantic_index_vec_vector_chunks00 SET vectors = zeroblob(4)" + ) + before = workspace_state(workspace) + + with pytest.raises(RuntimeError, match="migrate this workspace"): + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "mutation", + [ + "summary_extra_column", + "summary_missing_primary_and_unique", + "summary_missing_index", + "summary_extra_config_key", + "summary_vec_dimension_mismatch", + "cache_extra_column", + "cache_missing_primary_key", + ], +) +def test_cli_rejects_pseudo_v2_projection_without_mutating_workspace( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + projection_dir = create_projection_v2(workspace) + summary_path = projection_dir / "summary.sqlite" + cache_path = projection_dir / "embedding_cache.sqlite" + if mutation.startswith("summary_"): + with sqlite3.connect(summary_path) as connection: + if mutation == "summary_extra_column": + connection.execute( + "ALTER TABLE semantic_index_docs ADD COLUMN legacy_provider TEXT" + ) + elif mutation == "summary_missing_primary_and_unique": + connection.executescript( + """ + DROP INDEX idx_semantic_index_docs_external_id; + DROP INDEX idx_semantic_index_docs_source_type; + ALTER TABLE semantic_index_docs RENAME TO legacy_semantic_index_docs; + CREATE TABLE semantic_index_docs ( + rowid INTEGER, + file_ref TEXT NOT NULL, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_semantic_index_docs; + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + elif mutation == "summary_missing_index": + connection.execute("DROP INDEX idx_semantic_index_docs_external_id") + elif mutation == "summary_extra_config_key": + connection.execute( + "INSERT INTO semantic_index_config(key, value) " + "VALUES ('legacy_provider', 'openai')" + ) + else: + connection.execute( + "UPDATE semantic_index_config SET value = '4' WHERE key = 'dimension'" + ) + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + ( + json.dumps( + { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 4, + } + ), + ), + ) + else: + with sqlite3.connect(cache_path) as connection: + if mutation == "cache_extra_column": + connection.execute( + "ALTER TABLE embedding_cache ADD COLUMN provider TEXT" + ) + else: + connection.executescript( + """ + ALTER TABLE embedding_cache RENAME TO legacy_embedding_cache; + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_embedding_cache; + """ + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert not (workspace / "filesystem.sqlite").exists() + + +def test_cli_rejects_noncanonical_vec0_distance_without_mutating_workspace( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + with connect_summary_database(projection_dir / "summary.sqlite") as connection: + connection.execute("DROP TABLE semantic_index_vec") + connection.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + "vec0(source_type TEXT partition key, " + "embedding float[3] distance_metric=cosine)" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_accepts_canonical_vec0_declaration_with_spacing_and_case_variation( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + with connect_summary_database(projection_dir / "summary.sqlite") as connection: + connection.execute("DROP TABLE semantic_index_vec") + connection.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + "VEC0( source_type TEXT PARTITION KEY , embedding FLOAT [ 3 ] )" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "identity_case", + [ + "summary_base_url", + "summary_model", + "cache_base_url", + "cache_model", + ], +) +def test_cli_rejects_noncanonical_persisted_embedding_identity_without_mutation( + identity_case, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + if identity_case.startswith("summary_"): + with sqlite3.connect(projection_dir / "summary.sqlite") as connection: + metadata = json.loads( + connection.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + if identity_case == "summary_base_url": + metadata["base_url"] = "HTTPS://EXAMPLE.INVALID/v1/" + else: + metadata["model"] = " test-embedding " + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + (json.dumps(metadata, sort_keys=True),), + ) + else: + base_url = ( + "HTTPS://EXAMPLE.INVALID/v1/" + if identity_case == "cache_base_url" + else "https://example.invalid/v1" + ) + model = ( + " test-embedding " + if identity_case == "cache_model" + else "test-embedding" + ) + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as connection: + connection.execute( + "INSERT INTO embedding_cache(" + "base_url, model, dimensions, text_hash, vector_blob" + ") VALUES (?, ?, 3, 'cache-hash', ?)", + (base_url, model, b"\0" * 12), + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_allows_fresh_listing_when_projection_is_completely_absent(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert (workspace / "filesystem.sqlite").is_file() + assert not (workspace / "artifacts" / "projection_indexes").exists() + + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert not (workspace / "artifacts" / "projection_indexes").exists() + + +def test_pageindex_is_the_only_public_python_entry_point_for_pifs(): + import pageindex + import pageindex.filesystem as filesystem_module + + assert pageindex.PageIndexFileSystem is filesystem_module.PageIndexFileSystem + for internal_name in ( + "PIFSCommandExecutor", + "OpenResult", + "SearchResult", + "SummaryProjectionIndexer", + "SemanticProjectionSearchBackend", + "SQLiteVecSemanticIndex", + "SemanticIndexRecord", + "SemanticSearchResult", + ): + assert not hasattr(filesystem_module, internal_name) + assert not hasattr(filesystem_module, "_LAZY_EXPORTS") + + +def test_cli_add_creates_migration_compatible_summary_and_cache_v2( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + config = write_embedding_config(tmp_path, monkeypatch) + config_values = json.loads(config.read_text(encoding="utf-8")) + config_values["embedding_api_key"] = "config-secret" + config.write_text(json.dumps(config_values), encoding="utf-8") + monkeypatch.delenv("PIFS_EMBEDDING_API_KEY") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + output = capsys.readouterr().out + + projection_dir = workspace / "artifacts" / "projection_indexes" + with sqlite3.connect(projection_dir / "summary.sqlite") as summary: + summary_version = summary.execute("PRAGMA user_version").fetchone()[0] + summary_tables = logical_tables(summary) + metadata = json.loads( + summary.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as cache: + cache_version = cache.execute("PRAGMA user_version").fetchone()[0] + cache_tables = logical_tables(cache) + cache_columns = { + row[1] for row in cache.execute("PRAGMA table_info(embedding_cache)") + } + + assert summary_version == cache_version == 2 + assert summary_tables == { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", + } + assert cache_tables == {"embedding_cache"} + assert cache_columns == { + "base_url", + "model", + "dimensions", + "text_hash", + "vector_blob", + "created_at", + } + assert metadata == { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 3, + } + for database in ( + workspace / "filesystem.sqlite", + projection_dir / "summary.sqlite", + projection_dir / "embedding_cache.sqlite", + ): + assert b"config-secret" not in database.read_bytes() + assert "config-secret" not in output + + +def test_cli_add_reports_path_without_persistence_identity(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + + assert ( + main( + [ + "--workspace", + str(tmp_path / "workspace"), + "add", + str(source), + "/documents", + ] + ) + == 0 + ) + + output = capsys.readouterr().out + assert output == "added: /documents/notes.md\n" + + +def test_cli_browse_reopens_owned_file_with_canonical_result_fields( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + assert ( + main( + [ + "--workspace", + str(workspace), + "browse", + "/documents", + "alpha", + ] + ) + == 0 + ) + + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is True + document = payload["data"]["documents"][0] + assert set(document) == { + "path", + "document_id", + "title", + "status", + "rank", + "similarity", + "summary", + "metadata", + "folder_path", + "folder_paths", + } + assert document["path"] == "/documents/notes.md" + assert document["title"] == "notes.md" + assert document["status"] == "built" + assert document["rank"] == 1 + assert document["summary"].startswith("Summary for notes.md") + assert document["folder_path"] == "/documents" + + +def test_cli_stat_translates_persistence_identity_once(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "stat", "/documents/notes.md"]) == 0 + + document = json.loads(capsys.readouterr().out)["data"]["document"] + assert set(document) == { + "path", + "document_id", + "title", + "status", + "content_type", + "metadata", + "metadata_status", + "folder_paths", + } + assert document["path"] == "/documents/notes.md" + assert document["title"] == "notes.md" + assert document["status"] == "built" + assert document["folder_paths"] == ["/documents"] + + +def test_cli_cat_reads_structure_without_internal_identity(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + assert ( + main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--structure", + ] + ) + == 0 + ) + + data = json.loads(capsys.readouterr().out)["data"] + assert set(data["document"]) == { + "path", + "document_id", + "title", + "status", + "content_type", + "metadata", + "metadata_status", + "folder_paths", + "available", + } + assert data["structure"] == [ + {"title": "notes", "node_id": "0001", "line_num": 1, "nodes": []} + ] + + +def create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + records = [ + ( + "current.md", + "alpha current evidence", + "/documents", + {"year": 2024, "ticker": "AAPL", "sector": "finance/tech"}, + ), + ( + "filing.md", + "alpha filing evidence", + "/documents/sec-filings", + {"year": 2024, "ticker": "AAPL", "doc_type": "10-K"}, + ), + ( + "prior.md", + "alpha prior evidence", + "/documents", + {"year": 2023, "ticker": "MSFT"}, + ), + ("archive.md", "archive evidence", "/archive", {"region": "EMEA"}), + ] + for filename, content, folder, metadata in records: + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/{filename}", + json.dumps(metadata), + ] + ) == 0 + capsys.readouterr() + return workspace + + +def test_tree_folder_trailing_slash_lists_scope_local_metadata_axes( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + workspace = create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys) + + payloads = [] + for folder_path in ("/documents", "/documents/"): + assert main( + ["--workspace", str(workspace), "tree", folder_path, "-L", "1"] + ) == 0 + payloads.append(json.loads(capsys.readouterr().out)) + + assert payloads[1]["data"] == payloads[0]["data"] + tree = payloads[0]["data"]["tree"] + assert tree["path"] == "/documents" + axes = { + row["name"]: row + for row in tree["folders"] + if row["type"] == "metadata_axis" + } + + assert set(axes) == {"@doc_type", "@sector", "@ticker", "@year"} + assert {name: row["path"] for name, row in axes.items()} == { + "@doc_type": "/documents/@doc_type", + "@sector": "/documents/@sector", + "@ticker": "/documents/@ticker", + "@year": "/documents/@year", + } + assert {row["type"] for row in axes.values()} == {"metadata_axis"} + physical = [row for row in tree["folders"] if row["type"] == "folder"] + assert [(row["name"], row["path"]) for row in physical] == [ + ("sec-filings", "/documents/sec-filings") + ] + assert all(row["type"] != "file" for row in tree["folders"]) + assert "@region" not in axes + + +def test_tree_metadata_axis_trailing_slash_lists_actionable_paginated_values( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + workspace = create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys) + + payloads = [] + for axis_path in ("/documents/@year", "/documents/@year/"): + assert main( + ["--workspace", str(workspace), "tree", axis_path, "-L", "1"] + ) == 0 + payloads.append(json.loads(capsys.readouterr().out)) + + values = payloads[0]["data"]["tree"]["folders"] + assert payloads[1]["data"] == payloads[0]["data"] + assert [(row["name"], row["type"], row["path"]) for row in values] == [ + ("2024", "metadata_value", "/documents/@year/2024"), + ("2023", "metadata_value", "/documents/@year/2023"), + ] + assert payloads[0]["data"]["pagination"] == { + "page": 1, + "page_size": 50, + "has_more": False, + "next_page": None, + } + + selected = values[0]["path"] + scoped_payloads = [] + for value_path in (selected, f"{selected}/"): + assert main( + ["--workspace", str(workspace), "tree", value_path, "-L", "1"] + ) == 0 + scoped_payloads.append(json.loads(capsys.readouterr().out)) + assert scoped_payloads[1]["data"] == scoped_payloads[0]["data"] + selected_tree = scoped_payloads[0]["data"]["tree"] + assert selected_tree["path"] == selected + assert {row["title"] for row in selected_tree["files"]} == { + "current.md", + "filing.md", + } + selected_folders = { + (row["type"], row["name"]): row["path"] + for row in selected_tree["folders"] + } + assert selected_folders == { + ("folder", "sec-filings"): "/documents/sec-filings/@year/2024", + ("metadata_axis", "@doc_type"): "/documents/@year/2024/@doc_type", + ("metadata_axis", "@sector"): "/documents/@year/2024/@sector", + ("metadata_axis", "@ticker"): "/documents/@year/2024/@ticker", + } + assert main( + ["--workspace", str(workspace), "browse", selected, "alpha"] + ) == 0 + documents = json.loads(capsys.readouterr().out)["data"]["documents"] + assert {row["title"] for row in documents} == {"current.md", "filing.md"} + + locator = selected_tree["files"][0]["path"] + for command in ( + ["stat", locator], + ["cat", locator, "--structure"], + ["grep", "alpha", locator], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + capsys.readouterr() + + child_scope = selected_folders[("folder", "sec-filings")] + assert main(["--workspace", str(workspace), "tree", child_scope, "-L", "1"]) == 0 + child_tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert [row["title"] for row in child_tree["files"]] == ["filing.md"] + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@sector/", "-L", "1"] + ) == 0 + encoded = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"][0] + assert encoded["path"] == "/documents/@sector/finance%2Ftech" + assert main( + ["--workspace", str(workspace), "tree", encoded["path"], "-L", "1"] + ) == 0 + capsys.readouterr() + + +def test_tree_metadata_values_keep_fixed_fifty_item_pagination( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "tickers.md" + source.write_text("alpha ticker evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + tickers = [f"T{index:02d}" for index in range(55)] + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/tickers.md", + json.dumps({"ticker": tickers}), + ] + ) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@ticker/", "-L", "1"] + ) == 0 + first = json.loads(capsys.readouterr().out)["data"] + assert [row["value"] for row in first["tree"]["folders"]] == tickers[:50] + assert first["pagination"] == { + "page": 1, + "page_size": 50, + "has_more": True, + "next_page": 2, + } + + assert main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@ticker", + "--page", + "2", + ] + ) == 0 + second = json.loads(capsys.readouterr().out)["data"] + assert [row["value"] for row in second["tree"]["folders"]] == tickers[50:] + assert second["pagination"] == { + "page": 2, + "page_size": 50, + "has_more": False, + "next_page": None, + } + + +def test_tree_metadata_dot_values_return_actionable_encoded_scope_paths( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha dot value evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + json.dumps({"tag": [".", ".."]}), + ] + ) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents/@tag"]) == 0 + values = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"] + assert {row["value"]: row["path"] for row in values} == { + ".": "/documents/@tag/%2E", + "..": "/documents/@tag/%2E%2E", + } + + for scope_path in (row["path"] for row in values): + assert main( + ["--workspace", str(workspace), "tree", scope_path, "-L", "1"] + ) == 0 + file_path = json.loads(capsys.readouterr().out)["data"]["tree"]["files"][0][ + "path" + ] + assert main( + ["--workspace", str(workspace), "browse", scope_path, "alpha"] + ) == 0 + document = json.loads(capsys.readouterr().out)["data"]["documents"][0] + assert document["path"] == file_path + assert main(["--workspace", str(workspace), "stat", file_path]) == 0 + capsys.readouterr() + + +def test_metadata_virtual_paths_use_alternating_encoded_field_value_segments( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert ( + main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024, "sector": "finance/tech"}', + ] + ) + == 0 + ) + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents/@year"]) == 0 + year_payload = json.loads(capsys.readouterr().out) + values = year_payload["data"]["tree"]["folders"] + assert values[0]["path"] == "/documents/@year/2024" + assert year_payload["next_steps"] == [ + 'browse /documents/@year/ ""' + ] + + assert ( + main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@year/2024", + "-L", + "1", + ] + ) + == 0 + ) + scoped_tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert scoped_tree["files"][0]["path"] == "/documents/@year/2024/notes.md" + assert [folder["path"] for folder in scoped_tree["folders"]] == [ + "/documents/@year/2024/@sector" + ] + + assert main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@year/2024/@sector", + ] + ) == 0 + sector = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"][0] + assert sector["path"] == "/documents/@year/2024/@sector/finance%2Ftech" + + locator = f"{sector['path']}/notes.md" + for command in ( + ["stat", locator], + ["cat", locator, "--structure"], + ["grep", "alpha", locator], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@year=2024"] + ) == 2 + error = json.loads(capsys.readouterr().out)["error"] + assert "@field/value" in error["message"] + + +def test_setmeta_translates_identity_at_the_cli_boundary( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + document = json.loads(capsys.readouterr().out) + + assert set(document) == { + "path", + "document_id", + "title", + "status", + "metadata", + "metadata_status", + } + assert document["path"] == "/documents/notes.md" + assert document["metadata"]["year"] == 2024 + + +def test_setmeta_by_file_ref_returns_an_actionable_path_without_leaking_identity( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + file_ref = connection.execute("SELECT file_ref FROM files").fetchone()[0] + + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + file_ref, + '{"year": 2024}', + ] + ) == 0 + output = capsys.readouterr().out + document = json.loads(output) + + assert document["path"] == "/documents/notes.md" + assert file_ref not in output + assert main(["--workspace", str(workspace), "stat", document["path"]]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == document["path"] + + +@pytest.mark.parametrize("operation", ["replace", "clear"]) +def test_metadata_scoped_setmeta_returns_the_post_update_actionable_path( + operation, tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + old_path = "/documents/@year/2024/notes.md" + command = ["--workspace", str(workspace), "setmeta"] + if operation == "clear": + command.extend(["--clear", old_path]) + else: + command.extend([old_path, '{"year": 2025}']) + + assert main(command) == 0 + document = json.loads(capsys.readouterr().out) + + assert document["path"] == "/documents/notes.md" + assert document["path"] != old_path + if operation == "clear": + assert "year" not in document["metadata"] + else: + assert document["metadata"]["year"] == 2025 + assert main(["--workspace", str(workspace), "stat", document["path"]]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == document["path"] + + +def test_tree_file_records_use_only_command_identity_names( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) == 0 + file_record = json.loads(capsys.readouterr().out)["data"]["tree"]["files"][0] + + assert set(file_record) == { + "path", + "document_id", + "title", + "status", + "type", + "metadata", + } + + +def test_duplicate_virtual_leaves_use_actionable_paths_without_internal_ids( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for folder_name in ("a", "b"): + source_dir = tmp_path / folder_name + source_dir.mkdir() + source = source_dir / "notes.md" + source.write_text(f"alpha evidence {folder_name}", encoding="utf-8") + folder = f"/documents/{folder_name}" + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@year/2024", "-L", "1"] + ) == 0 + files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + paths = [row["path"] for row in files] + + assert len(paths) == len(set(paths)) == 2 + assert all("file_" not in path for path in paths) + for path in paths: + assert main(["--workspace", str(workspace), "stat", path]) == 0 + assert json.loads(capsys.readouterr().out)["data"]["document"]["path"] == path + + +@pytest.mark.parametrize( + ("filename", "content"), + [ + ("notes.md", b"markdown alpha evidence"), + ("report.pdf", b"%PDF-1.4 fake fixture"), + ], +) +def test_cli_add_owns_supported_documents_and_keeps_evidence_readable( + filename, content, tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / filename + source.write_bytes(content) + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + owned = list((workspace / "artifacts" / "uploads").glob(f"*/{filename}")) + assert len(owned) == 1 + assert owned[0].read_bytes() == content + assert main( + ["--workspace", str(workspace), "grep", "alpha", f"/documents/{filename}"] + ) == 0 + matches = json.loads(capsys.readouterr().out)["data"]["matches"] + assert matches[0]["text"].endswith("alpha evidence") + + +def test_cli_add_rejects_unsupported_type_before_registration( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.txt" + source.write_text("unsupported", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "Unsupported file type" in capsys.readouterr().err + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + + +def test_cli_add_rolls_back_when_pageindex_fails( + tmp_path, monkeypatch, capsys +): + from pageindex import PageIndexClient + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + + def fail_index(self, file_path, mode="auto"): + raise RuntimeError("pageindex unavailable") + + monkeypatch.setattr(PageIndexClient, "index", fail_index) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "pageindex unavailable" in capsys.readouterr().err + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + pageindex_cache = workspace / "artifacts" / "pageindex_client" + assert json.loads((pageindex_cache / "_meta.json").read_text(encoding="utf-8")) == {} + assert not [path for path in pageindex_cache.glob("*.json") if path.name != "_meta.json"] + + +def test_cli_add_rolls_back_when_summary_projection_fails( + tmp_path, monkeypatch, capsys +): + import openai + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + + class FailingOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + raise RuntimeError("embedding unavailable") + + monkeypatch.setattr(openai, "OpenAI", FailingOpenAI) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "summary projection" in capsys.readouterr().err.lower() + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + with sqlite3.connect( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM semantic_index_docs" + ).fetchone()[0] == 0 + + +@pytest.mark.parametrize("cache_state", ["fresh", "preexisting_shared_key"]) +def test_add_restores_embedding_cache_when_vec_upsert_fails( + cache_state, tmp_path, monkeypatch +): + from pageindex.filesystem.semantic_index import SQLiteVecSemanticIndex + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + source_dir = tmp_path / "add" + source_dir.mkdir() + source = source_dir / "notes.md" + source.write_text("alpha shared cache evidence", encoding="utf-8") + + if cache_state == "preexisting_shared_key": + existing_dir = tmp_path / "existing" + existing_dir.mkdir() + existing = existing_dir / "notes.md" + existing.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + filesystem.register_file( + storage_uri=existing.as_uri(), + folder_path="/documents/existing", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + ) + else: + create_projection_v2(workspace) + filesystem = open_test_filesystem(workspace) + + baseline = registration_logical_state(workspace) + baseline_uploads = sorted( + path.relative_to(workspace).as_posix() + for path in workspace.glob("artifacts/uploads/**/*") + ) + + def fail_vec_upsert(self, records): + raise RuntimeError("vec upsert unavailable after cache write") + + monkeypatch.setattr(SQLiteVecSemanticIndex, "upsert_many", fail_vec_upsert) + + with pytest.raises(RuntimeError, match="vec upsert unavailable after cache write"): + filesystem.add_file(source, "/documents/new") + + assert registration_logical_state(workspace) == baseline + assert sorted( + path.relative_to(workspace).as_posix() + for path in workspace.glob("artifacts/uploads/**/*") + ) == baseline_uploads + reopened = open_test_filesystem(workspace) + assert registration_logical_state(workspace) == baseline + if cache_state == "preexisting_shared_key": + assert reopened.store.file_refs_for_scope() == filesystem.store.file_refs_for_scope() + else: + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_completes_summary_projection_for_immediate_and_reopen_browse( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + caller_metadata = {"year": 2024, "labels": {"team": "alpha"}} + caller_metadata_baseline = json.loads(json.dumps(caller_metadata)) + + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata=caller_metadata, + ) + + projection_dir = workspace / "artifacts" / "projection_indexes" + assert (projection_dir / "summary.sqlite").is_file() + assert (projection_dir / "embedding_cache.sqlite").is_file() + stored = filesystem.store.get_file(file_ref) + assert caller_metadata == caller_metadata_baseline + assert "summary" not in caller_metadata + assert stored.metadata["summary"].startswith("Summary for notes.md:") + assert stored.metadata_status["summary_projection"]["status"] == "ready" + assert file_ref in { + row["file_ref"] + for row in filesystem.browse_semantic_files("/documents", "alpha")["data"] + } + + reopened = open_test_filesystem(workspace) + assert file_ref in { + row["file_ref"] + for row in reopened.browse_semantic_files("/documents", "alpha")["data"] + } + + +def test_register_file_opens_existing_projection_and_upserts_second_document( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + first_ref = filesystem.register_file( + storage_uri=first.as_uri(), + folder_path="/documents", + title=first.name, + content_type="text/markdown", + content=first.read_text(encoding="utf-8"), + ) + + reopened = open_test_filesystem(workspace) + second_ref = reopened.register_file( + storage_uri=second.as_uri(), + folder_path="/documents", + title=second.name, + content_type="text/markdown", + content=second.read_text(encoding="utf-8"), + ) + + assert reopened.store.get_file(second_ref).metadata_status["summary_projection"]["status"] == "ready" + assert second_ref in { + row["file_ref"] + for row in reopened.browse_semantic_files("/documents", "beta")["data"] + } + with connect_summary_database( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] == 2 + assert connection.execute("SELECT COUNT(*) FROM semantic_index_vec").fetchone()[0] == 2 + assert set(reopened.store.file_refs_for_scope()) == {first_ref, second_ref} + + +def test_register_files_rolls_back_new_batch_when_second_projection_upsert_fails( + tmp_path, monkeypatch +): + import openai + + install_network_fakes(monkeypatch) + + class FailingSecondOpenAI(FakeOpenAI): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = 0 + + def create(self, *, model, input, dimensions): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingSecondOpenAI) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + filesystem.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + } + for source in (first, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_restores_existing_document_when_later_projection_upsert_fails( + tmp_path, monkeypatch, capsys +): + import openai + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + original = tmp_path / "original.md" + replacement = tmp_path / "replacement.md" + failing = tmp_path / "failing.md" + original.write_text("alpha original evidence", encoding="utf-8") + replacement.write_text("alpha replacement evidence", encoding="utf-8") + failing.write_text("beta second evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=original.as_uri(), + external_id="doc-existing", + folder_path="/documents", + title=original.name, + content_type="text/markdown", + content=original.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + baseline = registration_logical_state(workspace) + + class FailingBetaOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + if any("beta" in str(text).lower() for text in input): + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingBetaOpenAI) + reopened = open_test_filesystem(workspace) + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + reopened.register_files( + [ + { + "storage_uri": replacement.as_uri(), + "external_id": "doc-existing", + "folder_path": "/documents/replacement", + "title": replacement.name, + "content_type": "text/markdown", + "content": replacement.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": failing.as_uri(), + "external_id": "doc-failing", + "folder_path": "/documents/new", + "title": failing.name, + "content_type": "text/markdown", + "content": failing.read_text(encoding="utf-8"), + "metadata": {"year": 2025}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + strict_reopen = open_test_filesystem(workspace) + assert strict_reopen.store.file_refs_for_scope() == [existing_ref] + entry = strict_reopen.store.get_file(existing_ref) + assert entry.metadata["year"] == 2023 + assert entry.storage_uri == original.as_uri() + assert strict_reopen.pageindex_structure(existing_ref)["available"] is True + for command in ( + ["stat", "/documents/original.md"], + ["cat", "/documents/original.md", "--structure"], + ["grep", "original", "/documents/original.md"], + ["browse", "/documents", "original"], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + output = json.loads(capsys.readouterr().out) + assert output["success"] is True + + +def test_register_files_rollback_preserves_preexisting_catalog_projection_and_cache( + tmp_path, monkeypatch +): + import openai + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + existing_dir = tmp_path / "existing" + batch_dir = tmp_path / "batch" + existing_dir.mkdir() + batch_dir.mkdir() + existing = existing_dir / "shared.md" + shared = batch_dir / "shared.md" + second = batch_dir / "second.md" + existing.write_text("alpha registration evidence", encoding="utf-8") + shared.write_text(existing.read_text(encoding="utf-8"), encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=existing.as_uri(), + folder_path="/documents", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + filesystem.metadata.register_schema( + {"fields": {"year": {"description": "preexisting definition"}}}, + source="manual", + ) + baseline = registration_logical_state(workspace) + + class FailingBetaOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + if any("beta" in str(text).lower() for text in input): + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingBetaOpenAI) + reopened = open_test_filesystem(workspace) + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + reopened.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024, "batch_only": "new"}, + } + for source in (shared, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + strict_reopen = open_test_filesystem(workspace) + assert strict_reopen.store.get_file(existing_ref).metadata["year"] == 2023 + structure = strict_reopen.pageindex_structure(existing_ref) + assert structure["available"] is True + assert structure["structure"][0]["title"] == existing.stem + + +def test_register_files_rolls_back_partial_pageindex_preparation( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + calls = 0 + + def fail_after_second_pageindex_write(self, file_path, mode="auto"): + nonlocal calls + calls += 1 + document_id = successful_index(self, file_path, mode=mode) + if calls == 2: + raise RuntimeError("PageIndex extraction failed for second registration") + return document_id + + monkeypatch.setattr(PageIndexClient, "index", fail_after_second_pageindex_write) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises( + RuntimeError, + match="PageIndex extraction failed for second registration", + ): + filesystem.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + } + for source in (first, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_prepare_failure_restores_existing_owned_artifacts( + tmp_path, monkeypatch +): + from pageindex.filesystem.commands import PIFSCommandExecutor + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + existing = tmp_path / "existing.md" + replacement = tmp_path / "replacement.md" + existing.write_text("alpha original registration evidence", encoding="utf-8") + replacement.write_text("beta replacement registration evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=existing.as_uri(), + external_id="doc-existing", + folder_path="/documents", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + existing_entry = filesystem.store.get_file(existing_ref) + text_path = Path(existing_entry.text_artifact_path) + raw_path = Path(existing_entry.raw_artifact_path) + original_text = text_path.read_bytes() + original_raw = raw_path.read_bytes() + original_pageindex_doc_id = existing_entry.pageindex_doc_id + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="requires PageIndex extraction"): + filesystem.register_files( + [ + { + "storage_uri": replacement.as_uri(), + "external_id": "doc-existing", + "folder_path": "/documents/replacement", + "title": replacement.name, + "content_type": "text/markdown", + "content": replacement.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": "https://example.invalid/unresolvable.md", + "external_id": "doc-failing", + "folder_path": "/documents/new", + "title": "unresolvable.md", + "content_type": "text/markdown", + "content": "must not be registered", + "metadata": {"year": 2025}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + assert text_path.read_bytes() == original_text + assert raw_path.read_bytes() == original_raw + + reopened = open_test_filesystem(workspace) + reopened_entry = reopened.store.get_file(existing_ref) + assert reopened_entry.pageindex_doc_id == original_pageindex_doc_id + executor = PIFSCommandExecutor(reopened) + locator = "/documents/existing.md" + assert json.loads(executor.execute(f"stat {locator}"))["success"] is True + cat = json.loads(executor.execute(f"cat {locator} --structure")) + assert cat["success"] is True + assert cat["data"]["pagination"]["available"] is True + grep = json.loads(executor.execute(f"grep alpha {locator}")) + assert grep["success"] is True + assert grep["data"]["matches"] + structure = reopened.pageindex_structure(existing_ref) + assert structure["available"] is True + assert structure["structure"][0]["title"] == existing.stem + + +@pytest.mark.parametrize("raw_path_kind", ["canonical", "custom"]) +def test_register_files_rollback_follows_explicit_raw_artifact_management_policy( + raw_path_kind, tmp_path, monkeypatch +): + import openai + from pageindex.filesystem.store import make_file_ref + + install_network_fakes(monkeypatch) + + class FailingSecondOpenAI(FakeOpenAI): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = 0 + + def create(self, *, model, input, dimensions): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingSecondOpenAI) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + first_external_id = f"doc-{raw_path_kind}-raw" + first_file_ref = make_file_ref(first_external_id) + raw_dir = workspace / "artifacts" / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_path = raw_dir / ( + f"{first_file_ref}.json" + if raw_path_kind == "canonical" + else "custom-sentinel.json" + ) + sentinel = f"{raw_path_kind} raw sentinel".encode() + raw_path.write_bytes(sentinel) + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + filesystem.register_files( + [ + { + "storage_uri": first.as_uri(), + "external_id": first_external_id, + "folder_path": "/documents/new", + "title": first.name, + "content_type": "text/markdown", + "content": first.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + "raw_artifact_path": str(raw_path), + }, + { + "storage_uri": second.as_uri(), + "external_id": "doc-second-raw", + "folder_path": "/documents/new", + "title": second.name, + "content_type": "text/markdown", + "content": second.read_text(encoding="utf-8"), + "metadata": {"year": 2025}, + }, + ] + ) + + assert raw_path.read_bytes() == sentinel + assert registration_logical_state(workspace) == baseline + reopened = open_test_filesystem(workspace) + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_rejects_non_json_metadata_before_side_effects( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + indexed_paths = [] + + def record_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path).name) + return successful_index(self, file_path, mode=mode) + + monkeypatch.setattr(PageIndexClient, "index", record_index) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(ValueError, match="metadata must be JSON serializable"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents/new", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata={"not_json": object()}, + ) + + assert registration_logical_state(workspace) == baseline + assert indexed_paths == [] + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_preflights_all_metadata_before_preparing_batch( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + indexed_paths = [] + + def record_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path).name) + return successful_index(self, file_path, mode=mode) + + monkeypatch.setattr(PageIndexClient, "index", record_index) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(ValueError, match="metadata must be JSON serializable"): + filesystem.register_files( + [ + { + "storage_uri": first.as_uri(), + "folder_path": "/documents/new", + "title": first.name, + "content_type": "text/markdown", + "content": first.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": second.as_uri(), + "folder_path": "/documents/new", + "title": second.name, + "content_type": "text/markdown", + "content": second.read_text(encoding="utf-8"), + "metadata": {"not_json": object()}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + assert indexed_paths == [] + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_rolls_back_partial_projection_creation_and_reopens( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + # SQLite's Unix VFS accepts summary.sqlite at this path length but rejects + # the eight-character-longer embedding_cache.sqlite path. + workspace = nested_path_with_length(tmp_path / "long-workspace", 453) + filesystem = open_test_filesystem(workspace) + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + + with pytest.raises(sqlite3.OperationalError, match="unable to open database file"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + + projection_dir = workspace / "artifacts" / "projection_indexes" + assert not (projection_dir / "summary.sqlite").exists() + assert not (projection_dir / "embedding_cache.sqlite").exists() + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "text").glob("*")) + assert not list((workspace / "artifacts" / "raw").glob("*")) + + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + + +def test_cli_add_rejects_duplicate_target_without_changing_owned_document( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + first_dir.mkdir() + second_dir.mkdir() + first = first_dir / "notes.md" + second = second_dir / "notes.md" + first.write_text("first alpha evidence", encoding="utf-8") + second.write_text("second beta evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + command = ["--workspace", str(workspace), "add"] + + assert main([*command, str(first), "/documents"]) == 0 + capsys.readouterr() + assert main([*command, str(second), "/documents"]) == 1 + + assert "already exists" in capsys.readouterr().err + assert catalog_file_count(workspace) == 1 + owned = list((workspace / "artifacts" / "uploads").glob("*/notes.md")) + assert len(owned) == 1 + assert owned[0].read_text(encoding="utf-8") == "first alpha evidence" + + +def test_physical_browse_preserves_collision_aware_tree_locators( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for filename, content in ( + ("alpha.md", "alpha evidence"), + ("beta.md", "beta evidence"), + ): + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE file_folders SET metadata_json = ?", + (json.dumps({"display_name": "same.md"}),), + ) + + assert main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) == 0 + tree_files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + tree_paths = {row["path"] for row in tree_files} + assert tree_paths == { + "/documents/same.md~1", + "/documents/same.md~2", + } + + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 0 + browse_documents = json.loads(capsys.readouterr().out)["data"]["documents"] + browse_paths = [row["path"] for row in browse_documents] + + assert len(browse_documents) == 2 + assert len(set(browse_paths)) == 2 + assert set(browse_paths) == tree_paths + for path in browse_paths: + assert main(["--workspace", str(workspace), "stat", path]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == path + + +def test_browse_recursively_returns_one_global_ranked_page_of_ten( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for index in range(12): + source = tmp_path / f"doc-{index:02d}.md" + relevance = "alpha" if index in {1, 10} else "beta" + source.write_text(f"{relevance} evidence {index}", encoding="utf-8") + folder = "/documents/a" if index % 2 == 0 else "/documents/b" + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + + command = [ + "--workspace", + str(workspace), + "browse", + "/documents", + "alpha", + "--recursive", + ] + assert main(command) == 0 + first = json.loads(capsys.readouterr().out)["data"] + assert len(first["documents"]) == 10 + assert first["pagination"] == { + "page": 1, + "page_size": 10, + "has_more": True, + "next_page": 2, + } + assert {row["title"] for row in first["documents"][:2]} == { + "doc-01.md", + "doc-10.md", + } + assert {row["folder_path"] for row in first["documents"]} == { + "/documents/a", + "/documents/b", + } + + assert main([*command, "--page", "2"]) == 0 + second = json.loads(capsys.readouterr().out)["data"] + assert len(second["documents"]) == 2 + assert second["pagination"] == { + "page": 2, + "page_size": 10, + "has_more": False, + "next_page": None, + } + assert not { + row["path"] for row in first["documents"] + }.intersection(row["path"] for row in second["documents"]) + + +def test_browse_respects_physical_and_virtual_scope( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + records = [ + ("current.md", "alpha current", "/documents/current", 2024), + ("old.md", "alpha old", "/documents/archive", 2023), + ] + for filename, content, folder, year in records: + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/{filename}", + json.dumps({"year": year}), + ] + ) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "browse", + "/documents/current", + "alpha", + ] + ) == 0 + physical = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [row["title"] for row in physical] == ["current.md"] + + assert main( + [ + "--workspace", + str(workspace), + "browse", + "/documents/@year/2024", + "alpha", + ] + ) == 0 + virtual = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [row["title"] for row in virtual] == ["current.md"] + assert virtual[0]["path"] == "/documents/@year/2024/current.md" + + +def test_browse_requires_query_and_an_existing_summary_projection( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "browse", "/documents"]) == 2 + missing_query = json.loads(capsys.readouterr().out) + assert "requires a query" in missing_query["error"]["message"] + + assert main(["--workspace", str(workspace), "browse", "/", "alpha"]) == 2 + missing_projection = json.loads(capsys.readouterr().out) + assert "Summary Projection is not available" in missing_projection["error"]["message"] + + +def test_browse_rejects_incompatible_embedding_identity_without_mutation( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + config = write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + databases = [ + workspace / "filesystem.sqlite", + workspace / "artifacts" / "projection_indexes" / "summary.sqlite", + workspace / "artifacts" / "projection_indexes" / "embedding_cache.sqlite", + ] + before = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases} + config.write_text( + json.dumps( + { + "embedding_base_url": "https://example.invalid/v1", + "embedding_model": "different-model", + "embedding_dimensions": 3, + } + ), + encoding="utf-8", + ) + + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 2 + + payload = json.loads(capsys.readouterr().out) + assert "Incompatible PIFS Summary Embedding Profile" in payload["error"]["message"] + assert { + path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases + } == before + + +def test_agent_prompts_describe_only_the_retained_command_surface(): + from pageindex.filesystem.agent import ( + AGENT_SYSTEM_PROMPT, + AGENT_TOOL_POLICY, + BASH_TOOL_DESCRIPTION, + ) + + prompts = "\n".join( + [AGENT_SYSTEM_PROMPT, BASH_TOOL_DESCRIPTION, AGENT_TOOL_POLICY] + ) + for command in ("tree", "browse", "stat", "cat", "grep"): + assert command in prompts + for retired_contract in ( + "ls as an alias", + "file_ref", + "--space", + "Do not use find", + "recursive grep", + ): + assert retired_contract not in prompts + + +def test_cat_page_reads_are_bounded_and_grep_is_single_document( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("line one\nalpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--page", + "1", + ] + ) == 0 + page = json.loads(capsys.readouterr().out)["data"] + assert page["requested_pages"] == "1" + assert page["content"]["text"] == "line one\nalpha evidence" + + assert main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--page", + "1-6", + ] + ) == 2 + too_wide = json.loads(capsys.readouterr().out) + assert "at most 5 pages" in too_wide["error"]["message"] + + assert main( + ["--workspace", str(workspace), "grep", "alpha", "/documents"] + ) == 2 + folder_grep = json.loads(capsys.readouterr().out) + assert "resolved file locator" in folder_grep["error"]["message"] + + +@pytest.mark.parametrize( + "command", + [ + ["ls", "/"], + ["find", "/"], + ["browse", "/", "alpha", "--space", "entity"], + ["browse", "/", "alpha", "--limit", "2"], + ["stat", "/", "--schema"], + ["stat", "/", "--field", "year"], + ["stat", "/one", "/two"], + ["cat", "/document"], + ["cat", "/document", "--all"], + ["cat", "/document", "--range", "1-2"], + ["grep", "alpha", "/document", "--recursive"], + ], +) +def test_retired_command_forms_return_structured_errors(command, tmp_path, capsys): + from pageindex.filesystem.cli import main + + status = main(["--workspace", str(tmp_path / "workspace"), *command]) + + payload = json.loads(capsys.readouterr().out) + assert status == 2 + assert payload["success"] is False + assert payload["error"]["code"] == "invalid_command" + + +def test_set_workspace_preserves_embedding_api_key_without_exposing_or_persisting_it_elsewhere( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + config = tmp_path / "pifs.json" + config.write_text( + json.dumps( + { + "embedding_base_url": "https://example.invalid/v1", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + "embedding_timeout": 12, + "embedding_api_key": "config-secret", + "embedding_provider": "legacy-provider", + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("PIFS_CONFIG_FILE", str(config)) + workspace = tmp_path / "workspace" + + assert main(["set", "workspace", str(workspace)]) == 0 + output = capsys.readouterr().out + + assert json.loads(config.read_text(encoding="utf-8")) == { + "embedding_api_key": "config-secret", + "embedding_base_url": "https://example.invalid/v1", + "embedding_dimensions": "3", + "embedding_model": "test-embedding", + "embedding_timeout": "12", + "workspace": str(workspace), + } + assert "config-secret" not in output + + +def test_public_example_is_a_short_supported_cli_walkthrough(): + example = Path(__file__).parents[1] / "examples" / "pifs_demo.py" + source = example.read_text(encoding="utf-8") + + assert len(source.splitlines()) <= 60 + for command in ("add", "tree", "browse", "stat", "cat", "grep"): + assert f'"{command}"' in source + for retired_surface in ( + "embedding_provider", + "metadata_provider", + "file_ref", + "SummaryProjectionIndexer", + "shutil.rmtree", + '"ls"', + ): + assert retired_surface not in source diff --git a/tests/test_pifs_string_metadata_cleanup.py b/tests/test_pifs_string_metadata_cleanup.py deleted file mode 100644 index e8df83f02..000000000 --- a/tests/test_pifs_string_metadata_cleanup.py +++ /dev/null @@ -1,189 +0,0 @@ -import json -from pathlib import Path - -import pytest - - -class StaticEmbedder: - def embed(self, texts): - return [[1.0, 0.0, 0.0] for _ in texts] - - -def _patch_summary_indexer(monkeypatch): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - def fake_from_provider(index_dir, **kwargs): - return SummaryProjectionIndexer( - Path(index_dir), - embedder=StaticEmbedder(), - embedding_provider="test", - embedding_model="static", - embedding_dimensions=3, - ) - - monkeypatch.setattr(SummaryProjectionIndexer, "from_provider", fake_from_provider) - - -def _patch_pageindex_client(monkeypatch, *, description="PageIndex summary"): - from pageindex import PageIndexClient - - def fake_index(self, file_path, mode="auto"): - doc_id = Path(file_path).stem + "_doc" - doc = { - "id": doc_id, - "type": "md", - "path": str(Path(file_path).resolve()), - "doc_name": Path(file_path).name, - "doc_description": description, - "line_count": 2, - "structure": [{"title": "Doc", "node_id": "0001", "nodes": []}], - "pages": [{"page": 1, "content": Path(file_path).read_text(encoding="utf-8")}], - } - self.documents[doc_id] = doc - self._save_doc(doc_id) - return doc_id - - monkeypatch.setattr(PageIndexClient, "index", fake_index) - - -def _register_md(filesystem, tmp_path, monkeypatch, name, metadata=None): - _patch_summary_indexer(monkeypatch) - _patch_pageindex_client(monkeypatch, description=f"summary for {name}") - path = tmp_path / name - path.write_text(f"# {name}\n\nbody", encoding="utf-8") - return filesystem.register_file( - storage_uri=path.as_uri(), - folder_path="/documents", - title=name, - content_type="text/markdown", - metadata=metadata or {}, - ) - - -def test_metadata_schema_is_string_registry_only(tmp_path): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - filesystem.metadata.register_schema( - {"fields": {"ticker": {"description": "stock ticker"}}} - ) - - assert filesystem.metadata.export_schema() == { - "fields": {"ticker": {"description": "stock ticker", "source": "manual"}} - } - with filesystem.store.connect() as conn: - tables = { - row["name"] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table'" - ).fetchall() - } - metadata_value_columns = { - row["name"] for row in conn.execute("PRAGMA table_info(metadata_values)") - } - - assert "metadata_schema" not in tables - assert metadata_value_columns == {"file_ref", "field_id", "value_text", "created_at"} - - -def test_range_filters_are_removed(tmp_path): - from pageindex.filesystem import PageIndexFileSystem - from pageindex.filesystem.metadata import MetadataQueryError - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - filesystem.metadata.register_schema({"fields": {"score": {}}}) - - with pytest.raises(MetadataQueryError, match="Unsupported metadata operator: \\$gte"): - filesystem.metadata.parse_filter({"score": {"$gte": 3}}) - - -def test_numeric_metadata_filter_is_string_equality(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - filesystem.metadata.register_schema({"fields": {"score": {}}}) - _register_md(filesystem, tmp_path, monkeypatch, "int.md", {"score": 3}) - _register_md(filesystem, tmp_path, monkeypatch, "float.md", {"score": 3.0}) - _register_md(filesystem, tmp_path, monkeypatch, "text.md", {"score": "3"}) - - rows = filesystem.search(None, metadata_filter={"score": {"$eq": 3}}) - - assert {row.title for row in rows} == {"int.md", "text.md"} - - -def test_register_uses_pageindex_description_as_summary_only(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - file_ref = _register_md(filesystem, tmp_path, monkeypatch, "summary.md", {"ticker": "AAPL"}) - info = filesystem.store.file_info(file_ref) - - assert info["metadata"]["summary"] == "summary for summary.md" - assert "summary" not in filesystem.metadata.export_schema()["fields"] - with filesystem.store.connect() as conn: - fields = [row["name"] for row in conn.execute("SELECT name FROM metadata_fields")] - assert fields == ["ticker"] - - -def test_register_rejects_plain_text_without_pageindex(tmp_path): - from pageindex.filesystem import PageIndexFileSystem - - source = tmp_path / "plain.txt" - source.write_text("plain text", encoding="utf-8") - filesystem = PageIndexFileSystem(tmp_path / "workspace") - - with pytest.raises(ValueError, match="PDF/Markdown"): - filesystem.register_file( - storage_uri=source.as_uri(), - folder_path="/documents", - title="plain.txt", - content_type="text/plain", - content="plain text", - ) - - -def test_set_metadata_replaces_custom_metadata_without_touching_summary(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - filesystem = PageIndexFileSystem(tmp_path / "workspace") - file_ref = _register_md(filesystem, tmp_path, monkeypatch, "meta.md", {"ticker": "AAPL"}) - - info = filesystem.set_metadata(file_ref, {"ticker": "MSFT", "year": 2026}) - - assert info["metadata"] == { - "summary": "summary for meta.md", - "ticker": "MSFT", - "year": 2026, - } - assert {field.name for field in filesystem.store.list_metadata_fields()} == { - "ticker", - "year", - } - assert filesystem.set_metadata(file_ref, {}, clear=True)["metadata"] == { - "summary": "summary for meta.md" - } - - -def test_set_metadata_keeps_summary_out_of_ordinary_lexical_search(tmp_path, monkeypatch): - from pageindex.filesystem import PageIndexFileSystem - - unique_summary_token = "summaryleaktoken" - filesystem = PageIndexFileSystem(tmp_path / "workspace") - _patch_summary_indexer(monkeypatch) - _patch_pageindex_client(monkeypatch, description=unique_summary_token) - path = tmp_path / "leak.md" - path.write_text("# Leak\n\nordinary body", encoding="utf-8") - file_ref = filesystem.register_file( - storage_uri=path.as_uri(), - folder_path="/documents", - title="leak.md", - content_type="text/markdown", - metadata={"ticker": "AAPL"}, - ) - - assert filesystem.search(unique_summary_token) == [] - - filesystem.set_metadata(file_ref, {"ticker": "MSFT"}) - - assert filesystem.search(unique_summary_token) == [] - assert {row.title for row in filesystem.search("MSFT")} == {"leak.md"} diff --git a/tests/test_semantic_index.py b/tests/test_semantic_index.py index 2b277dc08..0346f60f9 100644 --- a/tests/test_semantic_index.py +++ b/tests/test_semantic_index.py @@ -1,19 +1,19 @@ -import sys -from pathlib import Path +import hashlib import pytest -REPO_ROOT = Path(__file__).resolve().parents[1] -if str(REPO_ROOT) not in sys.path: - sys.path.insert(0, str(REPO_ROOT)) - from pageindex.filesystem.semantic_index import ( SemanticIndexRecord, SQLiteVecSemanticIndex, ) +from pageindex.filesystem.semantic_projection import ( + EmbeddingCache, + SummaryEmbeddingProfile, + SummaryProjection, +) -class FixedDimensionEmbedder: +class FixedEmbedder: def __init__(self, dimensions: int): self.dimensions = dimensions @@ -23,14 +23,13 @@ def embed(self, texts): def test_sqlite_vec_semantic_index_round_trip(tmp_path): index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") - index.reset(dimension=3, metadata={"field_mode": "summary"}) - + index.reset(dimension=3, metadata={"kind": "summary"}) index.upsert_many( [ SemanticIndexRecord( file_ref="file_a", external_id="doc_a", - source_type="github", + source_type="documents", title="Multipart upload limits", text="multipart upload limits", vector=[1.0, 0.0, 0.0], @@ -39,7 +38,7 @@ def test_sqlite_vec_semantic_index_round_trip(tmp_path): SemanticIndexRecord( file_ref="file_b", external_id="doc_b", - source_type="slack", + source_type="documents", title="GPU cache issue", text="gpu cache issue", vector=[0.0, 1.0, 0.0], @@ -48,23 +47,14 @@ def test_sqlite_vec_semantic_index_round_trip(tmp_path): ] ) - assert index.info()["document_count"] == 2 - - results = index.search([0.9, 0.1, 0.0], limit=2) - assert [item.external_id for item in results] == ["doc_a", "doc_b"] - - filtered = index.search( - [0.9, 0.1, 0.0], - limit=2, - filters={"source_type": "slack"}, - ) - assert [item.external_id for item in filtered] == ["doc_b"] + assert [ + item.external_id for item in index.search([0.9, 0.1, 0.0], limit=2) + ] == ["doc_a", "doc_b"] -def test_sqlite_vec_semantic_index_file_ref_filter_not_limited_by_global_rank(tmp_path): +def test_sqlite_vec_file_ref_filter_is_not_limited_by_global_rank(tmp_path): index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") - index.reset(dimension=2, metadata={"field_mode": "summary"}) - + index.reset(dimension=2, metadata={"kind": "summary"}) records = [ SemanticIndexRecord( file_ref=f"file_off_{item:02d}", @@ -97,237 +87,108 @@ def test_sqlite_vec_semantic_index_file_ref_filter_not_limited_by_global_rank(tm assert [item.file_ref for item in results] == ["file_in_scope"] -def test_summary_projection_indexes_unified_metadata_summary(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - indexer = SummaryProjectionIndexer( - tmp_path / "projection", - embedder=FixedDimensionEmbedder(3), - embedding_provider="test", - embedding_model="fake", - embedding_dimensions=3, +def test_summary_projection_owns_write_search_and_delete_lifecycle(tmp_path): + profile = SummaryEmbeddingProfile( + base_url="https://EXAMPLE.invalid/v1/", + model="fake", + dimensions=3, + api_key="runtime-only", ) - - result = indexer.upsert_summary( - { - "file_ref": "file_a", - "external_id": "doc_a", - "source_type": "documents", - "title": "A", - "metadata": { - "summary": "Unified metadata summary.", - "department": "ops", - }, - } - ) - - assert result["status"] == "ready" - hits = indexer.index.search([1.0, 0.0, 0.0], limit=1) - assert hits[0].external_id == "doc_a" - assert hits[0].metadata["summary"] == "Unified metadata summary." - assert hits[0].metadata["department"] == "ops" - - -def test_summary_projection_indexer_defaults_to_1024_dimensions(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - indexer = SummaryProjectionIndexer( + projection = SummaryProjection( tmp_path / "projection", - embedder=FixedDimensionEmbedder(1024), - embedding_provider="test", - embedding_model="fake", + profile=profile, + embedder=FixedEmbedder(3), + create=True, ) + record = { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": {"summary": "Unified summary", "department": "ops"}, + } + + assert projection.upsert_summary(record)["status"] == "ready" + assert projection.info()["embedding_identity"] == { + "base_url": "https://example.invalid/v1", + "model": "fake", + "dimensions": 3, + } + assert [candidate.file_ref for candidate in projection.search("summary")] == [ + "file_a" + ] + assert projection.delete_summary("file_a") == 1 + assert projection.available is False - info = indexer.index.info() - - assert info["dimension"] == 1024 - assert info["metadata"]["embedding_dimensions"] == 1024 - result = indexer.upsert_summary( - { - "file_ref": "file_a", - "external_id": "doc_a", - "source_type": "documents", - "title": "A", - "metadata": {"summary": "Default dimension summary."}, - } +def test_projection_profile_mismatch_preserves_existing_database(tmp_path): + index_dir = tmp_path / "projection" + profile = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="fake", + dimensions=3, + api_key="runtime-only", ) - - assert result["status"] == "ready" - assert result["embedding_dimensions"] == 1024 - - -def test_summary_projection_indexer_allows_explicit_256_dimensions(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - indexer = SummaryProjectionIndexer( - tmp_path / "projection", - embedder=FixedDimensionEmbedder(256), - embedding_provider="test", - embedding_model="fake", - embedding_dimensions=256, + projection = SummaryProjection( + index_dir, + profile=profile, + embedder=FixedEmbedder(3), + create=True, ) - - assert indexer.index.info()["dimension"] == 256 - assert indexer.upsert_summary( + projection.upsert_summary( { "file_ref": "file_a", "external_id": "doc_a", "source_type": "documents", "title": "A", - "metadata": {"summary": "Explicit 256 dimension summary."}, + "metadata": {"summary": "Preserve me"}, } - )["status"] == "ready" - - -def test_summary_projection_default_rejects_existing_256_index_for_writes(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - index_dir = tmp_path / "projection" - index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - index.reset( - dimension=256, - metadata={ - "channel": "summary", - "embedding_provider": "test", - "embedding_model": "fake", - "embedding_dimensions": 256, - }, - ) - - with pytest.raises(RuntimeError, match="configured embedding_dimensions is 1024"): - SummaryProjectionIndexer( - index_dir, - embedder=FixedDimensionEmbedder(1024), - embedding_provider="test", - embedding_model="fake", - ) - - assert SQLiteVecSemanticIndex(index.db_path).info()["dimension"] == 256 - - -def test_summary_projection_from_provider_rejects_dimension_mismatch_before_embedder( - tmp_path, monkeypatch -): - from pageindex.filesystem import semantic_projection - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - index_dir = tmp_path / "projection" - index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - index.reset( - dimension=256, - metadata={ - "channel": "summary", - "embedding_provider": "openai", - "embedding_model": "text-embedding-3-small", - "embedding_dimensions": 256, - }, ) - - def fail_make_embedder(*args, **kwargs): - raise AssertionError("embedder should not be constructed before dimension validation") - - monkeypatch.setattr(semantic_projection, "make_embedder", fail_make_embedder) - - with pytest.raises(RuntimeError, match="configured embedding_dimensions is 1024"): - SummaryProjectionIndexer.from_provider(index_dir) - - -def test_embedding_cache_key_separates_model_dimensions(tmp_path): - from pageindex.filesystem.semantic_projection import ( - EmbeddingCache, - embedding_cache_model_key, + summary_path = index_dir / "summary.sqlite" + before = hashlib.sha256(summary_path.read_bytes()).hexdigest() + mismatch = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="different", + dimensions=3, + api_key="runtime-only", ) - class CountingEmbedder: - def __init__(self, dimensions: int): - self.dimensions = dimensions - self.calls = 0 + with pytest.raises(Exception, match="Incompatible PIFS Summary Embedding Profile"): + SummaryProjection(index_dir, profile=mismatch, create=False) - def embed(self, texts): - self.calls += 1 - return [[float(self.dimensions), *([0.0] * (self.dimensions - 1))] for _ in texts] - - cache = EmbeddingCache(tmp_path / "cache.sqlite") - embedder_256 = CountingEmbedder(256) - embedder_1024 = CountingEmbedder(1024) - key_256 = embedding_cache_model_key("fake", 256) - key_1024 = embedding_cache_model_key("fake", 1024) - - assert key_256 != key_1024 - - vector_256 = cache.embed_texts( - ["same text"], - provider="test", - model=key_256, - embedder=embedder_256, - batch_size=1, - )[0] - vector_1024 = cache.embed_texts( - ["same text"], - provider="test", - model=key_1024, - embedder=embedder_1024, - batch_size=1, - )[0] - - assert len(vector_256) == 256 - assert len(vector_1024) == 1024 - assert embedder_256.calls == 1 - assert embedder_1024.calls == 1 + assert hashlib.sha256(summary_path.read_bytes()).hexdigest() == before def test_embedding_cache_rejects_response_length_mismatch(tmp_path): - from pageindex.filesystem.semantic_projection import EmbeddingCache - class ShortEmbedder: def embed(self, texts): return [[1.0, 0.0, 0.0]] - cache = EmbeddingCache(tmp_path / "cache.sqlite") + cache = EmbeddingCache(tmp_path / "cache.sqlite", create=True) + profile = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="fake", + dimensions=3, + api_key="runtime-only", + ) with pytest.raises(ValueError, match="embedding response length mismatch"): cache.embed_texts( ["first", "second"], - provider="test", - model="fake", + profile=profile, embedder=ShortEmbedder(), batch_size=2, ) -def test_embed_with_retry_does_not_retry_permanent_errors(monkeypatch): +def test_embed_with_retry_only_retries_transient_errors(monkeypatch): from pageindex.filesystem.semantic_projection import embed_with_retry sleeps = [] - class PermanentEmbeddingError(Exception): - status_code = 401 - - class FailingEmbedder: - calls = 0 - - def embed(self, texts): - self.calls += 1 - raise PermanentEmbeddingError("unauthorized") - - monkeypatch.setattr("pageindex.filesystem.semantic_projection.time.sleep", sleeps.append) - embedder = FailingEmbedder() - - with pytest.raises(PermanentEmbeddingError): - embed_with_retry(embedder, ["text"]) - - assert embedder.calls == 1 - assert sleeps == [] - - -def test_embed_with_retry_retries_transient_errors(monkeypatch): - from pageindex.filesystem.semantic_projection import embed_with_retry - - sleeps = [] - - class TransientEmbeddingError(Exception): - status_code = 500 + class EmbeddingError(Exception): + def __init__(self, status_code): + self.status_code = status_code class FlakyEmbedder: calls = 0 @@ -335,122 +196,25 @@ class FlakyEmbedder: def embed(self, texts): self.calls += 1 if self.calls == 1: - raise TransientEmbeddingError("server error") + raise EmbeddingError(500) return [[1.0, 0.0, 0.0]] - monkeypatch.setattr("pageindex.filesystem.semantic_projection.time.sleep", sleeps.append) + monkeypatch.setattr( + "pageindex.filesystem.semantic_projection.time.sleep", sleeps.append + ) embedder = FlakyEmbedder() - assert embed_with_retry(embedder, ["text"]) == [[1.0, 0.0, 0.0]] assert embedder.calls == 2 assert sleeps == [1.0] + class PermanentEmbedder: + calls = 0 -def test_summary_projection_dimension_mismatch_preserves_existing_index(tmp_path): - from pageindex.filesystem.semantic_projection import SummaryProjectionIndexer - - index_dir = tmp_path / "projection" - index = SQLiteVecSemanticIndex(index_dir / "summary.sqlite") - index.reset( - dimension=3, - metadata={ - "channel": "summary", - "embedding_provider": "test", - "embedding_model": "fake", - "embedding_dimensions": 3, - }, - ) - index.upsert_many( - [ - SemanticIndexRecord( - file_ref="file_a", - external_id="doc_a", - source_type="documents", - title="A", - text="summary", - vector=[1.0, 0.0, 0.0], - ) - ] - ) - - with pytest.raises(RuntimeError, match="summary projection index dimension mismatch"): - SummaryProjectionIndexer( - index_dir, - embedder=FixedDimensionEmbedder(4), - embedding_provider="test", - embedding_model="fake", - embedding_dimensions=4, - ) - - preserved = SQLiteVecSemanticIndex(index.db_path) - assert preserved.info()["dimension"] == 3 - assert preserved.info()["document_count"] == 1 - assert preserved.search([1.0, 0.0, 0.0], limit=1)[0].external_id == "doc_a" - - -def test_hash_embedding_provider_is_not_available(): - from pageindex.filesystem.semantic_projection import make_embedder - - with pytest.raises(ValueError, match="unknown embedding provider: hash"): - make_embedder("hash", "unused", dimensions=256, timeout=1) - - -def test_make_embedder_accepts_direct_runtime_config(monkeypatch): - from pageindex.filesystem.semantic_projection import make_embedder - - calls = [] - - class FakeOpenAI: - def __init__(self, **kwargs): - calls.append(kwargs) - - fake_openai = type(sys)("openai") - fake_openai.OpenAI = FakeOpenAI - monkeypatch.setitem(sys.modules, "openai", fake_openai) - monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") - monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") - monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") - monkeypatch.setenv("OPENAI_BASE_URL", "https://ignored-openai.invalid/") - - make_embedder( - "openai", - "gemini-embedding-2-preview", - dimensions=3072, - timeout=12.5, - api_key="direct-key", - base_url="https://example.invalid/openai/", - ) - - assert calls == [ - { - "api_key": "direct-key", - "base_url": "https://example.invalid/openai/", - "timeout": 12.5, - } - ] - - -def test_make_embedder_requires_direct_api_key(monkeypatch): - from pageindex.filesystem.semantic_projection import make_embedder - - calls = [] - - class FakeOpenAI: - def __init__(self, **kwargs): - calls.append(kwargs) - - fake_openai = type(sys)("openai") - fake_openai.OpenAI = FakeOpenAI - monkeypatch.setitem(sys.modules, "openai", fake_openai) - monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") - monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") - - with pytest.raises(ValueError, match="embedding_api_key is required"): - make_embedder( - "openai", - "gemini-embedding-2-preview", - dimensions=3072, - timeout=12.5, - ) + def embed(self, texts): + self.calls += 1 + raise EmbeddingError(401) - assert calls == [] + permanent = PermanentEmbedder() + with pytest.raises(EmbeddingError): + embed_with_retry(permanent, ["text"]) + assert permanent.calls == 1