refactor(filesystem): simplify PIFS core surface

pageindex/filesystem/__init__.py

@@ -1,51 +1,3 @@
-from importlib import import_module
-from typing import TYPE_CHECKING
-
-from .commands import PIFSCommandExecutor
 from .core import PageIndexFileSystem
-from .types import OpenResult, SearchResult
-
-if TYPE_CHECKING:
-    from .semantic_projection import SemanticProjectionSearchBackend
-    from .semantic_projection import SummaryProjectionIndexer
-    from .semantic_index import (
-        RebuildableSemanticIndex,
-        SemanticIndexRecord,
-        SemanticSearchResult,
-        SQLiteVecSemanticIndex,
-    )
-
-_LAZY_EXPORTS = {
-    "SemanticProjectionSearchBackend": (".semantic_projection", "SemanticProjectionSearchBackend"),
-    "RebuildableSemanticIndex": (".semantic_index", "RebuildableSemanticIndex"),
-    "SemanticIndexRecord": (".semantic_index", "SemanticIndexRecord"),
-    "SemanticSearchResult": (".semantic_index", "SemanticSearchResult"),
-    "SQLiteVecSemanticIndex": (".semantic_index", "SQLiteVecSemanticIndex"),
-    "SummaryProjectionIndexer": (".semantic_projection", "SummaryProjectionIndexer"),
-}
-
-__all__ = [
-    "OpenResult",
-    "SemanticProjectionSearchBackend",
-    "PIFSCommandExecutor",
-    "PageIndexFileSystem",
-    "RebuildableSemanticIndex",
-    "SearchResult",
-    "SemanticIndexRecord",
-    "SemanticSearchResult",
-    "SummaryProjectionIndexer",
-    "SQLiteVecSemanticIndex",
-]
-
-
-def __getattr__(name: str):
-    if name in _LAZY_EXPORTS:
-        module_name, attribute_name = _LAZY_EXPORTS[name]
-        value = getattr(import_module(module_name, __name__), attribute_name)
-        globals()[name] = value
-        return value
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
 
-def __dir__() -> list[str]:
-    return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS))
+__all__ = ["PageIndexFileSystem"]

pageindex/filesystem/_embedding_identity.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from urllib.parse import urlsplit, urlunsplit
+
+
+DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
+
+
+def normalize_base_url(value: str | None) -> str:
+    raw = str(value or DEFAULT_OPENAI_BASE_URL).strip()
+    parsed = urlsplit(raw)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError("embedding base URL must be an absolute HTTP(S) URL")
+    if parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise ValueError(
+            "embedding base URL must not contain credentials, query parameters, or a fragment"
+        )
+    return urlunsplit(
+        (
+            parsed.scheme.lower(),
+            parsed.netloc.lower(),
+            parsed.path.rstrip("/"),
+            "",
+            "",
+        )
+    )
+
+
+def normalize_model(value: str | None) -> str:
+    model = str(value or "").strip()
+    if not model:
+        raise ValueError("embedding model must not be empty")
+    return model

pageindex/filesystem/_projection_topology.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def projection_database_paths(index_dir: str | Path) -> tuple[Path, Path]:
+    index_dir = Path(index_dir).expanduser()
+    return index_dir / "summary.sqlite", index_dir / "embedding_cache.sqlite"
+
+
+def projection_database_path_present(path: str | Path) -> bool:
+    path = Path(path)
+    return path.exists() or path.is_symlink()
+
+
+def projection_database_pair(
+    index_dir: str | Path,
+) -> tuple[Path, Path] | None:
+    summary_path, cache_path = projection_database_paths(index_dir)
+    summary_exists = projection_database_path_present(summary_path)
+    cache_exists = projection_database_path_present(cache_path)
+    if not summary_exists and not cache_exists:
+        return None
+    if not summary_exists or not cache_exists:
+        raise RuntimeError(
+            "PIFS Summary Projection topology is incomplete; migrate this workspace with "
+            "pifs-data/scripts/migrate_pifs_workspace.py before opening it."
+        )
+    return summary_path, cache_path

pageindex/filesystem/_sqlite_schema.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import sqlite3
+from typing import Any
+
+
+def regular_table_names(connection: sqlite3.Connection) -> set[str]:
+    return {
+        str(row[0])
+        for row in connection.execute(
+            "SELECT name FROM sqlite_master "
+            "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
+        )
+    }
+
+
+def sqlite_schema_signature(
+    connection: sqlite3.Connection,
+    tables: set[str],
+) -> dict[str, Any]:
+    columns = {
+        table: tuple(
+            tuple(row)
+            for row in connection.execute(f'PRAGMA table_xinfo("{table}")')
+        )
+        for table in sorted(tables)
+    }
+    indexes: dict[str, tuple[tuple[Any, ...], ...]] = {}
+    foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {}
+    for table in sorted(tables):
+        table_indexes = []
+        for row in connection.execute(f'PRAGMA index_list("{table}")'):
+            name = str(row[1])
+            origin = str(row[3])
+            index_columns = tuple(
+                str(column[2])
+                for column in connection.execute(f'PRAGMA index_info("{name}")')
+            )
+            table_indexes.append(
+                (
+                    name if origin == "c" else None,
+                    int(row[2]),
+                    origin,
+                    int(row[4]),
+                    index_columns,
+                )
+            )
+        indexes[table] = tuple(sorted(table_indexes, key=repr))
+        foreign_keys[table] = tuple(
+            sorted(
+                (
+                    str(row[2]),
+                    str(row[3]),
+                    str(row[4]),
+                    str(row[5]),
+                    str(row[6]),
+                    str(row[7]),
+                )
+                for row in connection.execute(f'PRAGMA foreign_key_list("{table}")')
+            )
+        )
+    return {
+        "tables": tables,
+        "columns": columns,
+        "indexes": indexes,
+        "foreign_keys": foreign_keys,
+    }
+
+
+def normalized_table_sql(connection: sqlite3.Connection, table: str) -> str:
+    row = connection.execute(
+        "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?",
+        (table,),
+    ).fetchone()
+    return " ".join(str(row[0] if row else "").lower().split())

pageindex/filesystem/_workspace_consistency.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+
+def _readonly_connection(path: Path) -> sqlite3.Connection:
+    connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
+    connection.row_factory = sqlite3.Row
+    return connection
+
+
+def _inconsistent_workspace_error(detail: str) -> RuntimeError:
+    return RuntimeError(
+        f"Inconsistent PIFS workspace: {detail}; migrate this workspace with "
+        "pifs-data/scripts/migrate_pifs_workspace.py before opening it."
+    )
+
+
+def validate_catalog_root(catalog_path: str | Path) -> None:
+    try:
+        with _readonly_connection(Path(catalog_path)) as catalog:
+            root = catalog.execute(
+                "SELECT folder_id, parent_id, name FROM folders WHERE path = '/'"
+            ).fetchone()
+    except sqlite3.Error as exc:
+        raise _inconsistent_workspace_error(
+            "the catalog canonical root folder could not be read"
+        ) from exc
+    if (
+        root is None
+        or root["folder_id"] != "folder_root"
+        or root["parent_id"] is not None
+        or root["name"] != "/"
+    ):
+        raise _inconsistent_workspace_error(
+            "the catalog is missing its canonical root folder"
+        )
+
+
+def validate_workspace_consistency(
+    catalog_path: str | Path,
+    summary_path: str | Path,
+) -> None:
+    catalog_path = Path(catalog_path)
+    validate_catalog_root(catalog_path)
+    try:
+        with _readonly_connection(catalog_path) as catalog:
+            active_file_refs = {
+                str(row[0])
+                for row in catalog.execute(
+                    "SELECT file_ref FROM files WHERE deleted_at IS NULL"
+                )
+            }
+        with _readonly_connection(Path(summary_path)) as summary:
+            projected_file_refs = {
+                str(row[0])
+                for row in summary.execute(
+                    "SELECT file_ref FROM semantic_index_docs"
+                )
+            }
+    except sqlite3.Error as exc:
+        raise _inconsistent_workspace_error(
+            "catalog and Summary Projection references could not be read"
+        ) from exc
+    orphaned = sorted(projected_file_refs - active_file_refs)
+    if orphaned:
+        shown = ", ".join(orphaned[:5])
+        raise _inconsistent_workspace_error(
+            "Summary Projection file_ref values do not reference active catalog files: "
+            f"{shown}"
+        )

pageindex/filesystem/agent.py

@@ -35,9 +35,9 @@
 
 If the user asks what tools or capabilities you have, describe only the PIFS
 virtual shell capabilities available inside this workspace: tree, browse,
-stat, cat, grep, and ls as an alias for tree -L 1. Do not mention host runtime
-tools, SDK internals, or orchestration helpers that are not part of the PIFS
-shell.
+stat, cat, and grep. Describe PIFS Listing as tree <scope> -L 1. Do not mention
+host runtime tools, SDK internals, or orchestration helpers that are not part of
+the PIFS shell.
 
 If the user asks a workspace-related topic question without naming a specific
 file, treat it as a retrieval task. Start with tree to understand folder
@@ -52,7 +52,7 @@
 BASH_TOOL_DESCRIPTION = """
 Run a command in the PageIndex FileSystem virtual shell. This is not a real
 operating-system shell. The tool is read-only and always returns JSON.
-Use only: tree, browse, stat, cat, grep, and ls as an exact alias for tree -L 1.
+Use only: tree, browse, stat, cat, and grep.
 Start broad workspace questions with tree. Then choose a physical folder,
 inspect metadata-derived virtual axes such as @year or @ticker with tree, and
 copy returned scope paths into later commands. Run browse <scope> "<query>" for document
@@ -61,32 +61,31 @@
 candidates only, not final evidence. Use stat only for identity, status, or
 metadata. Use cat <file> --structure before any cat <file> --page N[-M] for
 that file. Use grep <query> <file> only as a single-document lexical fallback.
-Do not use find, recursive grep, folder grep, pipes, stat schema/field modes,
-browse spaces, implicit cat reads, or general knowledge when workspace evidence
-is not found.
+Other shell commands, pipes, stat modes, folder-wide lexical searches, and
+implicit cat reads are unsupported.
 """
 
 AGENT_TOOL_POLICY = """
 Tool policy:
 - The bash tool is a PageIndex virtual shell, not an operating-system shell.
 - The default agent tool surface is read-only.
-- Use only tree, browse, stat, cat, grep, and ls as an alias for tree -L 1.
+- Use only tree, browse, stat, cat, and grep. PIFS Listing is tree <scope> -L 1.
 - Folder paths such as /documents are positional command targets; never put folder paths in --where.
 - Start with tree to understand workspace and folder structure before document retrieval.
 - After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree <scope>/@field to inspect metadata-derived virtual value nodes, and copy returned scope paths instead of handwriting encoded values.
 - After choosing a folder or scope, use browse <scope> "<query>" for relevance-ranked document candidates; quote multi-word queries, for example browse /documents "Federal Reserve".
 - Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query.
 - If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder.
 - Only after that persistence protocol may you say the workspace lacks evidence.
-- browse uses summary retrieval only; do not use browse --space.
+- browse always uses summary retrieval.
 - Use metadata scope paths for exact filters. Keep --where as a fallback for OR, IN, or contains only when tree paths cannot express the predicate.
 - browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims.
 - For financial statement, ratio, or calculation questions, prefer citing a primary statement, reconciliation table, or official table when available; use summaries, MD&A, or footnotes as supporting evidence.
-- Use grep <query> <path|file_ref|document_id> for a selected single file only.
-- Do not use find, recursive grep, folder grep, pipes, schema browsing, batch metadata commands, browse spaces, implicit cat reads, or general-knowledge fallback.
+- Use grep <query> <path> for a selected single file only.
+- Other shell commands, pipes, schema browsing, batch metadata commands, folder-wide lexical searches, implicit cat reads, and general-knowledge fallback are unsupported.
 - Command errors are JSON; recover by trying an available command.
 - Use cat or grep to gather evidence before making source-backed claims.
-- Do not reconstruct a file path from a title. Use exact paths returned by PIFS commands, or use file_ref/document_id when available; quote paths that contain spaces.
+- Do not reconstruct a file path from a title. Use the exact path returned by PIFS commands and quote paths that contain spaces.
 - For broad topic, method, or "what solution" questions that are likely about the workspace, browse for candidate documents before asking the user to choose a document.
 - Use stat only for identity, metadata, or status questions. Do not run stat merely to understand what a document says.
 - Prefer target-first cat syntax with stable targets: cat <path> --structure, cat <path> --page 31-59.
@@ -100,7 +99,7 @@
 - Avoid fetching a broad page span unless page-level citation or verification is required.
 - Do not call cat --page <target> <start> <end>; if you need a page span, use cat <target> --page <start>-<end>.
 - For metadata or summary-field questions, run stat <target> for relevant files before answering.
-- Distinguish default/register metadata from caller-provided custom metadata when the evidence supports it.
+- Distinguish generated metadata from caller-provided custom metadata when the evidence supports it.
 - Do a final consistency check before answering: the headline number, calculation, unit, sign, and yes/no polarity must agree with each other.
 """