refactor(filesystem): simplify PIFS core surface

pageindex/filesystem/__init__.py

@@ -1,51 +1,3 @@
-from importlib import import_module
-from typing import TYPE_CHECKING
-
-from .commands import PIFSCommandExecutor
 from .core import PageIndexFileSystem
-from .types import OpenResult, SearchResult
-
-if TYPE_CHECKING:
-    from .semantic_projection import SemanticProjectionSearchBackend
-    from .semantic_projection import SummaryProjectionIndexer
-    from .semantic_index import (
-        RebuildableSemanticIndex,
-        SemanticIndexRecord,
-        SemanticSearchResult,
-        SQLiteVecSemanticIndex,
-    )
-
-_LAZY_EXPORTS = {
-    "SemanticProjectionSearchBackend": (".semantic_projection", "SemanticProjectionSearchBackend"),
-    "RebuildableSemanticIndex": (".semantic_index", "RebuildableSemanticIndex"),
-    "SemanticIndexRecord": (".semantic_index", "SemanticIndexRecord"),
-    "SemanticSearchResult": (".semantic_index", "SemanticSearchResult"),
-    "SQLiteVecSemanticIndex": (".semantic_index", "SQLiteVecSemanticIndex"),
-    "SummaryProjectionIndexer": (".semantic_projection", "SummaryProjectionIndexer"),
-}
-
-__all__ = [
-    "OpenResult",
-    "SemanticProjectionSearchBackend",
-    "PIFSCommandExecutor",
-    "PageIndexFileSystem",
-    "RebuildableSemanticIndex",
-    "SearchResult",
-    "SemanticIndexRecord",
-    "SemanticSearchResult",
-    "SummaryProjectionIndexer",
-    "SQLiteVecSemanticIndex",
-]
-
-
-def __getattr__(name: str):
-    if name in _LAZY_EXPORTS:
-        module_name, attribute_name = _LAZY_EXPORTS[name]
-        value = getattr(import_module(module_name, __name__), attribute_name)
-        globals()[name] = value
-        return value
-    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
-
 
-def __dir__() -> list[str]:
-    return sorted(set(globals()) | set(__all__) | set(_LAZY_EXPORTS))
+__all__ = ["PageIndexFileSystem"]

pageindex/filesystem/agent.py

@@ -35,9 +35,9 @@
 
 If the user asks what tools or capabilities you have, describe only the PIFS
 virtual shell capabilities available inside this workspace: tree, browse,
-stat, cat, grep, and ls as an alias for tree -L 1. Do not mention host runtime
-tools, SDK internals, or orchestration helpers that are not part of the PIFS
-shell.
+stat, cat, and grep. Describe PIFS Listing as tree <scope> -L 1. Do not mention
+host runtime tools, SDK internals, or orchestration helpers that are not part of
+the PIFS shell.
 
 If the user asks a workspace-related topic question without naming a specific
 file, treat it as a retrieval task. Start with tree to understand folder
@@ -52,7 +52,7 @@
 BASH_TOOL_DESCRIPTION = """
 Run a command in the PageIndex FileSystem virtual shell. This is not a real
 operating-system shell. The tool is read-only and always returns JSON.
-Use only: tree, browse, stat, cat, grep, and ls as an exact alias for tree -L 1.
+Use only: tree, browse, stat, cat, and grep.
 Start broad workspace questions with tree. Then choose a physical folder,
 inspect metadata-derived virtual axes such as @year or @ticker with tree, and
 copy returned scope paths into later commands. Run browse <scope> "<query>" for document
@@ -61,32 +61,31 @@
 candidates only, not final evidence. Use stat only for identity, status, or
 metadata. Use cat <file> --structure before any cat <file> --page N[-M] for
 that file. Use grep <query> <file> only as a single-document lexical fallback.
-Do not use find, recursive grep, folder grep, pipes, stat schema/field modes,
-browse spaces, implicit cat reads, or general knowledge when workspace evidence
-is not found.
+Other shell commands, pipes, stat modes, folder-wide lexical searches, and
+implicit cat reads are unsupported.
 """
 
 AGENT_TOOL_POLICY = """
 Tool policy:
 - The bash tool is a PageIndex virtual shell, not an operating-system shell.
 - The default agent tool surface is read-only.
-- Use only tree, browse, stat, cat, grep, and ls as an alias for tree -L 1.
+- Use only tree, browse, stat, cat, and grep. PIFS Listing is tree <scope> -L 1.
 - Folder paths such as /documents are positional command targets; never put folder paths in --where.
 - Start with tree to understand workspace and folder structure before document retrieval.
 - After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree <scope>/@field to inspect metadata-derived virtual value nodes, and copy returned scope paths instead of handwriting encoded values.
 - After choosing a folder or scope, use browse <scope> "<query>" for relevance-ranked document candidates; quote multi-word queries, for example browse /documents "Federal Reserve".
 - Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query.
 - If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder.
 - Only after that persistence protocol may you say the workspace lacks evidence.
-- browse uses summary retrieval only; do not use browse --space.
+- browse always uses summary retrieval.
 - Use metadata scope paths for exact filters. Keep --where as a fallback for OR, IN, or contains only when tree paths cannot express the predicate.
 - browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims.
 - For financial statement, ratio, or calculation questions, prefer citing a primary statement, reconciliation table, or official table when available; use summaries, MD&A, or footnotes as supporting evidence.
-- Use grep <query> <path|file_ref|document_id> for a selected single file only.
-- Do not use find, recursive grep, folder grep, pipes, schema browsing, batch metadata commands, browse spaces, implicit cat reads, or general-knowledge fallback.
+- Use grep <query> <path> for a selected single file only.
+- Other shell commands, pipes, schema browsing, batch metadata commands, folder-wide lexical searches, implicit cat reads, and general-knowledge fallback are unsupported.
 - Command errors are JSON; recover by trying an available command.
 - Use cat or grep to gather evidence before making source-backed claims.
-- Do not reconstruct a file path from a title. Use exact paths returned by PIFS commands, or use file_ref/document_id when available; quote paths that contain spaces.
+- Do not reconstruct a file path from a title. Use the exact path returned by PIFS commands and quote paths that contain spaces.
 - For broad topic, method, or "what solution" questions that are likely about the workspace, browse for candidate documents before asking the user to choose a document.
 - Use stat only for identity, metadata, or status questions. Do not run stat merely to understand what a document says.
 - Prefer target-first cat syntax with stable targets: cat <path> --structure, cat <path> --page 31-59.
@@ -100,7 +99,7 @@
 - Avoid fetching a broad page span unless page-level citation or verification is required.
 - Do not call cat --page <target> <start> <end>; if you need a page span, use cat <target> --page <start>-<end>.
 - For metadata or summary-field questions, run stat <target> for relevant files before answering.
-- Distinguish default/register metadata from caller-provided custom metadata when the evidence supports it.
+- Distinguish generated metadata from caller-provided custom metadata when the evidence supports it.
 - Do a final consistency check before answering: the headline number, calculation, unit, sign, and yes/no polarity must agree with each other.
 """
 

pageindex/filesystem/cli.py

@@ -26,6 +26,13 @@
 ANSI_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|.)")
 PIFS_CONFIG_FILE_ENV = "PIFS_CONFIG_FILE"
 PIFS_WORKSPACE_ENV = "PIFS_WORKSPACE"
+PERSISTED_CONFIG_KEYS = {
+    "workspace",
+    "embedding_base_url",
+    "embedding_model",
+    "embedding_dimensions",
+    "embedding_timeout",
+}
 
 
 def _config_path() -> Path:
@@ -45,14 +52,23 @@ def _read_config() -> dict[str, str]:
         payload = json.load(handle)
     if not isinstance(payload, dict):
         raise ValueError(f"invalid PIFS config file: {path}")
-    return {str(key): str(value) for key, value in payload.items() if value is not None}
+    return {
+        str(key): str(value)
+        for key, value in payload.items()
+        if key in PERSISTED_CONFIG_KEYS and value is not None
+    }
 
 
 def _write_config(config: dict[str, str]) -> Path:
     path = _config_path()
     path.parent.mkdir(parents=True, exist_ok=True)
+    persisted = {
+        key: value
+        for key, value in config.items()
+        if key in PERSISTED_CONFIG_KEYS and value is not None
+    }
     with path.open("w", encoding="utf-8") as handle:
-        json.dump(config, handle, indent=2, sort_keys=True)
+        json.dump(persisted, handle, indent=2, sort_keys=True)
         handle.write("\n")
     return path
 
@@ -182,8 +198,7 @@ def _filesystem_embedding_config() -> dict[str, object]:
     config_values = _read_config()
     config: dict[str, object] = {}
     base_url = config_values.get("embedding_base_url")
-    api_key = config_values.get("embedding_api_key")
-    provider = config_values.get("embedding_provider")
+    api_key = os.environ.get("PIFS_EMBEDDING_API_KEY") or os.environ.get("OPENAI_API_KEY")
     model = config_values.get("embedding_model")
     dimensions = _optional_int(
         "embedding_dimensions",
@@ -193,8 +208,6 @@ def _filesystem_embedding_config() -> dict[str, object]:
         "embedding_timeout",
         config_values.get("embedding_timeout"),
     )
-    if provider and provider.strip():
-        config["summary_projection_embedding_provider"] = provider.strip()
     if model and model.strip():
         config["summary_projection_embedding_model"] = model.strip()
     if dimensions is not None:
@@ -344,7 +357,6 @@ def _run_add(argv: list[str], *, workspace: str) -> int:
     filesystem = _filesystem_from_workspace(workspace)
     info = filesystem.add_file(args.physical_path, args.virtual_target)
     print(f"added: {info.get('path')}")
-    print(f"file_ref: {info['file_ref']}")
     return 0
 
 
@@ -371,7 +383,15 @@ def _run_setmeta(argv: list[str], *, workspace: str) -> int:
 
     filesystem = _filesystem_from_workspace(workspace)
     info = filesystem.set_metadata(args.target, metadata, clear=args.clear)
-    print(json.dumps(info, ensure_ascii=False, sort_keys=True))
+    document = {
+        "path": args.target if args.target.startswith("/") else info.get("path"),
+        "document_id": info.get("external_id"),
+        "title": info.get("title"),
+        "status": info.get("pageindex_tree_status"),
+        "metadata": info.get("metadata", {}),
+        "metadata_status": info.get("metadata_status", {}),
+    }
+    print(json.dumps(document, ensure_ascii=False, sort_keys=True))
     return 0