Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion DESIGN/live-mem/MCP_TOOLS_SPEC.md
Original file line number Diff line number Diff line change
Expand Up @@ -452,15 +452,21 @@ Synchronizes the bank into Graph Memory. Deletes old documents and re-ingests up

```python
@mcp.tool()
async def graph_push(space_id: str) -> dict:
async def graph_push(space_id: str, include_volatile: bool = False) -> dict:
```

**Behavior**:
- The space must first be connected via `graph_connect`
- Skips volatile bank files by default (`GRAPH_PUSH_VOLATILE_FILES`, default:
`activeContext.md,progress.md`)
- `include_volatile=True` re-includes volatile files for explicit debug /
migration only, requires `manage`/`admin`, and emits an audit log
- Intelligent delete + re-ingestion (graph recalculation)
- Orphan cleanup (files removed from the bank)
- ~10-30s per file (LLM entity/relation extraction + embeddings)
- Updates metrics in `_meta.json`
- Response keeps `pushed` as the numeric counter and adds `pushed_files`,
`skipped_volatile`, and a `warning` when files are skipped

---

Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ docker compose logs -f live-mem-service --tail 50 # Logs
| Tool | Parameters | Description |
| ------------------ | ---------------------------------------------------- | --------------------------------------------------------------------------------------------------------- |
| `graph_connect` | `space_id`, `url`, `token`, `memory_id`, `ontology?` | Connects a space to Graph Memory. Tests connection, creates memory if needed. Default ontology: `general` |
| `graph_push` | `space_id` | Synchronizes bank → graph. Smart delete + re-ingest, orphan cleanup. ~30s/file |
| `graph_push` | `space_id`, `include_volatile?=false` | Synchronizes non-volatile bank files → graph. Smart delete + re-ingest, orphan cleanup. ~30s/file |
| `graph_status` | `space_id` | Connection status + graph stats (documents, entities, relations, top entities, documents list) |
| `graph_disconnect` | `space_id` | Disconnects (data remains in graph) |

Expand Down Expand Up @@ -374,7 +374,7 @@ docker compose logs -f live-mem-service --tail 50 # Logs
>
> Therefore, **`graph_push` is NOT a routine action**: pushing the full bank into the graph teaches it transient content that a later compaction strands as stale. Routine flows should ingest **canonical repository documents** directly into Graph Memory from the agent / tooling layer, using stable `source_path` keys. `graph_push` remains available for one-off bootstrap and explicit debug / migration only.
>
> In particular, `activeContext.md` and `progress.md` **must never** end up in Graph Memory. A future revision (tracked in [`DESIGN/live-mem/EVOLUTION_LIVE_GRAPH_INTEGRATION.md`](DESIGN/live-mem/EVOLUTION_LIVE_GRAPH_INTEGRATION.md)) will turn this into a server-side guardrail. See [`WORKSPACE_CLINE_ADVANCE_RULES.md`](WORKSPACE_CLINE_ADVANCE_RULES.md) for the agent-side template.
> By default, `graph_push` skips volatile bank files listed by `GRAPH_PUSH_VOLATILE_FILES` (`activeContext.md,progress.md`). An operator can pass `include_volatile=true` for explicit debug / migration pushes only; this requires `manage`/`admin` permission and emits an audit log. See [`WORKSPACE_CLINE_ADVANCE_RULES.md`](WORKSPACE_CLINE_ADVANCE_RULES.md) for the agent-side template.

Live Memory can push its Memory Bank into a [Graph Memory](https://github.com/Cloud-Temple/graph-memory) instance for long-term memory. The knowledge graph extracts entities, relations, and embeddings from bank files.

Expand Down
6 changes: 6 additions & 0 deletions src/live_mem/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,12 @@ def _normalize_proxy_url(cls, v: str | None) -> str | None:
# ─── Response limits ──────────────────────────────────────
response_max_bytes: int = 512 * 1024 # Max response body size (512 KB)

# ─── Graph push guardrails ────────────────────────────────
# CSV of Memory Bank filenames excluded from graph_push by default.
# Operators can override with GRAPH_PUSH_VOLATILE_FILES, or opt in per
# call with include_volatile=True (manage/admin only).
graph_push_volatile_files: str = "activeContext.md,progress.md"

# ─── Admin console /api/tool (ADM-05 fix) ─────────────
api_tool_max_body_bytes: int = 1_048_576 # Max request body for /api/tool (1 MB)

Expand Down
99 changes: 95 additions & 4 deletions src/live_mem/core/graph_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,15 +33,55 @@
import base64
import asyncio
import logging
import unicodedata
from datetime import datetime, timezone

from mcp import ClientSession
from mcp.client.streamable_http import streamablehttp_client

from ..config import get_settings
from .storage import get_storage, bank_relpath
from .models import GraphMemoryConfig

logger = logging.getLogger("live_mem.graph_bridge")
audit_logger = logging.getLogger("live_mem.audit")


def _parse_graph_push_volatile_files(value: str | None = None) -> set[str]:
"""Parse the CSV volatile-file policy into normalized basenames."""
if value is None:
value = get_settings().graph_push_volatile_files

files: set[str] = set()
for raw in value.split(","):
normalized = _volatile_policy_key(raw)
if normalized:
files.add(normalized)
return files


def _volatile_policy_key(filename: str) -> str:
"""Normalize a bank relpath for volatile-policy matching."""
clean = unicodedata.normalize("NFC", str(filename).strip()).replace("\\", "/")
basename = clean.rsplit("/", 1)[-1].strip()
return basename


def _split_volatile_bank_files(
bank_files: dict[str, str],
volatile_files: set[str],
) -> tuple[dict[str, str], list[str]]:
"""Return pushable bank files and skipped volatile filenames."""
pushable: dict[str, str] = {}
skipped: list[str] = []

for filename, content in bank_files.items():
if _volatile_policy_key(filename) in volatile_files:
skipped.append(filename)
else:
pushable[filename] = content

return pushable, skipped


# ─────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -378,7 +418,12 @@ async def connect(
# PUSH — Pousser la bank dans graph-memory
# ─────────────────────────────────────────────────────────

async def push(self, space_id: str) -> dict:
async def push(
self,
space_id: str,
include_volatile: bool = False,
audit_caller: str | None = None,
) -> dict:
"""
Pousse les fichiers bank du space dans graph-memory.

Expand All @@ -396,6 +441,8 @@ async def push(self, space_id: str) -> dict:

Args:
space_id: Identifiant du space live-memory
include_volatile: Inclure explicitement les fichiers bank volatiles
audit_caller: Nom de l'appelant à tracer si include_volatile=True

Returns:
{"status": "ok", "pushed": N, "deleted": N, "errors": N, ...}
Expand Down Expand Up @@ -429,17 +476,43 @@ async def push(self, space_id: str) -> dict:
bank_files = {
bank_relpath(item["key"], space_id): item["content"] for item in bank_data
}
volatile_files = _parse_graph_push_volatile_files()
push_bank_files, skipped_volatile = (
(bank_files, [])
if include_volatile
else _split_volatile_bank_files(bank_files, volatile_files)
)

if not bank_files:
return {
"status": "ok",
"space_id": space_id,
"message": "Aucun fichier bank à pousser",
"pushed": 0,
"pushed_files": [],
"skipped_volatile": [],
"deleted": 0,
"errors": 0,
}

if include_volatile:
volatile_in_scope = [
filename
for filename in bank_files
if _volatile_policy_key(filename) in volatile_files
]
audit_logger.info(
json.dumps(
{
"event": "graph_push_volatile_optin",
"caller": audit_caller or "unknown",
"space_id": space_id,
"files": volatile_in_scope,
},
ensure_ascii=False,
)
)

# Connexion à graph-memory
gm = GraphMemoryClient(config.url, config.token, timeout=180.0)

Expand All @@ -457,18 +530,20 @@ async def push(self, space_id: str) -> dict:
existing_docs.add(doc.get("filename", ""))

logger.info(
"Push '%s' → '%s' : %d fichiers bank, %d docs existants",
"Push '%s' → '%s' : %d fichiers bank, %d poussables, %d volatiles sautés, %d docs existants",
space_id,
memory_id,
len(bank_files),
len(push_bank_files),
len(skipped_volatile),
len(existing_docs),
)

# 2. Construire le batch d'appels (delete + ingest pour chaque fichier)
calls = []
call_metadata = [] # Pour tracker quel appel fait quoi

for filename, content in bank_files.items():
for filename, content in push_bank_files.items():
# Si le document existe → supprimer d'abord
if filename in existing_docs:
calls.append(
Expand Down Expand Up @@ -498,6 +573,9 @@ async def push(self, space_id: str) -> dict:
call_metadata.append(("ingest", filename))

# 3. Nettoyage des orphelins
# Important: orphan cleanup stays scoped to the full bank, not the
# filtered push set. Otherwise a default push would delete already
# ingested volatile docs solely because this guard skipped re-ingest.
orphan_docs = existing_docs - set(bank_files.keys())
for orphan in orphan_docs:
calls.append(
Expand All @@ -512,10 +590,11 @@ async def push(self, space_id: str) -> dict:
call_metadata.append(("clean", orphan))

# 4. Exécuter tout le batch dans une seule session
results = await gm.call_tools_batch(calls)
results = await gm.call_tools_batch(calls) if calls else []

# 5. Analyser les résultats
pushed = 0
pushed_files = []
deleted_before_reingest = 0
cleaned = 0
errors = 0
Expand Down Expand Up @@ -547,6 +626,7 @@ async def push(self, space_id: str) -> dict:
else:
if action == "ingest":
pushed += 1
pushed_files.append(filename)
logger.info("Ingéré '%s'", filename)
elif action == "delete":
deleted_before_reingest += 1
Expand Down Expand Up @@ -580,12 +660,23 @@ async def push(self, space_id: str) -> dict:
"space_id": space_id,
"memory_id": memory_id,
"pushed": pushed,
"pushed_files": pushed_files,
"skipped_volatile": skipped_volatile,
"deleted_before_reingest": deleted_before_reingest,
"cleaned_orphans": cleaned,
"errors": errors,
"duration_seconds": duration,
}

if skipped_volatile:
result["warning"] = (
"Volatile bank files were skipped by default; pass "
"include_volatile=True with manage/admin permission for explicit "
"debug or migration pushes."
)
if not pushed_files and skipped_volatile and errors == 0:
result["message"] = "No non-volatile bank files to push"

if error_details:
result["error_details"] = error_details

Expand Down
58 changes: 51 additions & 7 deletions src/live_mem/tools/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
"""

import ipaddress
import json
import logging
from typing import Annotated, Optional
from urllib.parse import urlparse

Expand All @@ -34,6 +36,7 @@
# metadata cloud 169.254.169.254, ...). L'URL persiste ensuite dans
# `_meta.json` et est ré-utilisée à chaque graph_push.
_ALLOWED_GM_SCHEMES = ("http", "https")
_audit_logger = logging.getLogger("live_mem.audit")


def _validate_gm_url(url: str) -> Optional[str]:
Expand Down Expand Up @@ -203,6 +206,16 @@ async def graph_push(
space_id: Annotated[
str, Field(description="Identifiant du space live-memory à synchroniser")
],
include_volatile: Annotated[
bool,
Field(
default=False,
description=(
"Inclure explicitement les fichiers bank volatiles "
"(activeContext.md, progress.md par défaut). Requiert manage/admin."
),
),
] = False,
) -> dict:
"""
⚠️ **Advanced / debug tool — NOT for routine flows.**
Expand Down Expand Up @@ -254,23 +267,30 @@ async def graph_push(
stabilised bank,
- explicit debug / migration scenarios under operator control.

In particular, `activeContext.md` and `progress.md` **must not**
end up in Graph Memory. A future revision (tracked in
`DESIGN/live-mem/EVOLUTION_LIVE_GRAPH_INTEGRATION.md`) will turn this
into a server-side guardrail. For now, the contract is doctrinal:
do not invoke this tool as part of session-end consolidation.
By default, volatile bank files (`activeContext.md`, `progress.md`,
configurable via `GRAPH_PUSH_VOLATILE_FILES`) are skipped and returned
in `skipped_volatile`. `include_volatile=True` is an explicit
debug / migration opt-in, requires manage/admin permission, and emits
an audit log.

See `WORKSPACE_CLINE_ADVANCE_RULES.md`, README "Live ↔ Graph
Memory" section, and `DESIGN/live-mem/ARCHITECTURE.md` for the
full responsibility separation.

Args:
space_id: Identifiant du space live-memory
include_volatile: Inclure explicitement les fichiers volatiles
(manage/admin uniquement)

Returns:
Métriques de push : fichiers poussés, nettoyés, erreurs, durée
"""
from ..auth.context import check_access, check_write_permission
from ..auth.context import (
check_access,
check_manage_permission,
check_write_permission,
current_token_info,
)
from ..core.graph_bridge import get_graph_bridge

try:
Expand All @@ -282,7 +302,31 @@ async def graph_push(
if write_err:
return write_err

return await get_graph_bridge().push(space_id)
caller = "unknown"
token_info = current_token_info.get() or {}
if token_info.get("client_name"):
caller = token_info["client_name"]

if include_volatile:
manage_err = check_manage_permission()
if manage_err:
return manage_err
_audit_logger.info(
json.dumps(
{
"event": "graph_push_include_volatile_requested",
"caller": caller,
"space_id": space_id,
},
ensure_ascii=False,
)
)

return await get_graph_bridge().push(
space_id,
include_volatile=include_volatile,
audit_caller=caller,
)
except Exception as e:
from ..auth.context import safe_error

Expand Down
Loading