feat(export): add --with-sources to wire Obsidian notes to source content (#1968)

CHANGELOG.md

@@ -4,6 +4,7 @@ Full release notes with details on each version: [GitHub Releases](https://githu
 
 ## 0.9.18 (2026-07-17)
 
+- Feature: `graphify export obsidian --with-sources` wires each node note to its source content instead of leaving it a bare stub (#1968, thanks @SinghAman21). Every distinct `source_file` is copied into the vault under `sources/` as a real note (`_src_<flattened-path>.md`, collision-safe) — markdown/text verbatim, other sources embedded in a syntax-highlighted fence — and each node note gains a `> [!info] Source` callout wikilinking to it, displayed with the original path. Paths are resolved relative to the scan root (the `#1941` layout) with the out dir as a fallback for pre-0.9.17 graphs; `--source-root PATH` overrides where they're resolved. The source notes are graphify-owned, so a re-export without the flag prunes them and the vault returns to plain notes; a node whose source can't be resolved on disk (or is over a 2&nbsp;MB cap) simply gets no callout rather than a dangling link.
 - Fix: an incomplete extraction no longer force-writes a partial graph over a complete one (#1951, thanks @TPAteeq). A crashed AST/semantic pass, a some-chunks-failed run, or a walk that couldn't fully enumerate the corpus (permission-denied subtree) produced a smaller graph that the `to_json(force=True)` path wrote anyway, bypassing the #479 shrink guard; the `--no-cluster` raw dump had no guard at all. Both paths now refuse to overwrite a larger existing graph when the run was incomplete (exit 1, nothing written) unless `--allow-partial` is passed, and a present-but-unparseable existing graph fails closed (a corrupt/mid-write file could be hiding a complete graph). `detect()`'s `walk_errors` now count as incomplete too.
 - Fix: `graph.json`, `manifest.json`, and the other JSON artifacts are now written atomically (#1952, thanks @TPAteeq). A kill, OOM, or ENOSPC mid-write used to leave a truncated file — and a truncated `graph.json` then wedged every later run via the shrink guard's fail-safe. Writes now go to a temp file in the same directory and `os.replace` into place (writing *through* a symlink so shared-store setups keep working); the writers the original change missed (the `--no-cluster` dump, `merge-graphs`/`merge-chunks`/`merge-semantic`, the analysis/labels sidecars, and the global graph/manifest) are routed through it too.
 - Fix: truncated LLM chunks are no longer promoted to the semantic cache as complete (#1950, thanks @TPAteeq). A chunk that hit the output-token limit and couldn't be split further was cached and manifest-stamped as authoritative, so its incomplete node set replayed forever. Such chunks are now marked partial (treated as a cache miss and re-dispatched), including the common case where the truncation parses to *nothing*: the give-up sites record the chunk's own files independently of parsed items, so a sliced document whose later slice truncated empty is no longer stamped complete, and a clean slice can't re-promote a partial entry.

graphify/cli.py

@@ -1738,7 +1738,7 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
             print("  html      [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr)
             print("  callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr)
             print("            [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr)
-            print("  obsidian  [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr)
+            print("  obsidian  [--graph PATH] [--labels PATH] [--dir PATH] [--with-sources] [--source-root PATH]", file=sys.stderr)
             print("  wiki      [--graph PATH] [--labels PATH]", file=sys.stderr)
             print("  svg       [--graph PATH] [--labels PATH]", file=sys.stderr)
             print("  graphml   [--graph PATH]", file=sys.stderr)
@@ -1767,6 +1767,11 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
         node_limit = 5000
         no_viz = False
         obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian"
+        # --with-sources (#1968): copy each node's source file into the vault and
+        # wire a callout to it. --source-root overrides where the (scan-root-
+        # relative) source_file paths are resolved on disk.
+        obsidian_with_sources = False
+        obsidian_source_root: Path | None = None
         # Shared push-connection settings for the graph-database sinks (neo4j,
         # falkordb), parsed from the generic --push/--user/--password flags below.
         push_uri: str | None = None
@@ -1828,6 +1833,10 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
                 no_viz = True; i += 1
             elif a == "--dir" and i + 1 < len(args):
                 obsidian_dir = Path(args[i + 1]); i += 2
+            elif a == "--with-sources":
+                obsidian_with_sources = True; i += 1
+            elif a == "--source-root" and i + 1 < len(args):
+                obsidian_source_root = Path(args[i + 1]).expanduser(); i += 2
             elif a == "--push" and i + 1 < len(args):
                 push_uri = args[i + 1]; i += 2
             elif a == "--user" and i + 1 < len(args):
@@ -1973,9 +1982,22 @@ def _to_simple(g: "_nx.Graph") -> "_nx.Graph":
 
         elif subcmd == "obsidian":
             from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas
+            # source_file is stored relative to the scan root (#1941); resolve it
+            # against an explicit --source-root, else the graph's scan root
+            # (out_dir's parent under the default <root>/graphify-out/ layout) with
+            # the out_dir itself as a fallback for graphs written by <=0.9.16 that
+            # stored source_file relative to the out dir (#1968).
+            if obsidian_with_sources:
+                _src_roots = ([obsidian_source_root] if obsidian_source_root
+                              else [out_dir.parent, out_dir])
+            else:
+                _src_roots = None
             n = _to_obsidian(G, communities, str(obsidian_dir),
-                             community_labels=labels or None, cohesion=cohesion or None)
+                             community_labels=labels or None, cohesion=cohesion or None,
+                             with_sources=obsidian_with_sources, source_root=_src_roots)
             print(f"Obsidian vault: {n} notes in {obsidian_dir}/")
+            if obsidian_with_sources:
+                print(f"  --with-sources: source notes in {obsidian_dir}/sources/")
             _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"),
                        community_labels=labels or None)
             print(f"Canvas: {obsidian_dir}/graph.canvas")

graphify/export.py

@@ -451,19 +451,137 @@ def _dedup_node_filenames(G: nx.Graph, safe_name) -> dict[str, str]:
     return node_filenames
 
 
+# --with-sources (#1968): map a source file's extension to a fenced-code
+# language hint so non-markdown sources embed with syntax highlighting.
+_SOURCE_FENCE_LANG = {
+    "py": "python", "js": "javascript", "jsx": "javascript", "ts": "typescript",
+    "tsx": "typescript", "rs": "rust", "go": "go", "java": "java", "rb": "ruby",
+    "php": "php", "c": "c", "h": "c", "cpp": "cpp", "hpp": "cpp", "cc": "cpp",
+    "cs": "csharp", "swift": "swift", "kt": "kotlin", "scala": "scala",
+    "sh": "bash", "bash": "bash", "zsh": "bash", "sql": "sql", "yaml": "yaml",
+    "yml": "yaml", "toml": "toml", "json": "json", "html": "html", "css": "css",
+    "xml": "xml", "lua": "lua", "r": "r", "jl": "julia",
+}
+
+# Extensions Obsidian renders as prose, so their source note copies the file
+# verbatim rather than wrapping it in a code fence.
+_SOURCE_MARKDOWN_EXT = {"md", "mdx", "markdown", "qmd", "txt", ""}
+
+# Skip embedding a "source" larger than this (a mis-referenced binary/asset)
+# so --with-sources never bloats the vault with megabytes of noise (#1968).
+_MAX_SOURCE_BYTES = 2_000_000
+
+
+def _resolve_source_path(source_file: str, roots: list[Path]) -> "Path | None":
+    """Resolve a node's ``source_file`` to an existing readable file on disk.
+
+    ``source_file`` is stored relative to the scan root (#1941), so we try each
+    candidate root in order; an absolute path is tried as-is first. Returns the
+    first existing regular file, or None when nothing resolves."""
+    if not source_file:
+        return None
+    p = Path(source_file)
+    candidates: list[Path] = []
+    if p.is_absolute():
+        candidates.append(p)
+    else:
+        candidates.extend(r / source_file for r in roots)
+    for c in candidates:
+        try:
+            if c.is_file():
+                return c
+        except OSError:
+            continue
+    return None
+
+
+def _flatten_source_name(source_file: str) -> str:
+    """Flatten a (possibly nested, possibly Windows) source path into a single
+    collision-safe note stem prefixed ``_src_`` (e.g. ``en/best-practices.md``
+    -> ``_src_en_best-practices``)."""
+    norm = source_file.replace("\\", "/")
+    parts = [seg for seg in norm.split("/") if seg not in ("", ".", "..")]
+    flat = "_".join(parts) if parts else "source"
+    # Drop a trailing markdown-ish extension so the note isn't "_src_x.md.md".
+    flat = re.sub(r"\.(md|mdx|qmd|markdown)$", "", flat, flags=re.IGNORECASE)
+    # Same unsafe-char strip the node filenames use, so wikilinks stay valid.
+    flat = re.sub(r'[\\/*?:"<>|#^[\]]', "", flat).strip() or "source"
+    return _cap_filename("_src_" + flat)
+
+
+def _source_note_body(path: Path, source_file: str) -> "str | None":
+    """Build the markdown body for a copied source note, or None if the file is
+    unreadable or over the size cap. Markdown-like files are copied verbatim
+    (their own frontmatter/tags fall below our callout, so they no longer parse
+    as vault frontmatter); everything else is embedded in a code fence."""
+    try:
+        if path.stat().st_size > _MAX_SOURCE_BYTES:
+            return None
+    except OSError:
+        return None
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return None
+    header = f"> [!note] Source file\n> `{_yaml_str(source_file).strip(chr(34))}`\n\n"
+    ext = path.suffix.lower().lstrip(".")
+    if ext in _SOURCE_MARKDOWN_EXT:
+        return header + text
+    lang = _SOURCE_FENCE_LANG.get(ext, ext)
+    return header + "```" + lang + "\n" + text.rstrip("\n") + "\n```\n"
+
+
+def _write_source_notes(G: nx.Graph, roots: list[Path], owned_write) -> dict[str, str]:
+    """Copy every referenced, resolvable source file into ``sources/`` as one
+    note per distinct ``source_file`` and return a map ``source_file`` string ->
+    wikilink stem for the node notes to callout. Names are case-fold deduped so
+    two paths that flatten alike stay distinct on case-insensitive filesystems."""
+    source_note_for: dict[str, str] = {}
+    used: set[str] = set()
+    for _node_id, data in G.nodes(data=True):
+        sf = data.get("source_file", "")
+        if not sf or sf in source_note_for:
+            continue
+        resolved = _resolve_source_path(sf, roots)
+        if resolved is None:
+            continue
+        body = _source_note_body(resolved, sf)
+        if body is None:
+            continue
+        base = _flatten_source_name(sf)
+        stem = base
+        n = 1
+        while stem.lower() in used:
+            stem = f"{base}_{n}"
+            n += 1
+        if owned_write(f"sources/{stem}.md", body):
+            used.add(stem.lower())
+            source_note_for[sf] = stem
+    return source_note_for
+
+
 def to_obsidian(
     G: nx.Graph,
     communities: dict[int, list[str]],
     output_dir: str,
     community_labels: dict[int, str] | None = None,
     cohesion: dict[int, float] | None = None,
+    with_sources: bool = False,
+    source_root: "str | Path | list | tuple | None" = None,
 ) -> int:
     """Export graph as an Obsidian vault - one .md file per node with [[wikilinks]],
     plus one _COMMUNITY_name.md overview note per community (sorted to top by underscore prefix).
 
     Open the output directory as a vault in Obsidian to get an interactive
     graph view with community colors and full-text search over node metadata.
 
+    When ``with_sources`` is set (#1968), each node's ``source_file`` is copied
+    into ``sources/`` as a real note and every node note gets a callout linking
+    to it, so the vault is browsable standalone instead of every note reading as
+    a bare stub. ``source_file`` is stored relative to the scan root, so
+    ``source_root`` (default: the export's out dir and its parent) is where those
+    relative paths are resolved on disk.
+
     Returns the number of node notes + community notes written.
     """
     out = Path(output_dir)
@@ -512,6 +630,21 @@ def safe_name(label: str) -> str:
 
     node_filename = _dedup_node_filenames(G, safe_name)
 
+    # --with-sources (#1968): copy each referenced source file into sources/ and
+    # remember which source_file strings got a note, so node notes can callout to
+    # them below. Roots to resolve the (scan-root-relative) source_file against:
+    # an explicit source_root, else the out dir and its parent (the default
+    # <scanroot>/graphify-out/ layout puts the scan root one level up).
+    source_note_for: dict[str, str] = {}
+    if with_sources:
+        if source_root is None:
+            roots = [out.parent, out]
+        elif isinstance(source_root, (list, tuple)):
+            roots = [Path(r) for r in source_root]
+        else:
+            roots = [Path(source_root)]
+        source_note_for = _write_source_notes(G, roots, _owned_write)
+
     # Helper: compute dominant confidence for a node across all its edges
     def _dominant_confidence(node_id: str) -> str:
         confs = []
@@ -567,6 +700,19 @@ def _dominant_confidence(node_id: str) -> str:
             lines.append(f"  - {tag}")
         lines += ["---", "", f"# {label}", ""]
 
+        # --with-sources: a callout linking this note to its copied source note,
+        # displayed with the original source_file path (#1968).
+        node_sf = data.get("source_file", "")
+        src_stem = source_note_for.get(node_sf)
+        if src_stem:
+            # A ']' or '|' in the display path would truncate the wikilink alias.
+            disp = node_sf.replace("|", "-").replace("[", "(").replace("]", ")")
+            lines += [
+                "> [!info] Source",
+                f"> [[{src_stem}|{disp}]]",
+                "",
+            ]
+
         # Outgoing edges as wikilinks
         neighbors = list(G.neighbors(node_id))
         if neighbors:

tests/test_obsidian_with_sources.py

@@ -0,0 +1,112 @@
+"""Tests for `--with-sources` on the Obsidian export (issue #1968).
+
+Without the flag, node notes are bare stubs (frontmatter + Connections). With
+it, each node's `source_file` is copied into `sources/` as a real note and every
+node note gets a callout linking to that source, so the vault is usable
+standalone.
+"""
+import networkx as nx
+
+from graphify.export import to_obsidian
+
+
+def _graph():
+    G = nx.Graph()
+    # A markdown doc (copied verbatim) and a code file (fenced), plus a node
+    # whose source_file does not exist on disk (must get no callout).
+    G.add_node("n0", label="Best Practices", file_type="document",
+               source_file="en/best-practices.md", community=0)
+    G.add_node("n1", label="foo", file_type="code", source_file="util.py", community=0)
+    G.add_node("n2", label="Ghost", file_type="document",
+               source_file="missing/nope.md", community=0)
+    G.add_edge("n0", "n1", relation="mentions", confidence="EXTRACTED")
+    return G, {0: ["n0", "n1", "n2"]}
+
+
+def _make_sources(root):
+    (root / "en").mkdir()
+    (root / "en" / "best-practices.md").write_text(
+        "---\ntitle: BP\n---\n# Best Practices\n\nUse graphify wisely.\n", encoding="utf-8")
+    (root / "util.py").write_text("def foo():\n    return 42\n", encoding="utf-8")
+
+
+def test_default_export_has_no_source_notes(tmp_path):
+    """Absent the flag, behaviour is unchanged: no sources/ dir, no callout."""
+    _make_sources(tmp_path)
+    G, comms = _graph()
+    vault = tmp_path / "vault"
+    to_obsidian(G, comms, str(vault))
+    assert not (vault / "sources").exists()
+    assert "> [!info] Source" not in (vault / "Best Practices.md").read_text(encoding="utf-8")
+
+
+def test_with_sources_copies_and_wires_notes(tmp_path):
+    _make_sources(tmp_path)
+    G, comms = _graph()
+    vault = tmp_path / "vault"
+    to_obsidian(G, comms, str(vault), with_sources=True, source_root=tmp_path)
+
+    # Source notes are copied under sources/ with collision-safe flattened names.
+    src_dir = vault / "sources"
+    assert (src_dir / "_src_en_best-practices.md").exists()
+    assert (src_dir / "_src_util.py.md").exists()
+
+    # Markdown is copied verbatim; code is embedded in a fenced block.
+    md_src = (src_dir / "_src_en_best-practices.md").read_text(encoding="utf-8")
+    assert "Use graphify wisely." in md_src
+    py_src = (src_dir / "_src_util.py.md").read_text(encoding="utf-8")
+    assert "```python" in py_src and "return 42" in py_src
+
+    # Each resolvable node note gets a callout linking to its source note,
+    # displayed with the original source_file path.
+    note = (vault / "Best Practices.md").read_text(encoding="utf-8")
+    assert "> [!info] Source" in note
+    assert "[[_src_en_best-practices|en/best-practices.md]]" in note
+
+
+def test_unresolvable_source_gets_no_callout(tmp_path):
+    """A node whose source_file is missing on disk must not get a dangling
+    callout (and must not abort the export)."""
+    _make_sources(tmp_path)
+    G, comms = _graph()
+    vault = tmp_path / "vault"
+    to_obsidian(G, comms, str(vault), with_sources=True, source_root=tmp_path)
+    ghost = (vault / "Ghost.md").read_text(encoding="utf-8")
+    assert "> [!info] Source" not in ghost
+    assert not (vault / "sources" / "_src_missing_nope.md").exists()
+
+
+def test_rerun_without_sources_prunes_source_notes(tmp_path):
+    """Source notes are graphify-owned, so dropping --with-sources on a re-export
+    prunes them and removes the callout — the vault returns to bare notes."""
+    _make_sources(tmp_path)
+    G, comms = _graph()
+    vault = tmp_path / "vault"
+    to_obsidian(G, comms, str(vault), with_sources=True, source_root=tmp_path)
+    assert list((vault / "sources").glob("*.md"))
+
+    to_obsidian(G, comms, str(vault), with_sources=False)
+    assert not list((vault / "sources").glob("*.md"))
+    assert "> [!info] Source" not in (vault / "foo.md").read_text(encoding="utf-8")
+
+
+def test_source_root_list_and_fallback(tmp_path):
+    """source_root accepts a list of candidate roots (scan root, then out dir for
+    <=0.9.16 graphs); the first root that holds the file wins."""
+    scan_root = tmp_path / "project"
+    out_dir = tmp_path / "project" / "graphify-out"
+    scan_root.mkdir()
+    out_dir.mkdir()
+    # source_file stored relative to the SCAN root (the #1941 layout).
+    (scan_root / "a.py").write_text("x = 1\n", encoding="utf-8")
+    G = nx.Graph()
+    G.add_node("n0", label="a", file_type="code", source_file="a.py", community=0)
+    G.add_node("n1", label="b", file_type="code", source_file="a.py", community=0)
+    G.add_edge("n0", "n1", relation="r", confidence="EXTRACTED")
+    vault = tmp_path / "vault"
+    to_obsidian(G, {0: ["n0", "n1"]}, str(vault),
+                with_sources=True, source_root=[scan_root, out_dir])
+    # One shared source note for the shared source_file, both notes wired to it.
+    assert (vault / "sources" / "_src_a.py.md").exists()
+    assert "[[_src_a.py|a.py]]" in (vault / "a.md").read_text(encoding="utf-8")
+    assert "[[_src_a.py|a.py]]" in (vault / "b.md").read_text(encoding="utf-8")