ENG-3697: Bound Live and Get responses, and stop List deleting its own content

nodes/Hyperspell/resources/document/get.ts

@@ -1,5 +1,6 @@
 import type { INodeProperties } from 'n8n-workflow';
 import { sourceOptions } from '../shared';
+import { simplifyProperty } from '../simplify';
 
 const showOnlyForDocumentGet = {
 	operation: ['get'],
@@ -26,4 +27,5 @@ export const documentGetDescription: INodeProperties[] = [
 		displayOptions: { show: showOnlyForDocumentGet },
 		description: 'The resource ID returned when the document was added',
 	},
+	simplifyProperty(showOnlyForDocumentGet),
 ];

nodes/Hyperspell/resources/document/index.ts

@@ -1,7 +1,7 @@
 import type { INodeProperties } from 'n8n-workflow';
 import { raiseApiErrors } from '../errors';
 import { hintAppScopedEmpty } from '../actAsUser';
-import { simplifyDocuments } from '../simplify';
+import { simplifyDocuments, simplifyOne } from '../simplify';
 import { documentAddDescription } from './add';
 import { documentGetDescription } from './get';
 import { documentListDescription } from './list';
@@ -50,8 +50,10 @@ export const documentDescription: INodeProperties[] = [
 					},
 					// ignoreHttpStatusErrors is on globally, so EVERY operation needs this —
 					// without it a 4xx would flow through as if it were a result.
+					// Then bound the body: a get returns one full hyperdoc tree, which
+					// is the single largest per-item payload the node can emit.
 					output: {
-						postReceive: [raiseApiErrors],
+						postReceive: [raiseApiErrors, simplifyOne],
 					},
 				},
 			},

nodes/Hyperspell/resources/live/index.ts

@@ -2,6 +2,7 @@ import type { INodeProperties } from 'n8n-workflow';
 import { raiseApiErrors } from '../errors';
 import { hintAppScopedEmpty } from '../actAsUser';
 import { sourceOptions } from '../shared';
+import { simplifyProperty } from '../simplify';
 import { unwrapCursorPage, unwrapLiveEnvelope } from './output';
 import { liveSearchDescription } from './search';
 import { liveGetDescription } from './get';
@@ -93,6 +94,13 @@ export const liveDescription: INodeProperties[] = [
 		],
 		default: 'search',
 	},
+	// List Sources is excluded deliberately: it returns capability descriptors,
+	// not documents, so there is no tree to bound and a toggle there would only
+	// be noise.
+	simplifyProperty({
+		resource: ['live'],
+		operation: ['search', 'getResource', 'listResources'],
+	}),
 	{
 		// Used in the request URL path ({{$parameter.source}}) for all ops except List Sources.
 		displayName: 'Source',

nodes/Hyperspell/resources/live/output.ts

@@ -4,6 +4,7 @@ import type {
 	IN8nHttpFullResponse,
 	INodeExecutionData,
 } from 'n8n-workflow';
+import { simplifyDocument } from '../simplify';
 
 // The /live/* endpoints wrap results in envelopes (ENG-2479 Hyperdoc migration):
 //   Search / Get Resource → LiveResourceResponse { documents, indexed, notes }
@@ -27,6 +28,19 @@ interface CursorPageEnvelope {
 	next_cursor?: string | null;
 }
 
+// Every /live/* route returns the plain `DocumentResponse` — full hyperdoc
+// tree, no summary, no highlights. Until now nothing bounded it, so the one
+// surface explicitly built for an AI-agent node ("reach LIVE into a connected
+// source", api/live.py) was also the only one that handed a model an unbounded
+// payload. Same Simplify contract as Search/List: default on, one toggle off.
+function boundDocuments(
+	context: IExecuteSingleFunctions,
+	documents: IDataObject[],
+): IDataObject[] {
+	if (context.getNodeParameter('simplify', true) === false) return documents;
+	return documents.map((document) => simplifyDocument(document));
+}
+
 export async function unwrapLiveEnvelope(
 	this: IExecuteSingleFunctions,
 	_items: INodeExecutionData[],
@@ -44,7 +58,9 @@ export async function unwrapLiveEnvelope(
 		// stays zero-item so IF-node emptiness checks keep working.
 		return notes.length > 0 ? [{ json: { documents: [], indexed, notes } }] : [];
 	}
-	return documents.map((document) => ({ json: { ...document, indexed, notes } }));
+	return boundDocuments(this, documents).map((document) => ({
+		json: { ...document, indexed, notes },
+	}));
 }
 
 export async function unwrapCursorPage(
@@ -59,5 +75,7 @@ export async function unwrapCursorPage(
 	// Auto-pagination is unaffected — it reads next_cursor from the raw body
 	// before postReceive runs.
 	const nextCursor = body.next_cursor ?? null;
-	return pageItems.map((item) => ({ json: { ...item, next_cursor: nextCursor } }));
+	return boundDocuments(this, pageItems).map((item) => ({
+		json: { ...item, next_cursor: nextCursor },
+	}));
 }

nodes/Hyperspell/resources/simplify.ts

@@ -46,29 +46,73 @@ interface Highlight {
 	score?: number;
 }
 
+interface HyperdocNode extends IDataObject {
+	text?: string;
+	children?: HyperdocNode[];
+}
+
 interface HyperspellDocument extends IDataObject {
 	summary?: string;
 	highlights?: Highlight[];
-	document?: IDataObject;
+	document?: HyperdocNode;
 }
 
 // Result-array keys across the document-shaped responses:
-//   POST /memories/query → documents;  GET /memories/list → items.
+//   POST /memories/query → documents;  GET /memories/list → items;
+//   GET /live/{source}/resources → items.
 const RESULT_ARRAY_KEYS = ['documents', 'items'];
 
-/** The matched text, preferring the server's concatenation of the highlights. */
+// Only the QUERY path returns `ScoredDocumentResponse` (schemas.py:152), which
+// is the sole model carrying `summary`/`highlights`. `/memories/list`,
+// `/memories/get/*` and every `/live/*` route return the plain
+// `DocumentResponse` (schemas.py:75) — no summary, no highlights, body only in
+// the `document` tree. Simplifying those on the highlights-only path emitted
+// `text: ''` and dropped the tree, i.e. deleted the content outright (shipped
+// in 0.7.0; Document → List returned metadata-only rows). So the tree is the
+// fallback source of text, flattened and capped rather than passed through.
+//
+// 2000 chars mirrors the API's own per-highlight cap, which keeps a simplified
+// row roughly the size of one query hit no matter which endpoint produced it.
+// Callers who need the whole body turn Simplify off.
+export const MAX_SIMPLIFIED_TEXT = 2000;
+
+/** Depth-first concatenation of a hyperdoc tree's `text` nodes, capped. */
+function flattenHyperdoc(node: HyperdocNode | undefined, budget: number): string {
+	if (node === undefined || node === null || budget <= 0) return '';
+	const parts: string[] = [];
+	let remaining = budget;
+	const visit = (current: HyperdocNode | undefined): void => {
+		if (current === undefined || current === null || remaining <= 0) return;
+		if (typeof current.text === 'string' && current.text.length > 0) {
+			parts.push(current.text.slice(0, remaining));
+			remaining -= Math.min(current.text.length, remaining);
+		}
+		const children = Array.isArray(current.children) ? current.children : [];
+		for (const child of children) visit(child);
+	};
+	visit(node);
+	return parts.join('\n');
+}
+
+/**
+ * The document's text: the server's highlight concatenation on the query path,
+ * otherwise the flattened hyperdoc tree. Never the raw tree, and never empty
+ * when the document has any text at all.
+ */
 function matchedText(document: HyperspellDocument): string {
 	if (typeof document.summary === 'string' && document.summary.length > 0) {
 		return document.summary;
 	}
 	const highlights = Array.isArray(document.highlights) ? document.highlights : [];
-	return highlights
+	const fromHighlights = highlights
 		.map((highlight) => highlight?.text)
 		.filter((text): text is string => typeof text === 'string' && text.length > 0)
 		.join('\n\n');
+	if (fromHighlights.length > 0) return fromHighlights;
+	return flattenHyperdoc(document.document, MAX_SIMPLIFIED_TEXT);
 }
 
-function simplifyDocument(document: HyperspellDocument): IDataObject {
+export function simplifyDocument(document: HyperspellDocument): IDataObject {
 	// Explicit allow-list, not a `delete document.document`: `summary` and
 	// `highlights[].text` carry the SAME text (summary is defined as their
 	// concatenation), so passing both through would ship every matched chunk
@@ -117,3 +161,25 @@ export async function simplifyDocuments(
 		return { ...item, json: { ...json, [key]: documents.map(simplifyDocument) } };
 	});
 }
+
+/**
+ * postReceive for the single-document responses — `GET /memories/get/*` returns
+ * a bare `DocumentResponse`, not an array, so `simplifyDocuments` (which looks
+ * for a result-array key) passes it through untouched and the full tree reaches
+ * the caller. One document is exactly the case where the tree is largest per
+ * item, so this is the operation most worth bounding, not the least.
+ */
+export async function simplifyOne(
+	this: IExecuteSingleFunctions,
+	items: INodeExecutionData[],
+): Promise<INodeExecutionData[]> {
+	if (this.getNodeParameter('simplify', true) === false) return items;
+
+	return items.map((item) => {
+		const json = (item.json ?? {}) as HyperspellDocument;
+		// Only rewrite something that actually looks like a document response;
+		// an error body or a notice item must pass through untouched.
+		if (json.resource_id === undefined) return item;
+		return { ...item, json: simplifyDocument(json) };
+	});
+}

tests/live-output.test.mjs

@@ -38,19 +38,37 @@ const PROD_CURSOR_PAGE = {
 	next_cursor: 'opaque-cursor-token',
 };
 
-const call = (fn, body) => fn.call(undefined, [], { body, headers: {}, statusCode: 200 });
+/**
+ * Minimal IExecuteSingleFunctions stand-in. The unwrappers now read the
+ * Simplify parameter (every /live/* route returns a plain DocumentResponse
+ * whose hyperdoc tree is otherwise unbounded), so `this` must be a real
+ * context — n8n always binds one; only this harness ever passed undefined.
+ */
+const ctx = (options = {}) => ({
+	getNodeParameter: (name, fallback) =>
+		name === 'simplify' ? (options.simplify ?? fallback) : fallback,
+});
+
+const call = (fn, body, options = {}) =>
+	fn.call(ctx(options), [], { body, headers: {}, statusCode: 200 });
 
 // ── unwrapLiveEnvelope (Search / Get Resource) ─────────────────────────────
 
 test('search/get: one item per document, envelope indexed/notes merged onto each', async () => {
-	const out = await call(unwrapLiveEnvelope, {
-		documents: [
-			{ resource_id: 'a', title: 'A', document: { type: 'document', children: [] } },
-			{ resource_id: 'b', title: 'B', document: { type: 'document', children: [] } },
-		],
-		indexed: true,
-		notes: ['queued 2 documents'],
-	});
+	// Simplify off: this test is about the ENVELOPE contract (siblings must not
+	// be dropped), so it asserts the raw shape. Bounding is covered separately.
+	const out = await call(
+		unwrapLiveEnvelope,
+		{
+			documents: [
+				{ resource_id: 'a', title: 'A', document: { type: 'document', children: [] } },
+				{ resource_id: 'b', title: 'B', document: { type: 'document', children: [] } },
+			],
+			indexed: true,
+			notes: ['queued 2 documents'],
+		},
+		{ simplify: false },
+	);
 	assert.equal(out.length, 2);
 	assert.equal(out[0].json.resource_id, 'a');
 	assert.equal(out[1].json.resource_id, 'b');
@@ -100,7 +118,8 @@ test('search/get: defaults applied when envelope fields are absent (exclude_none
 // ── unwrapCursorPage (List Resources) ──────────────────────────────────────
 
 test('list: real prod fixture — one item per resource, next_cursor on each, document tree intact', async () => {
-	const out = await call(unwrapCursorPage, PROD_CURSOR_PAGE);
+	// Simplify off — asserts the unwrap itself preserves the raw document.
+	const out = await call(unwrapCursorPage, PROD_CURSOR_PAGE, { simplify: false });
 	assert.equal(out.length, 1);
 	const item = out[0].json;
 	assert.equal(item.resource_id, 'contact:479239644873');
@@ -123,3 +142,64 @@ test('list: empty or missing items emits zero items, never {json: undefined}', a
 		assert.deepEqual(out, [], `body=${JSON.stringify(body)}`);
 	}
 });
+
+// ── Bounding (ENG-3697: "the node pumps too many tokens") ──────────────────
+//
+// Every /live/* route returns the plain DocumentResponse — no summary, no
+// highlights, body only in the `document` tree — and nothing bounded it before
+// 0.7.2. The Live resource is the surface built for an AI-agent node, so it was
+// simultaneously the most likely to feed a model and the only one shipping an
+// unbounded payload.
+
+test('live search/get: Simplify (default on) replaces the tree with flattened text', async () => {
+	const body = 'x'.repeat(5000);
+	const out = await call(unwrapLiveEnvelope, {
+		documents: [
+			{
+				resource_id: 'a',
+				title: 'A',
+				document: { type: 'document', children: [{ type: 'paragraph', text: body }] },
+			},
+		],
+		indexed: true,
+		notes: [],
+	});
+	const item = out[0].json;
+	assert.equal(item.document, undefined, 'hyperdoc tree must be dropped when simplified');
+	assert.equal(typeof item.text, 'string');
+	assert.equal(item.text.length, 2000, 'text is capped at MAX_SIMPLIFIED_TEXT');
+	// Envelope siblings still ride along — bounding must not undo the unwrap contract.
+	assert.equal(item.indexed, true);
+	assert.equal(item.resource_id, 'a');
+});
+
+test('live list: Simplify (default on) bounds each row and keeps next_cursor', async () => {
+	const out = await call(unwrapCursorPage, PROD_CURSOR_PAGE);
+	const item = out[0].json;
+	assert.equal(item.document, undefined);
+	assert.equal(item.next_cursor, 'opaque-cursor-token');
+	assert.equal(item.resource_id, 'contact:479239644873');
+	// The fixture's hyperdoc carries no `text` nodes, only name/email/company
+	// fields, so flattening yields empty — correct, and still bounded.
+	assert.equal(typeof item.text, 'string');
+});
+
+test('live: a simplified document is dramatically smaller than the raw one', async () => {
+	const body = 'y'.repeat(200000);
+	const envelope = {
+		documents: [
+			{
+				resource_id: 'big',
+				document: { type: 'document', children: [{ type: 'paragraph', text: body }] },
+			},
+		],
+		indexed: false,
+		notes: [],
+	};
+	const [raw] = await call(unwrapLiveEnvelope, envelope, { simplify: false });
+	const [bounded] = await call(unwrapLiveEnvelope, envelope);
+	const rawSize = JSON.stringify(raw.json).length;
+	const boundedSize = JSON.stringify(bounded.json).length;
+	assert.ok(rawSize > 100000, `precondition: raw is large (${rawSize})`);
+	assert.ok(boundedSize < 2500, `bounded must be small, got ${boundedSize}`);
+});