diff --git a/nodes/Hyperspell/resources/document/get.ts b/nodes/Hyperspell/resources/document/get.ts index eb8ac4d..e1ff929 100644 --- a/nodes/Hyperspell/resources/document/get.ts +++ b/nodes/Hyperspell/resources/document/get.ts @@ -1,5 +1,6 @@ import type { INodeProperties } from 'n8n-workflow'; import { sourceOptions } from '../shared'; +import { simplifyProperty } from '../simplify'; const showOnlyForDocumentGet = { operation: ['get'], @@ -26,4 +27,5 @@ export const documentGetDescription: INodeProperties[] = [ displayOptions: { show: showOnlyForDocumentGet }, description: 'The resource ID returned when the document was added', }, + simplifyProperty(showOnlyForDocumentGet), ]; diff --git a/nodes/Hyperspell/resources/document/index.ts b/nodes/Hyperspell/resources/document/index.ts index ce82f5f..3200aa5 100644 --- a/nodes/Hyperspell/resources/document/index.ts +++ b/nodes/Hyperspell/resources/document/index.ts @@ -1,7 +1,7 @@ import type { INodeProperties } from 'n8n-workflow'; import { raiseApiErrors } from '../errors'; import { hintAppScopedEmpty } from '../actAsUser'; -import { simplifyDocuments } from '../simplify'; +import { simplifyDocuments, simplifyOne } from '../simplify'; import { documentAddDescription } from './add'; import { documentGetDescription } from './get'; import { documentListDescription } from './list'; @@ -50,8 +50,10 @@ export const documentDescription: INodeProperties[] = [ }, // ignoreHttpStatusErrors is on globally, so EVERY operation needs this — // without it a 4xx would flow through as if it were a result. + // Then bound the body: a get returns one full hyperdoc tree, which + // is the single largest per-item payload the node can emit. output: { - postReceive: [raiseApiErrors], + postReceive: [raiseApiErrors, simplifyOne], }, }, }, diff --git a/nodes/Hyperspell/resources/live/connectionId.ts b/nodes/Hyperspell/resources/live/connectionId.ts new file mode 100644 index 0000000..1b12b51 --- /dev/null +++ b/nodes/Hyperspell/resources/live/connectionId.ts @@ -0,0 +1,118 @@ +import type { + IExecuteSingleFunctions, + IHttpRequestOptions, + INodeProperties, +} from 'n8n-workflow'; +import { NodeOperationError } from 'n8n-workflow'; + +// Connection ID is the one Live field an AI Agent gets wrong in a way the API +// cannot recover from. +// +// The node sets `usableAsTool`, so when it runs as an Agent tool the model +// fills the parameters from their descriptions. "Specific connection ID when +// the user has multiple connections for this source" reads, to a model, like +// an invitation to name the source — and that is exactly what happens. +// Observed in prod on 2026-08-10: `connection_id` arriving as 'linear', +// 'github', 'google_drive'. +// +// Core takes that string straight into a UUID-typed column +// (`live_access.py`: `stmt.where(Connection.id == connection_id)`), asyncpg +// rejects it with a DataError, and a blanket `except Exception` in +// `api/live.py` turns it into `502 "Upstream source error."`. That message is +// worse than useless: it blames the provider when no provider was ever +// contacted, so the workflow author reasonably concludes Linear is down. +// +// Core should return a 400 here and will (tracked separately) — but the node +// should not be shipping a malformed request in the first place, and this +// guard keeps working against every already-deployed core version. +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * preSend: reject a Connection ID that isn't a UUID, naming the likely mistake. + * + * Deliberately NOT a silent drop. Silently ignoring the field would make a + * request scoped to the wrong connection look like it succeeded — the same + * class of silent-wrong-answer this node has been bitten by before. + */ +export async function validateConnectionId( + this: IExecuteSingleFunctions, + requestOptions: IHttpRequestOptions, +): Promise { + const raw = this.getNodeParameter('connection_id', '') as unknown; + if (raw === undefined || raw === null) return requestOptions; + + // An expression can resolve this string field to any type (`={{ $json.id }}` + // over a numeric column), and a non-string is never a UUID. Treating one as + // empty would wave it past the guard while the routing expression still put + // it on the wire — the exact bypass this function exists to close — so it + // earns the same named error a source name does. + const value = typeof raw === 'string' ? raw.trim() : raw; + if (value === '') return requestOptions; + if (typeof value === 'string' && UUID_RE.test(value)) return requestOptions; + + throw new NodeOperationError( + this.getNode(), + `Connection ID must be a connection UUID, not "${String(value)}".`, + { + description: + 'Connection ID identifies ONE specific connection when a user has connected the same source more than once — it is not the source name. Leave it empty to use the caller\'s connection automatically. If this node is running as an AI Agent tool, pin this field to empty so the model cannot fill it.', + }, + ); +} + +/** + * The Connection ID field, shared by the three document-shaped Live operations. + * + * `transport` differs by operation — Search POSTs a body, Get and List Resources + * put it on the query string — but the validation and the wording must not, so + * both go through here rather than being restated three times. + */ +export function connectionIdProperty( + show: NonNullable['show'], + transport: 'body' | 'query' = 'body', +): INodeProperties { + return { + displayName: 'Connection ID', + name: 'connection_id', + type: 'string', + default: '', + placeholder: '', + displayOptions: { show }, + // Description is the model's only input when this runs as a tool, so it + // states the format, and states the default, before it states the purpose. + description: + 'Leave empty. Optional UUID identifying one specific connection, used only when the same user has connected this source more than once. This is NOT the source name.', + routing: { + send: { + type: transport, + property: 'connection_id', + // `.trim()` BEFORE the falsy check, so what goes on the wire is what + // the guard above actually validated. The guard trims its input, so + // " " is accepted as "not provided" — but untrimmed `$value` is + // truthy, so without this the node sent " ", core put it in the + // UUID column, and the caller got back the exact + // `502 "Upstream source error."` this whole field exists to prevent. + // Verified against a pre-fix core on 2026-08-12. + // + // Trimming also rescues a pasted " ": core parses this value + // with `UUID()`, which does not tolerate surrounding whitespace and + // 400s on the padded form. + // + // `String(...)` because an expression can resolve this field to a + // non-string (`={{ $json.id }}` over a numeric column) and a bare + // `.trim()` would throw a raw TypeError from inside the expression — + // which runs BEFORE preSend, so it would pre-empt the guard's named + // error with an unreadable one. Coercing keeps the guard in charge of + // the message; it rejects every non-string anyway. + // + // `|| undefined` so an untouched field is OMITTED rather than sent as + // "". Core types it `str | None` and only falsy-checks it, so "" + // happens to be harmless — but the sibling Live ops say "no connection + // specified" by omission, and this one shouldn't rely on a + // falsy-string coincidence. + value: '={{ String($value ?? "").trim() || undefined }}', + preSend: [validateConnectionId], + }, + }, + }; +} diff --git a/nodes/Hyperspell/resources/live/get.ts b/nodes/Hyperspell/resources/live/get.ts index cba16b2..4e9b37a 100644 --- a/nodes/Hyperspell/resources/live/get.ts +++ b/nodes/Hyperspell/resources/live/get.ts @@ -1,4 +1,5 @@ import type { INodeProperties } from 'n8n-workflow'; +import { connectionIdProperty } from './connectionId'; const showOnlyForLiveGet = { resource: ['live'], @@ -25,13 +26,5 @@ export const liveGetDescription: INodeProperties[] = [ description: 'Whether to also queue the fetched resource for indexing so it is on-hand next time. The "indexed" and "notes" fields on each output item report what happened.', routing: { request: { qs: { index: '={{$value}}' } } }, }, - { - displayName: 'Connection ID', - name: 'connection_id', - type: 'string', - default: '', - displayOptions: { show: showOnlyForLiveGet }, - description: 'Specific connection ID when the user has multiple connections for this source', - routing: { request: { qs: { connection_id: '={{$value || undefined}}' } } }, - }, + connectionIdProperty(showOnlyForLiveGet, 'query'), ]; diff --git a/nodes/Hyperspell/resources/live/index.ts b/nodes/Hyperspell/resources/live/index.ts index 19884cb..9e22a53 100644 --- a/nodes/Hyperspell/resources/live/index.ts +++ b/nodes/Hyperspell/resources/live/index.ts @@ -2,6 +2,7 @@ import type { INodeProperties } from 'n8n-workflow'; import { raiseApiErrors } from '../errors'; import { hintAppScopedEmpty } from '../actAsUser'; import { sourceOptions } from '../shared'; +import { simplifyProperty } from '../simplify'; import { unwrapCursorPage, unwrapLiveEnvelope } from './output'; import { liveSearchDescription } from './search'; import { liveGetDescription } from './get'; @@ -93,6 +94,13 @@ export const liveDescription: INodeProperties[] = [ ], default: 'search', }, + // List Sources is excluded deliberately: it returns capability descriptors, + // not documents, so there is no tree to bound and a toggle there would only + // be noise. + simplifyProperty({ + resource: ['live'], + operation: ['search', 'getResource', 'listResources'], + }), { // Used in the request URL path ({{$parameter.source}}) for all ops except List Sources. displayName: 'Source', diff --git a/nodes/Hyperspell/resources/live/list.ts b/nodes/Hyperspell/resources/live/list.ts index c75811f..fd6aa3b 100644 --- a/nodes/Hyperspell/resources/live/list.ts +++ b/nodes/Hyperspell/resources/live/list.ts @@ -1,4 +1,5 @@ import type { INodeProperties } from 'n8n-workflow'; +import { connectionIdProperty } from './connectionId'; import { PAGED_QS } from '../shared'; const showOnlyForLiveList = { @@ -73,13 +74,5 @@ export const liveListDescription: INodeProperties[] = [ 'Opaque cursor from a previous page (the next_cursor field on returned items). Leave empty for the first page.', routing: { request: { qs: { cursor: '={{ $value || undefined }}' } } }, }, - { - displayName: 'Connection ID', - name: 'connection_id', - type: 'string', - default: '', - displayOptions: { show: showOnlyForLiveList }, - description: 'Specific connection ID when the user has multiple connections for this source', - routing: { request: { qs: { connection_id: '={{$value || undefined}}' } } }, - }, + connectionIdProperty(showOnlyForLiveList, 'query'), ]; diff --git a/nodes/Hyperspell/resources/live/output.ts b/nodes/Hyperspell/resources/live/output.ts index 21a2c62..d359a6e 100644 --- a/nodes/Hyperspell/resources/live/output.ts +++ b/nodes/Hyperspell/resources/live/output.ts @@ -4,6 +4,7 @@ import type { IN8nHttpFullResponse, INodeExecutionData, } from 'n8n-workflow'; +import { simplifyDocument } from '../simplify'; // The /live/* endpoints wrap results in envelopes (ENG-2479 Hyperdoc migration): // Search / Get Resource → LiveResourceResponse { documents, indexed, notes } @@ -27,6 +28,19 @@ interface CursorPageEnvelope { next_cursor?: string | null; } +// Every /live/* route returns the plain `DocumentResponse` — full hyperdoc +// tree, no summary, no highlights. Until now nothing bounded it, so the one +// surface explicitly built for an AI-agent node ("reach LIVE into a connected +// source", api/live.py) was also the only one that handed a model an unbounded +// payload. Same Simplify contract as Search/List: default on, one toggle off. +function boundDocuments( + context: IExecuteSingleFunctions, + documents: IDataObject[], +): IDataObject[] { + if (context.getNodeParameter('simplify', true) === false) return documents; + return documents.map((document) => simplifyDocument(document)); +} + export async function unwrapLiveEnvelope( this: IExecuteSingleFunctions, _items: INodeExecutionData[], @@ -44,7 +58,9 @@ export async function unwrapLiveEnvelope( // stays zero-item so IF-node emptiness checks keep working. return notes.length > 0 ? [{ json: { documents: [], indexed, notes } }] : []; } - return documents.map((document) => ({ json: { ...document, indexed, notes } })); + return boundDocuments(this, documents).map((document) => ({ + json: { ...document, indexed, notes }, + })); } export async function unwrapCursorPage( @@ -59,5 +75,7 @@ export async function unwrapCursorPage( // Auto-pagination is unaffected — it reads next_cursor from the raw body // before postReceive runs. const nextCursor = body.next_cursor ?? null; - return pageItems.map((item) => ({ json: { ...item, next_cursor: nextCursor } })); + return boundDocuments(this, pageItems).map((item) => ({ + json: { ...item, next_cursor: nextCursor }, + })); } diff --git a/nodes/Hyperspell/resources/live/search.ts b/nodes/Hyperspell/resources/live/search.ts index d9bf7d7..4b5d97a 100644 --- a/nodes/Hyperspell/resources/live/search.ts +++ b/nodes/Hyperspell/resources/live/search.ts @@ -1,4 +1,5 @@ import type { INodeProperties } from 'n8n-workflow'; +import { connectionIdProperty } from './connectionId'; const showOnlyForLiveSearch = { resource: ['live'], @@ -37,24 +38,5 @@ export const liveSearchDescription: INodeProperties[] = [ }, }, }, - { - displayName: 'Connection ID', - name: 'connection_id', - type: 'string', - default: '', - displayOptions: { show: showOnlyForLiveSearch }, - description: 'Specific connection ID when the user has multiple connections for this source', - routing: { - send: { - type: 'body', - property: 'connection_id', - // `|| undefined` so an untouched field is OMITTED rather than sent as - // "". Core types it `str | None` and today only falsy-checks it - // (live_access.py `if connection_id:`), so "" happens to be harmless — - // but the sibling Live ops say "no connection specified" by omission, - // and this one shouldn't rely on a falsy-string coincidence. - value: '={{ $value || undefined }}', - }, - }, - }, + connectionIdProperty(showOnlyForLiveSearch), ]; diff --git a/nodes/Hyperspell/resources/simplify.ts b/nodes/Hyperspell/resources/simplify.ts index 094a933..aaa585c 100644 --- a/nodes/Hyperspell/resources/simplify.ts +++ b/nodes/Hyperspell/resources/simplify.ts @@ -46,29 +46,122 @@ interface Highlight { score?: number; } +interface HyperdocNode extends IDataObject { + text?: string; + children?: HyperdocNode[]; +} + interface HyperspellDocument extends IDataObject { summary?: string; highlights?: Highlight[]; - document?: IDataObject; + document?: HyperdocNode; } // Result-array keys across the document-shaped responses: -// POST /memories/query → documents; GET /memories/list → items. +// POST /memories/query → documents; GET /memories/list → items; +// GET /live/{source}/resources → items. const RESULT_ARRAY_KEYS = ['documents', 'items']; -/** The matched text, preferring the server's concatenation of the highlights. */ +// Only the QUERY path returns `ScoredDocumentResponse` (schemas.py:152), which +// is the sole model carrying `summary`/`highlights`. `/memories/list`, +// `/memories/get/*` and every `/live/*` route return the plain +// `DocumentResponse` (schemas.py:75) — no summary, no highlights, body only in +// the `document` tree. Simplifying those on the highlights-only path emitted +// `text: ''` and dropped the tree, i.e. deleted the content outright (shipped +// in 0.7.0; Document → List returned metadata-only rows). So the tree is the +// fallback source of text, flattened and capped rather than passed through. +// +// 2000 chars mirrors the API's own per-highlight cap, which keeps a simplified +// row roughly the size of one query hit no matter which endpoint produced it. +// Callers who need the whole body turn Simplify off. +export const MAX_SIMPLIFIED_TEXT = 2000; + +// Addressing and structure, not content: `text` is read directly, `children` is +// the recursion. +const STRUCTURAL_KEYS = new Set(['type', 'id', 'text', 'children']); + +/** + * One node's content — its `text`, or its named scalar fields when it has none. + * + * Not every hyperdoc node carries a `text`. A HubSpot contact arrives as + * `{type:'person', name, email, company, children: []}` (recorded from prod in + * docs/incidents/2026-06-11-live-resource-hyperdoc-shape.md), so a walk that + * reads only `text`/`children` flattens it to '' — and `simplifyDocument` then + * drops the tree, deleting the email and company outright. That is the same + * defect this file fixes for Document → List, and Live → List Resources would + * have shipped it the moment Simplify defaulted on there. + * + * Labels are kept because `email: maria@hubspot.com` is worth materially more + * to whatever reads this next than a bare value would be. + * + * Only LEAF nodes are harvested this way. A node with children holds its + * content in them, and its own scalars are addressing or metadata that the + * simplified row already carries at top level — a document root's `title` + * being the case that matters, since folding it in would duplicate tokens, + * which is the thing this file's allow-list exists to prevent. + */ +function nodeText(node: HyperdocNode): string { + if (typeof node.text === 'string' && node.text.length > 0) return node.text; + if (Array.isArray(node.children) && node.children.length > 0) return ''; + const fields: string[] = []; + for (const [key, value] of Object.entries(node)) { + if (STRUCTURAL_KEYS.has(key)) continue; + if (typeof value === 'string' && value.length > 0) fields.push(`${key}: ${value}`); + else if (typeof value === 'number' || typeof value === 'boolean') { + fields.push(`${key}: ${value}`); + } + } + return fields.join('\n'); +} + +/** Depth-first concatenation of a hyperdoc tree's content, capped. */ +function flattenHyperdoc(node: HyperdocNode | undefined, budget: number): string { + if (node === undefined || node === null || budget <= 0) return ''; + const parts: string[] = []; + let remaining = budget; + const visit = (current: HyperdocNode | undefined): void => { + if (current === undefined || current === null || remaining <= 0) return; + const own = nodeText(current); + if (own.length > 0) { + // The '\n' this part needs once joined is charged to the same budget. + // Counting only node text let a tree of many tiny nodes return close to + // 2x MAX_SIMPLIFIED_TEXT in separators alone, so the documented cap was + // not the real one. + const separator = parts.length > 0 ? 1 : 0; + const slice = own.slice(0, Math.max(0, remaining - separator)); + if (slice.length > 0) { + parts.push(slice); + remaining -= slice.length + separator; + } else { + remaining = 0; + } + } + const children = Array.isArray(current.children) ? current.children : []; + for (const child of children) visit(child); + }; + visit(node); + return parts.join('\n'); +} + +/** + * The document's text: the server's highlight concatenation on the query path, + * otherwise the flattened hyperdoc tree. Never the raw tree, and never empty + * when the document has any text at all. + */ function matchedText(document: HyperspellDocument): string { if (typeof document.summary === 'string' && document.summary.length > 0) { return document.summary; } const highlights = Array.isArray(document.highlights) ? document.highlights : []; - return highlights + const fromHighlights = highlights .map((highlight) => highlight?.text) .filter((text): text is string => typeof text === 'string' && text.length > 0) .join('\n\n'); + if (fromHighlights.length > 0) return fromHighlights; + return flattenHyperdoc(document.document, MAX_SIMPLIFIED_TEXT); } -function simplifyDocument(document: HyperspellDocument): IDataObject { +export function simplifyDocument(document: HyperspellDocument): IDataObject { // Explicit allow-list, not a `delete document.document`: `summary` and // `highlights[].text` carry the SAME text (summary is defined as their // concatenation), so passing both through would ship every matched chunk @@ -117,3 +210,25 @@ export async function simplifyDocuments( return { ...item, json: { ...json, [key]: documents.map(simplifyDocument) } }; }); } + +/** + * postReceive for the single-document responses — `GET /memories/get/*` returns + * a bare `DocumentResponse`, not an array, so `simplifyDocuments` (which looks + * for a result-array key) passes it through untouched and the full tree reaches + * the caller. One document is exactly the case where the tree is largest per + * item, so this is the operation most worth bounding, not the least. + */ +export async function simplifyOne( + this: IExecuteSingleFunctions, + items: INodeExecutionData[], +): Promise { + if (this.getNodeParameter('simplify', true) === false) return items; + + return items.map((item) => { + const json = (item.json ?? {}) as HyperspellDocument; + // Only rewrite something that actually looks like a document response; + // an error body or a notice item must pass through untouched. + if (json.resource_id === undefined) return item; + return { ...item, json: simplifyDocument(json) }; + }); +} diff --git a/tests/connection-id.test.mjs b/tests/connection-id.test.mjs new file mode 100644 index 0000000..cc32e7a --- /dev/null +++ b/tests/connection-id.test.mjs @@ -0,0 +1,155 @@ +// Tests for the Live Connection ID guard (nodes/Hyperspell/resources/live/connectionId.ts). +// +// Regression: on 2026-08-10 prod logged `connection_id` arriving as 'linear', +// 'github' and 'google_drive' — source names, not UUIDs. The node sets +// usableAsTool, so an AI Agent fills this field from its description, and the +// old wording ("Specific connection ID when the user has multiple connections +// for this source") reads as an invitation to name the source. +// +// Core puts that value straight into a UUID column +// (live_access.py: `stmt.where(Connection.id == connection_id)`), asyncpg +// raises DataError, and a blanket handler in api/live.py returns +// `502 "Upstream source error."` — blaming a provider that was never called. +// +// Reproduced directly against a local core on 2026-08-12: +// POST /live/linear/search {"query":"test"} -> 400 ConnectionNotFound (clear) +// POST /live/linear/search {"query":"test","connection_id":"linear"} -> 502 "Upstream source error." +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import test from 'node:test'; + +const require = createRequire(import.meta.url); +const { validateConnectionId, connectionIdProperty } = require( + '../dist/nodes/Hyperspell/resources/live/connectionId.js', +); + +const ctx = (value) => ({ + getNodeParameter: (name, fallback) => (name === 'connection_id' ? value : fallback), + getNode: () => ({ name: 'Hyperspell', type: 'n8n-nodes-hyperspell.hyperspellTool' }), +}); + +const req = () => ({ body: {}, qs: {}, url: '/live/linear/search', method: 'POST' }); + +test('a real connection UUID passes through untouched', async () => { + const uuid = '3f2504e0-4f89-11d3-9a0c-0305e82c3301'; + const out = await validateConnectionId.call(ctx(uuid), req()); + assert.ok(out, 'request options returned'); +}); + +test('an empty or whitespace value passes (the field is optional)', async () => { + for (const value of ['', ' ', undefined]) { + const out = await validateConnectionId.call(ctx(value), req()); + assert.ok(out, `value=${JSON.stringify(value)}`); + } +}); + +test('a source name is rejected in the node, not sent as a 502-in-waiting', async () => { + // The exact values prod saw. + for (const value of ['linear', 'github', 'google_drive']) { + await assert.rejects( + async () => validateConnectionId.call(ctx(value), req()), + (err) => { + assert.match(err.message, /must be a connection UUID/); + assert.match(err.message, new RegExp(value)); + return true; + }, + `value=${value}`, + ); + } +}); + +test('a non-UUID that is not a source name is still rejected', async () => { + await assert.rejects(async () => validateConnectionId.call(ctx('12345'), req()), /connection UUID/); +}); + +test('rejection explains the fix, including the AI-tool case', async () => { + try { + await validateConnectionId.call(ctx('linear'), req()); + assert.fail('should have thrown'); + } catch (err) { + const text = `${err.description ?? ''}`; + assert.match(text, /not the source name/i); + assert.match(text, /empty/i); + assert.match(text, /AI Agent tool/i); + } +}); + +// Evaluate a routing `send.value` expression the way n8n does, so these assert +// what actually goes on the wire rather than the shape of the template string. +const sendValue = (prop, value) => { + const body = prop.routing.send.value.replace(/^=\{\{/, '').replace(/\}\}$/, ''); + return Function('$value', `return (${body});`)(value); +}; + +test('what goes on the wire is what the guard validated', async () => { + // The guard trims before validating, so " " is accepted as "not provided". + // Until this was fixed the routing expression did NOT trim, and " " is + // truthy in JS — so the node shipped it, core put it in the UUID column, and + // the response was `502 "Upstream source error."`: the precise failure this + // field exists to prevent, reproduced against a pre-fix core on 2026-08-12. + const uuid = '3f2504e0-4f89-11d3-9a0c-0305e82c3301'; + + for (const transport of ['body', 'query']) { + const prop = connectionIdProperty({ resource: ['live'], operation: ['search'] }, transport); + + // Anything the guard treats as "not provided" must be OMITTED, not sent. + for (const empty of ['', ' ', '\t\n ', undefined]) { + assert.equal(sendValue(prop, empty), undefined, `${transport}: ${JSON.stringify(empty)}`); + } + + // A real UUID is sent unchanged; a padded one is trimmed, because core + // parses with UUID() and 400s on surrounding whitespace. + assert.equal(sendValue(prop, uuid), uuid, transport); + assert.equal(sendValue(prop, ` ${uuid}\t`), uuid, transport); + } +}); + +test('a non-string from an expression is rejected, not coerced onto the wire', async () => { + // The field is type:'string', but an expression resolves to whatever it + // evaluates to — `={{ $json.id }}` over a numeric column yields a number. + // Treating that as empty passed it through the guard while the routing + // expression still sent it. + for (const value of [123, 0, {}, ['x'], true]) { + await assert.rejects( + async () => validateConnectionId.call(ctx(value), req()), + /must be a connection UUID/, + `value=${JSON.stringify(value)}`, + ); + } +}); + +test('the wire expression survives a non-string without throwing', () => { + // It is evaluated BEFORE preSend, so a bare .trim() on a number would throw a + // raw TypeError and pre-empt the guard's readable error. + const prop = connectionIdProperty({ resource: ['live'], operation: ['search'] }); + assert.equal(sendValue(prop, 123), '123'); + assert.equal(sendValue(prop, null), undefined); +}); + +test('a value the guard rejects never reaches the wire check', async () => { + // Belt and braces: 'linear' would survive the expression (it is truthy and + // unchanged by trim) — it is the preSend guard, not the expression, that + // stops it. Assert both halves so neither can silently regress alone. + const prop = connectionIdProperty({ resource: ['live'], operation: ['search'] }); + assert.equal(sendValue(prop, 'linear'), 'linear'); + await assert.rejects( + async () => validateConnectionId.call(ctx('linear'), req()), + /must be a connection UUID/, + ); +}); + +test('the shared property wires the guard on both transports', () => { + const body = connectionIdProperty({ resource: ['live'], operation: ['search'] }); + const query = connectionIdProperty({ resource: ['live'], operation: ['listResources'] }, 'query'); + + assert.equal(body.routing.send.type, 'body'); + assert.equal(query.routing.send.type, 'query'); + for (const prop of [body, query]) { + assert.equal(prop.name, 'connection_id'); + assert.equal(prop.routing.send.property, 'connection_id'); + assert.equal(prop.routing.send.preSend.length, 1); + // Wording is the model's only input when used as a tool. + assert.match(prop.description, /Leave empty/); + assert.match(prop.description, /NOT the source name/); + } +}); diff --git a/tests/live-output.test.mjs b/tests/live-output.test.mjs index 41311a6..98d6f59 100644 --- a/tests/live-output.test.mjs +++ b/tests/live-output.test.mjs @@ -38,19 +38,37 @@ const PROD_CURSOR_PAGE = { next_cursor: 'opaque-cursor-token', }; -const call = (fn, body) => fn.call(undefined, [], { body, headers: {}, statusCode: 200 }); +/** + * Minimal IExecuteSingleFunctions stand-in. The unwrappers now read the + * Simplify parameter (every /live/* route returns a plain DocumentResponse + * whose hyperdoc tree is otherwise unbounded), so `this` must be a real + * context — n8n always binds one; only this harness ever passed undefined. + */ +const ctx = (options = {}) => ({ + getNodeParameter: (name, fallback) => + name === 'simplify' ? (options.simplify ?? fallback) : fallback, +}); + +const call = (fn, body, options = {}) => + fn.call(ctx(options), [], { body, headers: {}, statusCode: 200 }); // ── unwrapLiveEnvelope (Search / Get Resource) ───────────────────────────── test('search/get: one item per document, envelope indexed/notes merged onto each', async () => { - const out = await call(unwrapLiveEnvelope, { - documents: [ - { resource_id: 'a', title: 'A', document: { type: 'document', children: [] } }, - { resource_id: 'b', title: 'B', document: { type: 'document', children: [] } }, - ], - indexed: true, - notes: ['queued 2 documents'], - }); + // Simplify off: this test is about the ENVELOPE contract (siblings must not + // be dropped), so it asserts the raw shape. Bounding is covered separately. + const out = await call( + unwrapLiveEnvelope, + { + documents: [ + { resource_id: 'a', title: 'A', document: { type: 'document', children: [] } }, + { resource_id: 'b', title: 'B', document: { type: 'document', children: [] } }, + ], + indexed: true, + notes: ['queued 2 documents'], + }, + { simplify: false }, + ); assert.equal(out.length, 2); assert.equal(out[0].json.resource_id, 'a'); assert.equal(out[1].json.resource_id, 'b'); @@ -100,7 +118,8 @@ test('search/get: defaults applied when envelope fields are absent (exclude_none // ── unwrapCursorPage (List Resources) ────────────────────────────────────── test('list: real prod fixture — one item per resource, next_cursor on each, document tree intact', async () => { - const out = await call(unwrapCursorPage, PROD_CURSOR_PAGE); + // Simplify off — asserts the unwrap itself preserves the raw document. + const out = await call(unwrapCursorPage, PROD_CURSOR_PAGE, { simplify: false }); assert.equal(out.length, 1); const item = out[0].json; assert.equal(item.resource_id, 'contact:479239644873'); @@ -123,3 +142,69 @@ test('list: empty or missing items emits zero items, never {json: undefined}', a assert.deepEqual(out, [], `body=${JSON.stringify(body)}`); } }); + +// ── Bounding (ENG-3697: "the node pumps too many tokens") ────────────────── +// +// Every /live/* route returns the plain DocumentResponse — no summary, no +// highlights, body only in the `document` tree — and nothing bounded it before +// 0.7.2. The Live resource is the surface built for an AI-agent node, so it was +// simultaneously the most likely to feed a model and the only one shipping an +// unbounded payload. + +test('live search/get: Simplify (default on) replaces the tree with flattened text', async () => { + const body = 'x'.repeat(5000); + const out = await call(unwrapLiveEnvelope, { + documents: [ + { + resource_id: 'a', + title: 'A', + document: { type: 'document', children: [{ type: 'paragraph', text: body }] }, + }, + ], + indexed: true, + notes: [], + }); + const item = out[0].json; + assert.equal(item.document, undefined, 'hyperdoc tree must be dropped when simplified'); + assert.equal(typeof item.text, 'string'); + assert.equal(item.text.length, 2000, 'text is capped at MAX_SIMPLIFIED_TEXT'); + // Envelope siblings still ride along — bounding must not undo the unwrap contract. + assert.equal(item.indexed, true); + assert.equal(item.resource_id, 'a'); +}); + +test('live list: Simplify (default on) bounds each row and keeps next_cursor', async () => { + const out = await call(unwrapCursorPage, PROD_CURSOR_PAGE); + const item = out[0].json; + assert.equal(item.document, undefined); + assert.equal(item.next_cursor, 'opaque-cursor-token'); + assert.equal(item.resource_id, 'contact:479239644873'); + // The fixture's hyperdoc carries no `text` nodes — its content lives in + // name/email/company. Flattening those to '' and then dropping the tree + // deleted the contact's details outright: the very defect this simplifier + // exists to fix, reintroduced the moment Simplify defaulted on for Live. + assert.match(item.text, /Maria Johnson/); + assert.match(item.text, /emailmaria@hubspot\.com/); + assert.match(item.text, /HubSpot/); + assert.ok(item.text.length <= 2000, `bounded, got ${item.text.length}`); +}); + +test('live: a simplified document is dramatically smaller than the raw one', async () => { + const body = 'y'.repeat(200000); + const envelope = { + documents: [ + { + resource_id: 'big', + document: { type: 'document', children: [{ type: 'paragraph', text: body }] }, + }, + ], + indexed: false, + notes: [], + }; + const [raw] = await call(unwrapLiveEnvelope, envelope, { simplify: false }); + const [bounded] = await call(unwrapLiveEnvelope, envelope); + const rawSize = JSON.stringify(raw.json).length; + const boundedSize = JSON.stringify(bounded.json).length; + assert.ok(rawSize > 100000, `precondition: raw is large (${rawSize})`); + assert.ok(boundedSize < 2500, `bounded must be small, got ${boundedSize}`); +}); diff --git a/tests/simplify.test.mjs b/tests/simplify.test.mjs index b9b1081..1dc24af 100644 --- a/tests/simplify.test.mjs +++ b/tests/simplify.test.mjs @@ -121,16 +121,123 @@ test('falls back to concatenated highlights when the server sends no summary', a assert.equal(out.json.documents[0].text, 'first chunk\n\nsecond chunk'); }); -test('a hit with neither summary nor highlights yields empty text, not undefined', async () => { +test('no summary and no highlights falls back to the flattened hyperdoc tree', async () => { + // Was asserted as `text === ''` until 0.7.2. That was the bug, not the + // contract: only the QUERY path returns ScoredDocumentResponse (the sole + // model carrying summary/highlights). /memories/list, /memories/get/* and + // every /live/* route return the plain DocumentResponse, so the + // highlights-only path emitted empty text AND dropped the tree — deleting + // the content outright. const response = structuredClone(PROD_QUERY_RESPONSE); delete response.documents[0].summary; delete response.documents[0].highlights; const [out] = await simplifyDocuments.call(ctx(), item(response)); + assert.equal(out.json.documents[0].text, MATCHED_TEXT); + assert.equal(out.json.documents[0].document, undefined, 'tree still dropped'); +}); + +test('text is empty only when the document genuinely has no text anywhere', async () => { + const response = structuredClone(PROD_QUERY_RESPONSE); + delete response.documents[0].summary; + delete response.documents[0].highlights; + response.documents[0].document = { type: 'document', id: 'x', children: [] }; + + const [out] = await simplifyDocuments.call(ctx(), item(response)); + assert.equal(out.json.documents[0].text, ''); }); +// ── /memories/list shape (the 0.7.0 regression) ──────────────────────────── +// +// GET /memories/list returns CursorPage[DocumentResponse] (documents.py:267) — +// verified against core: no summary, no highlights. Every fixture in this file +// used to be a QUERY response, which is why nothing caught that Simplify was +// blanking List. + +// Verbatim `GET /memories/list?size=2` response bytes, recorded 2026-08-12 +// against a local core (app 1 / hyperdev) after adding one document through +// `POST /memories/add`. Recorded rather than hand-rolled, matching the fixture +// convention above — an idealized fixture built by reading schemas.py would +// only prove the code agrees with my reading of the schema. +// +// The properties that matter, and that the recording establishes as fact: +// * item keys are exactly resource_id, source, type, title, status, +// collection, metadata, ingested_at, last_modified_at, document_date, +// document — no `summary`, no `highlights`, no `score` +// * the body exists ONLY inside the `document` tree +// * `next_cursor` is absent entirely on a last page (response_model_exclude_none), +// not null — the same variance the google_mail case in live-output.test.mjs hits +const LIST_BODY = + 'The quarterly board review covered three items: pipeline health, the EU cell rollout, and hiring. Pipeline coverage sits at 3.1x. The EU cell is live in eu-west-1 and serving traffic.'; +const RECORDED_LIST_RESPONSE = { + items: [ + { + resource_id: '8GJ-0mBk9j3RHg', + source: 'vault', + type: 'document', + title: 'Q3 board review notes', + status: 'pending', + collection: 'meetings', + metadata: { _content_hash: '063587a0ab6ecc9c' }, + ingested_at: '2026-08-12T16:08:31.514373', + last_modified_at: '2026-08-12T16:08:31.561204', + document_date: '2026-08-12T16:08:31.367402', + document: { + type: 'document', + id: '6ada6477c132', + children: [{ type: 'paragraph', id: '87c170396fbc', text: LIST_BODY }], + title: 'Q3 board review notes', + }, + }, + ], +}; + +test('recorded list response really does lack summary/highlights (fixture guard)', () => { + // Guards the premise the rest of this section rests on. If core ever adds + // summary/highlights to the list path, this fails loudly instead of the + // fallback silently becoming dead code. + const row = RECORDED_LIST_RESPONSE.items[0]; + assert.equal(row.summary, undefined); + assert.equal(row.highlights, undefined); + assert.equal(row.score, undefined); + assert.ok(row.document, 'body lives only in the tree'); +}); + +test('list: simplified rows carry the body text, not an empty string', async () => { + const [out] = await simplifyDocuments.call(ctx(), item(RECORDED_LIST_RESPONSE)); + const row = out.json.items[0]; + + assert.equal(row.text, LIST_BODY); + assert.equal(row.document, undefined, 'tree dropped — that part was always right'); + assert.equal(row.status, RECORDED_LIST_RESPONSE.items[0].status); + assert.equal(row.score, undefined, 'list rows have no score; do not invent one'); +}); + +test('list: a huge body is capped rather than passed through', async () => { + const response = structuredClone(RECORDED_LIST_RESPONSE); + response.items[0].document.children[0].text = 'z'.repeat(500000); + + const [out] = await simplifyDocuments.call(ctx(), item(response)); + const row = out.json.items[0]; + + assert.equal(row.text.length, 2000); + assert.ok(JSON.stringify(row).length < 2600, 'row stays small'); +}); + +test('flattening walks nested children depth-first', async () => { + const response = structuredClone(RECORDED_LIST_RESPONSE); + response.items[0].document.children = [ + { type: 'heading', text: 'Title' }, + { type: 'section', children: [{ type: 'paragraph', text: 'Nested body' }] }, + ]; + + const [out] = await simplifyDocuments.call(ctx(), item(response)); + + assert.equal(out.json.items[0].text, 'Title\nNested body'); +}); + test('a body with no documents array is left alone', async () => { // The app-scoped-empty notice item and any error body flow through here too. const notice = { notice: 'No results — no Act as User was set.' }; @@ -206,3 +313,119 @@ test('list: Simplify = false leaves the page untouched', async () => { assert.deepEqual(out, input); }); + +// ── simplifyOne: GET /memories/get/* (single DocumentResponse) ───────────── +// +// A get returns a bare document, not a result array, so simplifyDocuments +// passed it straight through — one document is exactly where the tree is +// largest per item, making it the operation most worth bounding. + +const { simplifyOne } = require('../dist/nodes/Hyperspell/resources/simplify.js'); + +test('get: the single document is bounded, envelope fields preserved', async () => { + const doc = structuredClone(RECORDED_LIST_RESPONSE.items[0]); + const [out] = await simplifyOne.call(ctx(), [{ json: doc }]); + + assert.equal(out.json.text, LIST_BODY); + assert.equal(out.json.document, undefined); + assert.equal(out.json.resource_id, RECORDED_LIST_RESPONSE.items[0].resource_id); + assert.equal(out.json.title, RECORDED_LIST_RESPONSE.items[0].title); + assert.equal(out.json.collection, 'meetings', 'collection survives simplification'); +}); + +test('get: Simplify = false returns the document untouched', async () => { + const doc = structuredClone(RECORDED_LIST_RESPONSE.items[0]); + const [out] = await simplifyOne.call(ctx({ simplify: false }), [{ json: doc }]); + + assert.deepEqual(out.json, doc); +}); + +test('get: a non-document body (error, notice) passes through untouched', async () => { + const notice = { notice: 'No results — no Act as User was set.' }; + const [out] = await simplifyOne.call(ctx(), [{ json: notice }]); + + assert.deepEqual(out.json, notice); +}); + +// ── structured hyperdoc nodes: content in named fields, not `text` ────────── +// +// Not every hyperdoc node has a `text`. A HubSpot contact comes back as +// {type:'person', name, email, company, children: []} — recorded from prod in +// docs/incidents/2026-06-11-live-resource-hyperdoc-shape.md. A walk that reads +// only text/children flattens that to '', and the simplifier then drops the +// tree, so the email and company vanish. Live → List Resources would have +// shipped exactly that the moment Simplify defaulted on for it. +const { simplifyDocument, MAX_SIMPLIFIED_TEXT } = require( + '../dist/nodes/Hyperspell/resources/simplify.js', +); + +test('a node carrying content in named fields keeps that content', () => { + const out = simplifyDocument({ + resource_id: 'contact:479239644873', + source: 'hubspot', + type: 'person', + title: 'Maria Johnson', + document: { + type: 'person', + id: '73b33537684c', + children: [], + name: 'Maria Johnson', + email: 'emailmaria@hubspot.com', + company: 'HubSpot', + }, + }); + + assert.match(out.text, /Maria Johnson/); + assert.match(out.text, /emailmaria@hubspot\.com/); + assert.match(out.text, /HubSpot/); + // Labelled, because "email: x@y.com" tells a model more than a bare value. + assert.match(out.text, /email: /); + assert.equal(out.document, undefined, 'still bounded — the tree is dropped'); +}); + +test('structural keys are addressing, not content, and stay out of the text', () => { + const out = simplifyDocument({ + resource_id: 'r', + document: { type: 'person', id: 'abc123', children: [], name: 'Ada' }, + }); + + assert.match(out.text, /Ada/); + assert.doesNotMatch(out.text, /abc123/, 'id is addressing'); + assert.doesNotMatch(out.text, /type: person/, 'type is structure'); +}); + +test('a genuinely empty tree still yields empty text', () => { + const out = simplifyDocument({ + resource_id: 'r', + document: { type: 'document', id: 'x', children: [{ type: 'paragraph', children: [] }] }, + }); + + assert.equal(out.text, ''); +}); + +test('separators are charged to the same budget as the text', () => { + // A tree of many tiny nodes: counting only node text let the joined result + // reach ~2x the documented cap in newlines alone. + const children = Array.from({ length: 3000 }, () => ({ type: 'paragraph', text: 'x' })); + const out = simplifyDocument({ + resource_id: 'r', + document: { type: 'document', children }, + }); + + assert.ok( + out.text.length <= MAX_SIMPLIFIED_TEXT, + `must not exceed ${MAX_SIMPLIFIED_TEXT}, got ${out.text.length}`, + ); +}); + +test('deep text is still collected and still capped', () => { + const out = simplifyDocument({ + resource_id: 'r', + document: { + type: 'document', + children: [{ type: 'section', children: [{ type: 'paragraph', text: 'z'.repeat(9000) }] }], + }, + }); + + assert.equal(out.text.length, MAX_SIMPLIFIED_TEXT); +});