diff --git a/package.json b/package.json index 0de39840af..ffbee84446 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "packages/sdk/server-ai/examples/chat-observability", "packages/sdk/server-ai/examples/openai-observability", "packages/sdk/server-ai/examples/vercel-ai", + "packages/sdk/server-ai/examples/agent-graph-traversal", "packages/telemetry/browser-telemetry", "packages/sdk/combined-browser", "packages/sdk/shopify-oxygen", diff --git a/packages/sdk/server-ai/__tests__/AgentGraphDefinition.test.ts b/packages/sdk/server-ai/__tests__/AgentGraphDefinition.test.ts new file mode 100644 index 0000000000..8839a3474a --- /dev/null +++ b/packages/sdk/server-ai/__tests__/AgentGraphDefinition.test.ts @@ -0,0 +1,418 @@ +import { randomUUID } from 'crypto'; + +import { LDContext } from '@launchdarkly/js-server-sdk-common'; + +import { LDAIAgentConfig } from '../src/api/config'; +import { AgentGraphDefinition } from '../src/api/graph/AgentGraphDefinition'; +import { LDAgentGraphFlagValue, LDGraphEdge } from '../src/api/graph/types'; +import { LDClientMin } from '../src/LDClientMin'; +import { LDGraphTrackerImpl } from '../src/LDGraphTrackerImpl'; + +const mockLdClient: LDClientMin = { + track: jest.fn(), + variation: jest.fn(), +}; + +const testContext: LDContext = { kind: 'user', key: 'test-user' }; + +// --------------------------------------------------------------------------- +// Helper builders +// --------------------------------------------------------------------------- + +function makeAgentConfig(key: string, enabled = true): LDAIAgentConfig { + return { key, enabled, instructions: `You are ${key}.` } as LDAIAgentConfig; +} + +function makeGraph( + root: string, + edges: Record = {}, + variationKey?: string, + version = 1, +): LDAgentGraphFlagValue { + return { + _ldMeta: { variationKey, version }, + root, + edges, + }; +} + +function makeDefinition( + graph: LDAgentGraphFlagValue, + agentConfigs: Record, + enabled = true, +): AgentGraphDefinition { + const nodes = AgentGraphDefinition.buildNodes(graph, agentConfigs); + return new AgentGraphDefinition( + graph, + nodes, + enabled, + () => + new LDGraphTrackerImpl( + mockLdClient, + randomUUID(), + graph.root, + // eslint-disable-next-line no-underscore-dangle + graph._ldMeta?.variationKey, + // eslint-disable-next-line no-underscore-dangle + graph._ldMeta?.version ?? 1, + testContext, + ), + ); +} + +// --------------------------------------------------------------------------- +// buildNodes +// --------------------------------------------------------------------------- + +it('buildNodes creates a node for every unique key in the graph', () => { + const graph = makeGraph('root', { + root: [{ key: 'child-a' }, { key: 'child-b' }], + 'child-a': [{ key: 'leaf' }], + }); + const configs: Record = { + root: makeAgentConfig('root'), + 'child-a': makeAgentConfig('child-a'), + 'child-b': makeAgentConfig('child-b'), + leaf: makeAgentConfig('leaf'), + }; + + const nodes = AgentGraphDefinition.buildNodes(graph, configs); + expect(Object.keys(nodes).sort()).toEqual(['child-a', 'child-b', 'leaf', 'root']); +}); + +it('buildNodes skips keys whose agent config is missing', () => { + const graph = makeGraph('root', { root: [{ key: 'orphan' }] }); + const nodes = AgentGraphDefinition.buildNodes(graph, { root: makeAgentConfig('root') }); + expect(nodes.root).toBeDefined(); + expect(nodes.orphan).toBeUndefined(); +}); + +it('buildNodes assigns correct edges to each node', () => { + const graph = makeGraph('root', { + root: [{ key: 'child', handoff: { someOption: true } }], + }); + const configs = { + root: makeAgentConfig('root'), + child: makeAgentConfig('child'), + }; + const nodes = AgentGraphDefinition.buildNodes(graph, configs); + expect(nodes.root.getEdges()).toEqual([{ key: 'child', handoff: { someOption: true } }]); + expect(nodes.child.getEdges()).toEqual([]); +}); + +// --------------------------------------------------------------------------- +// collectAllKeys +// --------------------------------------------------------------------------- + +it('collectAllKeys includes root, edge sources, and edge targets', () => { + const graph = makeGraph('root', { + root: [{ key: 'a' }, { key: 'b' }], + a: [{ key: 'c' }], + }); + const keys = AgentGraphDefinition.collectAllKeys(graph); + expect([...keys].sort()).toEqual(['a', 'b', 'c', 'root']); +}); + +it('collectAllKeys works for a graph with no edges', () => { + const graph = makeGraph('solo'); + const keys = AgentGraphDefinition.collectAllKeys(graph); + expect([...keys]).toEqual(['solo']); +}); + +// --------------------------------------------------------------------------- +// enabled +// --------------------------------------------------------------------------- + +it('enabled reflects the value passed at construction', () => { + const graph = makeGraph('r'); + const enabled = makeDefinition(graph, { r: makeAgentConfig('r') }, true); + expect(enabled.enabled).toBe(true); + + const disabled = makeDefinition(graph, { r: makeAgentConfig('r') }, false); + expect(disabled.enabled).toBe(false); +}); + +// --------------------------------------------------------------------------- +// rootNode / getNode / terminalNodes +// --------------------------------------------------------------------------- + +it('rootNode returns the root node', () => { + const graph = makeGraph('root', { root: [{ key: 'leaf' }] }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + leaf: makeAgentConfig('leaf'), + }); + expect(def.rootNode().getKey()).toBe('root'); +}); + +it('getNode returns null for unknown key', () => { + const graph = makeGraph('root'); + const def = makeDefinition(graph, { root: makeAgentConfig('root') }); + expect(def.getNode('nonexistent')).toBeNull(); +}); + +it('terminalNodes returns nodes with no outgoing edges', () => { + const graph = makeGraph('root', { + root: [{ key: 'mid' }], + mid: [{ key: 'leaf-a' }, { key: 'leaf-b' }], + }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + mid: makeAgentConfig('mid'), + 'leaf-a': makeAgentConfig('leaf-a'), + 'leaf-b': makeAgentConfig('leaf-b'), + }); + const terminalKeys = def + .terminalNodes() + .map((n) => n.getKey()) + .sort(); + expect(terminalKeys).toEqual(['leaf-a', 'leaf-b']); +}); + +// --------------------------------------------------------------------------- +// getChildNodes / getParentNodes +// --------------------------------------------------------------------------- + +it('getChildNodes returns direct children', () => { + const graph = makeGraph('root', { + root: [{ key: 'a' }, { key: 'b' }], + }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + a: makeAgentConfig('a'), + b: makeAgentConfig('b'), + }); + const childKeys = def + .getChildNodes('root') + .map((n) => n.getKey()) + .sort(); + expect(childKeys).toEqual(['a', 'b']); +}); + +it('getChildNodes returns empty array for terminal node', () => { + const graph = makeGraph('root'); + const def = makeDefinition(graph, { root: makeAgentConfig('root') }); + expect(def.getChildNodes('root')).toEqual([]); +}); + +it('getChildNodes returns empty array for unknown key', () => { + const graph = makeGraph('root'); + const def = makeDefinition(graph, { root: makeAgentConfig('root') }); + expect(def.getChildNodes('unknown')).toEqual([]); +}); + +it('getParentNodes returns nodes that have direct edges to the given key', () => { + const graph = makeGraph('root', { + root: [{ key: 'child' }], + sibling: [{ key: 'child' }], + }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + sibling: makeAgentConfig('sibling'), + child: makeAgentConfig('child'), + }); + const parentKeys = def + .getParentNodes('child') + .map((n) => n.getKey()) + .sort(); + expect(parentKeys).toEqual(['root', 'sibling']); +}); + +it('getParentNodes returns empty array for root node', () => { + const graph = makeGraph('root', { root: [{ key: 'child' }] }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + child: makeAgentConfig('child'), + }); + expect(def.getParentNodes('root')).toEqual([]); +}); + +// --------------------------------------------------------------------------- +// traverse +// --------------------------------------------------------------------------- + +it('traverse calls fn for every node in BFS order (root first)', () => { + // root + // / \ + // a b + // | + // c + const graph = makeGraph('root', { + root: [{ key: 'a' }, { key: 'b' }], + a: [{ key: 'c' }], + }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + a: makeAgentConfig('a'), + b: makeAgentConfig('b'), + c: makeAgentConfig('c'), + }); + + const order: string[] = []; + def.traverse((node) => { + order.push(node.getKey()); + }); + + expect(order[0]).toBe('root'); + // a and b must both appear before c + const aIdx = order.indexOf('a'); + const bIdx = order.indexOf('b'); + const cIdx = order.indexOf('c'); + expect(aIdx).toBeLessThan(cIdx); + expect(bIdx).toBeLessThan(cIdx); + expect(order).toHaveLength(4); +}); + +it('traverse stores fn return values in execution context', () => { + const graph = makeGraph('root', { root: [{ key: 'child' }] }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + child: makeAgentConfig('child'), + }); + + const contextCaptures: Record[] = []; + def.traverse((node, ctx) => { + contextCaptures.push({ ...ctx }); + return `result-of-${node.getKey()}`; + }); + + // After root is processed, the child's context should contain root's result + expect(contextCaptures[1]).toHaveProperty('root', 'result-of-root'); +}); + +it('traverse accepts and uses initial execution context', () => { + const graph = makeGraph('root'); + const def = makeDefinition(graph, { root: makeAgentConfig('root') }); + + const captured: Record[] = []; + def.traverse( + (node, ctx) => { + captured.push({ ...ctx }); + }, + { initialKey: 'initialValue' }, + ); + + expect(captured[0]).toHaveProperty('initialKey', 'initialValue'); +}); + +it('traverse handles a single-node graph', () => { + const graph = makeGraph('solo'); + const def = makeDefinition(graph, { solo: makeAgentConfig('solo') }); + const visited: string[] = []; + def.traverse((node) => { + visited.push(node.getKey()); + }); + expect(visited).toEqual(['solo']); +}); + +// --------------------------------------------------------------------------- +// reverseTraverse +// --------------------------------------------------------------------------- + +it('reverseTraverse processes terminal nodes before their parents, root last', () => { + // root + // / \ + // a b ← mid-level + // | + // c ← terminal (deepest) + const graph = makeGraph('root', { + root: [{ key: 'a' }, { key: 'b' }], + a: [{ key: 'c' }], + }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + a: makeAgentConfig('a'), + b: makeAgentConfig('b'), + c: makeAgentConfig('c'), + }); + + const order: string[] = []; + def.reverseTraverse((node) => { + order.push(node.getKey()); + }); + + expect(order[order.length - 1]).toBe('root'); // root always last + // c must appear before a (c is a descendant of a) + expect(order.indexOf('c')).toBeLessThan(order.indexOf('a')); + // all four nodes visited + expect(order.sort()).toEqual(['a', 'b', 'c', 'root']); +}); + +it('reverseTraverse stores fn return values in execution context', () => { + const graph = makeGraph('root', { root: [{ key: 'child' }] }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + child: makeAgentConfig('child'), + }); + + const contextWhenRootRuns: Record[] = []; + def.reverseTraverse((node, ctx) => { + if (node.getKey() === 'root') { + contextWhenRootRuns.push({ ...ctx }); + } + return `result-of-${node.getKey()}`; + }); + + // root runs last; at that point, child's result should be in context + expect(contextWhenRootRuns[0]).toHaveProperty('child', 'result-of-child'); +}); + +it('reverseTraverse visits a node with multiple parents only once', () => { + // root → a → d → c + // root → b → c ← c has two parents + const graph = makeGraph('root', { + root: [{ key: 'a' }, { key: 'b' }], + a: [{ key: 'd' }], + b: [{ key: 'c' }], + d: [{ key: 'c' }], + }); + const def = makeDefinition(graph, { + root: makeAgentConfig('root'), + a: makeAgentConfig('a'), + b: makeAgentConfig('b'), + c: makeAgentConfig('c'), + d: makeAgentConfig('d'), + }); + + const order: string[] = []; + def.reverseTraverse((node) => { + order.push(node.getKey()); + }); + + // c is the only terminal — it goes first + expect(order[0]).toBe('c'); + // root is always last + expect(order[order.length - 1]).toBe('root'); + // every node visited exactly once + expect(order.sort()).toEqual(['a', 'b', 'c', 'd', 'root']); +}); + +it('reverseTraverse visits each node once on a cyclic graph', () => { + // A → B → A (no terminals) + const graph = makeGraph('a', { + a: [{ key: 'b' }], + b: [{ key: 'a' }], + }); + const def = makeDefinition(graph, { + a: makeAgentConfig('a'), + b: makeAgentConfig('b'), + }); + + const visited: string[] = []; + def.reverseTraverse((node) => { + visited.push(node.getKey()); + }); + + // No terminals → returns without visiting anything (same as Python) + expect(visited).toEqual([]); +}); + +// --------------------------------------------------------------------------- +// getConfig +// --------------------------------------------------------------------------- + +it('getConfig returns the raw flag value', () => { + const graph = makeGraph('root', {}, 'var-key', 5); + const def = makeDefinition(graph, { root: makeAgentConfig('root') }); + expect(def.getConfig()).toBe(graph); +}); diff --git a/packages/sdk/server-ai/__tests__/LDGraphTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDGraphTrackerImpl.test.ts index 77af551302..9f734eb5d0 100644 --- a/packages/sdk/server-ai/__tests__/LDGraphTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDGraphTrackerImpl.test.ts @@ -4,446 +4,351 @@ import { LDClientMin } from '../src/LDClientMin'; import { LDGraphTrackerImpl } from '../src/LDGraphTrackerImpl'; const mockTrack = jest.fn(); +const mockWarn = jest.fn(); const mockLdClient: LDClientMin = { track: mockTrack, variation: jest.fn(), + logger: { warn: mockWarn, error: jest.fn(), info: jest.fn(), debug: jest.fn() }, }; const testContext: LDContext = { kind: 'user', key: 'test-user' }; -const graphKey = 'test-graph'; +const graphKey = 'my-agent-graph'; const variationKey = 'v1'; const version = 2; -const getExpectedTrackData = () => ({ - graphKey, - variationKey, - version, -}); +const makeTracker = (runId = 'test-run-id') => + new LDGraphTrackerImpl(mockLdClient, runId, graphKey, variationKey, version, testContext); beforeEach(() => { jest.clearAllMocks(); }); -it('returns track data', () => { +// --------------------------------------------------------------------------- +// getTrackData +// --------------------------------------------------------------------------- + +it('returns correct track data with variationKey', () => { + const tracker = makeTracker('fixed-run-id'); + expect(tracker.getTrackData()).toEqual({ + runId: 'fixed-run-id', + graphKey, + version, + variationKey, + }); +}); + +it('omits variationKey when not provided', () => { const tracker = new LDGraphTrackerImpl( mockLdClient, + 'some-run-id', graphKey, - variationKey, + undefined, version, testContext, ); + const data = tracker.getTrackData(); + expect(data.variationKey).toBeUndefined(); + expect(data.graphKey).toBe(graphKey); + expect(data.version).toBe(version); + expect(data.runId).toBe('some-run-id'); +}); - expect(tracker.getTrackData()).toEqual(getExpectedTrackData()); +it('uses provided runId', () => { + const tracker = makeTracker('my-custom-run-id'); + expect(tracker.getTrackData().runId).toBe('my-custom-run-id'); }); -it('tracks invocation success', () => { +// --------------------------------------------------------------------------- +// resumptionToken round-trip +// --------------------------------------------------------------------------- + +it('encodes a resumption token with correct field order', () => { + const tracker = makeTracker('550e8400-e29b-41d4-a716-446655440000'); + const token = tracker.resumptionToken; + const decoded = Buffer.from(token, 'base64url').toString('utf8'); + expect(decoded).toBe( + '{"runId":"550e8400-e29b-41d4-a716-446655440000","graphKey":"my-agent-graph","variationKey":"v1","version":2}', + ); +}); + +it('omits variationKey from token when not set', () => { const tracker = new LDGraphTrackerImpl( mockLdClient, + 'run-abc', graphKey, - variationKey, + undefined, version, testContext, ); + const token = tracker.resumptionToken; + const decoded = Buffer.from(token, 'base64url').toString('utf8'); + expect(decoded).toBe('{"runId":"run-abc","graphKey":"my-agent-graph","version":2}'); +}); + +it('fromResumptionToken reconstructs the tracker with original runId', () => { + const original = makeTracker('orig-run-id'); + const token = original.resumptionToken; + + const reconstructed = LDGraphTrackerImpl.fromResumptionToken(token, mockLdClient, testContext); + expect(reconstructed.getTrackData()).toEqual({ + runId: 'orig-run-id', + graphKey, + version, + variationKey, + }); +}); + +// --------------------------------------------------------------------------- +// getSummary +// --------------------------------------------------------------------------- + +it('returns an empty summary initially', () => { + const tracker = makeTracker('r'); + expect(tracker.getSummary()).toEqual({}); +}); + +it('returns a copy of the summary (not a reference)', () => { + const tracker = makeTracker('r'); tracker.trackInvocationSuccess(); + const summary1 = tracker.getSummary(); + const summary2 = tracker.getSummary(); + expect(summary1).not.toBe(summary2); + expect(summary1).toEqual(summary2); +}); + +// --------------------------------------------------------------------------- +// trackInvocationSuccess / trackInvocationFailure – at-most-once +// --------------------------------------------------------------------------- +it('trackInvocationSuccess sets success=true and emits event', () => { + const tracker = makeTracker('r'); + tracker.trackInvocationSuccess(); + expect(tracker.getSummary().success).toBe(true); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:invocation_success', testContext, - getExpectedTrackData(), + tracker.getTrackData(), 1, ); }); -it('tracks invocation failure', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); +it('trackInvocationFailure sets success=false and emits event', () => { + const tracker = makeTracker('r'); tracker.trackInvocationFailure(); - + expect(tracker.getSummary().success).toBe(false); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:invocation_failure', testContext, - getExpectedTrackData(), + tracker.getTrackData(), 1, ); }); -it('tracks latency', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, +it('drops second trackInvocationSuccess call and warns', () => { + const tracker = makeTracker('r'); + tracker.trackInvocationSuccess(); + tracker.trackInvocationSuccess(); + expect(mockTrack).toHaveBeenCalledTimes(1); + expect(mockWarn).toHaveBeenCalledWith( + expect.stringContaining('invocation success/failure already recorded for this run'), + ); +}); + +it('drops trackInvocationFailure after trackInvocationSuccess and warns', () => { + const tracker = makeTracker('r'); + tracker.trackInvocationSuccess(); + tracker.trackInvocationFailure(); + expect(mockTrack).toHaveBeenCalledTimes(1); + expect(mockWarn).toHaveBeenCalledWith( + expect.stringContaining('invocation success/failure already recorded for this run'), ); - tracker.trackLatency(1500); +}); +// --------------------------------------------------------------------------- +// trackLatency – at-most-once +// --------------------------------------------------------------------------- + +it('trackLatency sets durationMs and emits event', () => { + const tracker = makeTracker('r'); + tracker.trackLatency(1234); + expect(tracker.getSummary().durationMs).toBe(1234); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:latency', testContext, - getExpectedTrackData(), - 1500, + tracker.getTrackData(), + 1234, ); }); -it('tracks total tokens', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackTotalTokens({ total: 200, input: 80, output: 120 }); +it('drops second trackLatency call and warns', () => { + const tracker = makeTracker('r'); + tracker.trackLatency(100); + tracker.trackLatency(200); + expect(mockTrack).toHaveBeenCalledTimes(1); + expect(tracker.getSummary().durationMs).toBe(100); + expect(mockWarn).toHaveBeenCalled(); +}); +// --------------------------------------------------------------------------- +// trackTotalTokens – at-most-once +// --------------------------------------------------------------------------- + +it('trackTotalTokens sets tokens and emits event with total as metric value', () => { + const tracker = makeTracker('r'); + const tokens = { total: 500, input: 200, output: 300 }; + tracker.trackTotalTokens(tokens); + expect(tracker.getSummary().tokens).toEqual(tokens); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:total_tokens', testContext, - getExpectedTrackData(), - 200, + tracker.getTrackData(), + 500, ); }); -it('does not track total tokens when total is zero', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackTotalTokens({ total: 0, input: 0, output: 0 }); - - expect(mockTrack).not.toHaveBeenCalled(); +it('drops second trackTotalTokens call and warns', () => { + const tracker = makeTracker('r'); + tracker.trackTotalTokens({ total: 100, input: 50, output: 50 }); + tracker.trackTotalTokens({ total: 200, input: 100, output: 100 }); + expect(mockTrack).toHaveBeenCalledTimes(1); + expect(tracker.getSummary().tokens?.total).toBe(100); + expect(mockWarn).toHaveBeenCalled(); }); -it('tracks path', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - const path = ['node-a', 'node-b', 'node-c']; - tracker.trackPath(path); +// --------------------------------------------------------------------------- +// trackPath – at-most-once +// --------------------------------------------------------------------------- +it('trackPath sets path and emits event with path in data payload', () => { + const tracker = makeTracker('r'); + const path = ['root-agent', 'research-agent', 'write-agent']; + tracker.trackPath(path); + expect(tracker.getSummary().path).toEqual(path); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:path', testContext, - { ...getExpectedTrackData(), path }, + { ...tracker.getTrackData(), path }, 1, ); }); -it('tracks judge result', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); +it('drops second trackPath call and warns', () => { + const tracker = makeTracker('r'); + tracker.trackPath(['a', 'b']); + tracker.trackPath(['c', 'd']); + expect(mockTrack).toHaveBeenCalledTimes(1); + expect(tracker.getSummary().path).toEqual(['a', 'b']); + expect(mockWarn).toHaveBeenCalled(); +}); + +// --------------------------------------------------------------------------- +// trackJudgeResult – NOT at-most-once +// --------------------------------------------------------------------------- + +it('trackJudgeResult emits an event for a sampled, successful result', () => { + const tracker = makeTracker('r'); tracker.trackJudgeResult({ - judgeConfigKey: 'my-judge', + judgeConfigKey: 'judge-1', + metricKey: 'relevance-score', + score: 0.9, + reasoning: 'good', success: true, sampled: true, - score: 0.9, - reasoning: 'Relevant', - metricKey: 'relevance', }); - + expect(mockTrack).toHaveBeenCalledTimes(1); expect(mockTrack).toHaveBeenCalledWith( - 'relevance', + 'relevance-score', testContext, - { ...getExpectedTrackData(), judgeConfigKey: 'my-judge' }, + { ...tracker.getTrackData(), judgeConfigKey: 'judge-1' }, 0.9, ); }); -it('tracks judge result without judgeConfigKey', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); +it('trackJudgeResult emits event without judgeConfigKey', () => { + const tracker = makeTracker('r'); tracker.trackJudgeResult({ + metricKey: 'relevance-score', + score: 0.7, success: true, sampled: true, - score: 0.7, - reasoning: 'Somewhat relevant', - metricKey: 'relevance', }); - - expect(mockTrack).toHaveBeenCalledWith('relevance', testContext, getExpectedTrackData(), 0.7); -}); - -it('does not track judge result when not sampled', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, + expect(mockTrack).toHaveBeenCalledWith( + 'relevance-score', testContext, + tracker.getTrackData(), + 0.7, ); - tracker.trackJudgeResult({ - judgeConfigKey: 'my-judge', - success: false, - sampled: false, - }); +}); + +it('trackJudgeResult can fire multiple times', () => { + const tracker = makeTracker('r'); + tracker.trackJudgeResult({ metricKey: 'relevance', score: 0.5, success: true, sampled: true }); + tracker.trackJudgeResult({ metricKey: 'relevance', score: 0.7, success: true, sampled: true }); + expect(mockTrack).toHaveBeenCalledTimes(2); + expect(mockWarn).not.toHaveBeenCalled(); +}); +it('trackJudgeResult does not emit when not sampled', () => { + const tracker = makeTracker('r'); + tracker.trackJudgeResult({ judgeConfigKey: 'j', success: false, sampled: false }); expect(mockTrack).not.toHaveBeenCalled(); }); -it('does not track judge result when success is false', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); +it('trackJudgeResult does not emit when success is false', () => { + const tracker = makeTracker('r'); tracker.trackJudgeResult({ - judgeConfigKey: 'my-judge', + judgeConfigKey: 'j', + metricKey: 'relevance', + score: 0.9, success: false, sampled: true, - score: 0.9, - metricKey: 'relevance', }); - expect(mockTrack).not.toHaveBeenCalled(); }); -it('tracks redirect', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackRedirect('agent-a', 'agent-b'); +// --------------------------------------------------------------------------- +// Edge-level methods – multi-fire, NOT at-most-once +// --------------------------------------------------------------------------- +it('trackRedirect emits event with sourceKey and redirectedTarget', () => { + const tracker = makeTracker('r'); + tracker.trackRedirect('source-agent', 'redirected-agent'); expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:redirect', testContext, - { ...getExpectedTrackData(), sourceKey: 'agent-a', redirectedTarget: 'agent-b' }, + { ...tracker.getTrackData(), sourceKey: 'source-agent', redirectedTarget: 'redirected-agent' }, 1, ); }); -it('tracks handoff success', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); +it('trackHandoffSuccess emits event with sourceKey and targetKey', () => { + const tracker = makeTracker('r'); tracker.trackHandoffSuccess('agent-a', 'agent-b'); - expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:handoff_success', testContext, - { ...getExpectedTrackData(), sourceKey: 'agent-a', targetKey: 'agent-b' }, + { ...tracker.getTrackData(), sourceKey: 'agent-a', targetKey: 'agent-b' }, 1, ); }); -it('tracks handoff failure', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); +it('trackHandoffFailure emits event with sourceKey and targetKey', () => { + const tracker = makeTracker('r'); tracker.trackHandoffFailure('agent-a', 'agent-b'); - expect(mockTrack).toHaveBeenCalledWith( '$ld:ai:graph:handoff_failure', testContext, - { ...getExpectedTrackData(), sourceKey: 'agent-a', targetKey: 'agent-b' }, + { ...tracker.getTrackData(), sourceKey: 'agent-a', targetKey: 'agent-b' }, 1, ); }); -it('returns empty summary when no metrics tracked', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - - expect(tracker.getSummary()).toEqual({}); -}); - -it('summarizes tracked graph metrics', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - - tracker.trackInvocationSuccess(); - tracker.trackLatency(2000); - tracker.trackTotalTokens({ total: 300, input: 100, output: 200 }); - tracker.trackPath(['node-a', 'node-b']); - - expect(tracker.getSummary()).toEqual({ - success: true, - durationMs: 2000, - tokens: { total: 300, input: 100, output: 200 }, - path: ['node-a', 'node-b'], - }); -}); - -describe('at-most-once semantics for graph-level metrics', () => { - it('drops duplicate trackInvocationSuccess calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackInvocationSuccess(); - tracker.trackInvocationSuccess(); - - expect(mockTrack).toHaveBeenCalledTimes(1); - }); - - it('drops trackInvocationFailure after trackInvocationSuccess', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackInvocationSuccess(); - tracker.trackInvocationFailure(); - - expect(mockTrack).toHaveBeenCalledTimes(1); - expect(mockTrack).toHaveBeenCalledWith( - '$ld:ai:graph:invocation_success', - expect.anything(), - expect.anything(), - expect.anything(), - ); - }); - - it('drops duplicate trackLatency calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackLatency(1000); - tracker.trackLatency(2000); - - expect(mockTrack).toHaveBeenCalledTimes(1); - expect(mockTrack).toHaveBeenCalledWith( - '$ld:ai:graph:latency', - testContext, - getExpectedTrackData(), - 1000, - ); - }); - - it('drops duplicate trackTotalTokens calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackTotalTokens({ total: 100, input: 40, output: 60 }); - tracker.trackTotalTokens({ total: 200, input: 80, output: 120 }); - - expect(mockTrack).toHaveBeenCalledTimes(1); - expect(mockTrack).toHaveBeenCalledWith( - '$ld:ai:graph:total_tokens', - testContext, - getExpectedTrackData(), - 100, - ); - }); - - it('drops duplicate trackPath calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackPath(['node-a']); - tracker.trackPath(['node-b', 'node-c']); - - expect(mockTrack).toHaveBeenCalledTimes(1); - expect(mockTrack).toHaveBeenCalledWith( - '$ld:ai:graph:path', - testContext, - { ...getExpectedTrackData(), path: ['node-a'] }, - 1, - ); - }); -}); - -describe('edge-level methods can be called multiple times', () => { - it('allows multiple trackRedirect calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackRedirect('a', 'b'); - tracker.trackRedirect('b', 'c'); - - expect(mockTrack).toHaveBeenCalledTimes(2); - }); - - it('allows multiple trackHandoffSuccess calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackHandoffSuccess('a', 'b'); - tracker.trackHandoffSuccess('b', 'c'); - - expect(mockTrack).toHaveBeenCalledTimes(2); - }); - - it('allows multiple trackHandoffFailure calls', () => { - const tracker = new LDGraphTrackerImpl( - mockLdClient, - graphKey, - variationKey, - version, - testContext, - ); - tracker.trackHandoffFailure('a', 'b'); - tracker.trackHandoffFailure('b', 'c'); - - expect(mockTrack).toHaveBeenCalledTimes(2); - }); +it('edge-level methods can fire multiple times without warning', () => { + const tracker = makeTracker('r'); + tracker.trackHandoffSuccess('a', 'b'); + tracker.trackHandoffSuccess('a', 'b'); + tracker.trackRedirect('a', 'c'); + tracker.trackHandoffFailure('x', 'y'); + expect(mockTrack).toHaveBeenCalledTimes(4); + expect(mockWarn).not.toHaveBeenCalled(); }); diff --git a/packages/sdk/server-ai/__tests__/agentGraph.test.ts b/packages/sdk/server-ai/__tests__/agentGraph.test.ts new file mode 100644 index 0000000000..e5a52d836f --- /dev/null +++ b/packages/sdk/server-ai/__tests__/agentGraph.test.ts @@ -0,0 +1,200 @@ +import { LDContext } from '@launchdarkly/js-server-sdk-common'; + +import { AgentGraphDefinition } from '../src/api/graph/AgentGraphDefinition'; +import { LDAIClientImpl } from '../src/LDAIClientImpl'; +import { LDClientMin } from '../src/LDClientMin'; + +// --------------------------------------------------------------------------- +// Mocks +// --------------------------------------------------------------------------- + +const mockTrack = jest.fn(); +const mockVariation = jest.fn(); +const mockDebug = jest.fn(); + +const mockLdClient: LDClientMin = { + track: mockTrack, + variation: mockVariation, + logger: { + debug: mockDebug, + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, +}; + +const testContext: LDContext = { kind: 'user', key: 'test-user' }; + +const makeClient = () => new LDAIClientImpl(mockLdClient); + +beforeEach(() => { + jest.clearAllMocks(); +}); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function makeGraphFlagValue( + root: string, + edges: Record> = {}, + variationKey = 'v1', + version = 1, +) { + return { _ldMeta: { variationKey, version }, root, edges }; +} + +function makeAgentFlagValue(key: string, enabled = true) { + return { + _ldMeta: { variationKey: `${key}-v1`, enabled, version: 1, mode: 'agent' }, + instructions: `Instructions for ${key}`, + }; +} + +// --------------------------------------------------------------------------- +// agentGraph – disabled / validation failures +// --------------------------------------------------------------------------- + +it('returns a disabled graph when _ldMeta.enabled is false', async () => { + const client = makeClient(); + mockVariation.mockResolvedValueOnce({ _ldMeta: { enabled: false }, root: 'root' }); + const graph = await client.agentGraph('my-graph', testContext); + expect(graph).toBeInstanceOf(AgentGraphDefinition); + expect(graph.enabled).toBe(false); +}); + +it('logs debug when graph is disabled via _ldMeta.enabled', async () => { + const client = makeClient(); + mockVariation.mockResolvedValueOnce({ _ldMeta: { enabled: false }, root: 'root' }); + await client.agentGraph('my-graph', testContext); + expect(mockDebug).toHaveBeenCalledWith(expect.stringContaining('disabled')); +}); + +it('returns a disabled graph when graph flag has no root', async () => { + const client = makeClient(); + mockVariation.mockResolvedValueOnce({ root: '' }); + const graph = await client.agentGraph('my-graph', testContext); + expect(graph).toBeInstanceOf(AgentGraphDefinition); + expect(graph.enabled).toBe(false); +}); + +it('logs debug when graph has no root', async () => { + const client = makeClient(); + mockVariation.mockResolvedValueOnce({ root: '' }); + await client.agentGraph('my-graph', testContext); + expect(mockDebug).toHaveBeenCalledWith(expect.stringContaining('not fetchable')); +}); + +it('returns a disabled graph when a node is unconnected (not reachable from root)', async () => { + const client = makeClient(); + const graphValue = makeGraphFlagValue('root', { + root: [{ key: 'child' }], + orphan: [{ key: 'other' }], + }); + mockVariation.mockResolvedValueOnce(graphValue); + const graph = await client.agentGraph('my-graph', testContext); + expect(graph).toBeInstanceOf(AgentGraphDefinition); + expect(graph.enabled).toBe(false); + expect(mockDebug).toHaveBeenCalledWith(expect.stringContaining('unconnected node')); +}); + +it('returns an enabled graph and traverses a cyclic graph (each node visited once)', async () => { + const client = makeClient(); + const graphValue = makeGraphFlagValue('a', { + a: [{ key: 'b' }], + b: [{ key: 'a' }], + }); + mockVariation + .mockResolvedValueOnce(graphValue) + .mockResolvedValue(makeAgentFlagValue('agent', true)); + + const graph = await client.agentGraph('my-graph', testContext); + expect(graph.enabled).toBe(true); + + const visited: string[] = []; + graph.traverse((node) => { + visited.push(node.getKey()); + }); + expect(visited.sort()).toEqual(['a', 'b']); +}); + +it('returns a disabled graph when a child agent config is disabled', async () => { + const client = makeClient(); + const graphValue = makeGraphFlagValue('root', { root: [{ key: 'child' }] }); + mockVariation + .mockResolvedValueOnce(graphValue) + .mockResolvedValueOnce(makeAgentFlagValue('root', true)) + .mockResolvedValueOnce(makeAgentFlagValue('child', false)); + const graph = await client.agentGraph('my-graph', testContext); + expect(graph).toBeInstanceOf(AgentGraphDefinition); + expect(graph.enabled).toBe(false); + expect(mockDebug).toHaveBeenCalledWith(expect.stringContaining('not enabled')); +}); + +// --------------------------------------------------------------------------- +// agentGraph – success path +// --------------------------------------------------------------------------- + +it('returns an enabled graph for a valid graph with a single node', async () => { + const client = makeClient(); + const graphValue = makeGraphFlagValue('solo-agent'); + mockVariation + .mockResolvedValueOnce(graphValue) + .mockResolvedValueOnce(makeAgentFlagValue('solo-agent')); + const graph = await client.agentGraph('my-graph', testContext); + expect(graph).toBeInstanceOf(AgentGraphDefinition); + expect(graph.enabled).toBe(true); + expect(graph.rootNode().getKey()).toBe('solo-agent'); +}); + +it('returns an enabled graph with correct nodes for multi-node graph', async () => { + const client = makeClient(); + const graphValue = makeGraphFlagValue('root', { + root: [{ key: 'child-a' }, { key: 'child-b' }], + 'child-a': [{ key: 'leaf' }], + }); + mockVariation + .mockResolvedValueOnce(graphValue) + .mockResolvedValue(makeAgentFlagValue('agent', true)); + + const graph = await client.agentGraph('my-graph', testContext); + expect(graph.enabled).toBe(true); + expect(graph.rootNode().getKey()).toBe('root'); + expect( + graph + .getChildNodes('root') + .map((n) => n.getKey()) + .sort(), + ).toEqual(['child-a', 'child-b']); + expect( + graph + .terminalNodes() + .map((n) => n.getKey()) + .sort(), + ).toEqual(['child-b', 'leaf']); +}); + +it('tracks usage event when agentGraph is called', async () => { + const client = makeClient(); + mockVariation.mockResolvedValue({ root: '' }); + await client.agentGraph('my-graph', testContext); + expect(mockTrack).toHaveBeenCalledWith('$ld:ai:usage:agent-graph', testContext, 'my-graph', 1); +}); + +// --------------------------------------------------------------------------- +// createGraphTracker +// --------------------------------------------------------------------------- + +it('createGraphTracker reconstructs a tracker from a resumption token', () => { + const client = makeClient(); + const token = Buffer.from( + '{"runId":"run-1","graphKey":"g-key","variationKey":"v99","version":7}', + ).toString('base64url'); + + const tracker = client.createGraphTracker(token, testContext); + + expect(tracker.getTrackData().graphKey).toBe('g-key'); + expect(tracker.getTrackData().version).toBe(7); + expect(tracker.getTrackData().variationKey).toBe('v99'); + expect(tracker.getTrackData().runId).toBe('run-1'); +}); diff --git a/packages/sdk/server-ai/examples/agent-graph-traversal/README.md b/packages/sdk/server-ai/examples/agent-graph-traversal/README.md new file mode 100644 index 0000000000..2281901f1b --- /dev/null +++ b/packages/sdk/server-ai/examples/agent-graph-traversal/README.md @@ -0,0 +1,106 @@ +# Agent Graph Traversal Example + +Demonstrates how to fetch an agent graph from LaunchDarkly and wire it into +an AI framework using forward or reverse traversal. + +## Setup + +```bash +export LAUNCHDARKLY_SDK_KEY= +export LAUNCHDARKLY_GRAPH_KEY=sample-graph # optional, this is the default +yarn start +``` + +## What it does + +1. Fetches the graph flag and validates that it is enabled. +2. Runs a **forward traversal** (root → terminals), simulating how you would + build agents in a framework that constructs parents before children. +3. Runs a **reverse traversal** (terminals → root), simulating how you would + build agents in a framework that constructs children before parents. +4. Creates a tracker and records a successful invocation. + +## Choosing a traversal direction + +Both methods visit every node exactly once and pass an `executionContext` map +to each callback. The return value of your callback is stored under the node's +key, making it available to all subsequent nodes in that traversal. + +### Forward traversal (`graph.traverse`) + +Processes nodes from the root down to the terminals (BFS order). Use this when +your framework requires a **parent to be defined first** so that child agents +can be registered as handoff targets on it afterward. + +``` +orchestrator-agent → specialist-agent-a → summarizer-agent + ↘ specialist-agent-b ↗ +``` + +When `specialist-agent-a` runs, `orchestrator-agent` is already in +`executionContext`. When `summarizer-agent` runs, both specialists are there. + +Typical frameworks: **OpenAI Agents SDK** — you create the orchestrator agent +first and then attach child agents as handoff targets. + +### Reverse traversal (`graph.reverseTraverse`) + +Processes nodes from the terminals up to the root (upward BFS). Use this when +your framework requires **children to be defined first** so they can be +attached to their parent as tools or sub-graphs. + +``` +summarizer-agent → specialist-agent-a → orchestrator-agent + ↗ specialist-agent-b +``` + +When `specialist-agent-a` runs, `summarizer-agent` is already in +`executionContext`. When `orchestrator-agent` runs, both specialists are there. + +Typical frameworks: **LangGraph** — you define leaf nodes first, then compose +them into parent nodes by attaching them as edges in the graph. + +### Cyclic graphs + +Both traversal methods are cycle-safe via a visited set. For `reverseTraverse`, +a graph with no terminal nodes (every node has at least one outgoing edge) +produces no iterations — there is no starting point for upward BFS. +`traverse` handles cycles normally; the cycle back-edge is simply skipped once +the target node has already been visited. + +## Tracking + +### Graph-level tracker + +Call `graph.createTracker()` once per invocation. The tracker groups all +telemetry events (latency, tokens, success/failure) under a shared `runId` +that appears in LaunchDarkly's AI metrics. + +```typescript +const tracker = graph.createTracker(); +try { + // ... execute graph ... + tracker.trackInvocationSuccess(); +} catch { + tracker.trackInvocationFailure(); +} +``` + +If you need to record tracking events across multiple requests (e.g. streaming), +use `tracker.resumptionToken` to serialize the tracker and reconstruct it later +via `aiClient.createGraphTracker(token, context)`. + +### Node-level tracker + +Each node also carries its own `LDAIConfigTracker` for recording metrics +against the underlying agent config (tokens, latency, model usage). Access it +inside your traversal callback via `node.getConfig().createTracker?.()`. + +```typescript +graph.traverse((node, executionContext) => { + const nodeTracker = node.getConfig().createTracker?.(); + // ... invoke the node's agent ... + nodeTracker?.trackSuccess({ totalTokens: 120, inputTokens: 80, outputTokens: 40 }); + return result; +}); +``` diff --git a/packages/sdk/server-ai/examples/agent-graph-traversal/package.json b/packages/sdk/server-ai/examples/agent-graph-traversal/package.json new file mode 100644 index 0000000000..7a3fddc707 --- /dev/null +++ b/packages/sdk/server-ai/examples/agent-graph-traversal/package.json @@ -0,0 +1,19 @@ +{ + "name": "@launchdarkly/server-sdk-ai-agent-graph-traversal", + "private": true, + "version": "1.0.0", + "description": "Example demonstrating LaunchDarkly AI SDK agent graph traversal", + "type": "module", + "scripts": { + "build": "tsc", + "start": "yarn build && node ./dist/index.js" + }, + "dependencies": { + "@launchdarkly/node-server-sdk": "9.10.11", + "@launchdarkly/server-sdk-ai": "0.16.8" + }, + "devDependencies": { + "@tsconfig/node20": "20.1.4", + "typescript": "^5.5.3" + } +} diff --git a/packages/sdk/server-ai/examples/agent-graph-traversal/src/index.ts b/packages/sdk/server-ai/examples/agent-graph-traversal/src/index.ts new file mode 100644 index 0000000000..6bf1d1cf42 --- /dev/null +++ b/packages/sdk/server-ai/examples/agent-graph-traversal/src/index.ts @@ -0,0 +1,134 @@ +/* eslint-disable no-console */ +import { init, type LDContext } from '@launchdarkly/node-server-sdk'; +import { initAi } from '@launchdarkly/server-sdk-ai'; +import type { AgentGraphNode } from '@launchdarkly/server-sdk-ai'; + +const GRAPH_KEY = process.env.LAUNCHDARKLY_GRAPH_KEY || 'sample-graph'; + +const sdkKey = process.env.LAUNCHDARKLY_SDK_KEY; +if (!sdkKey) { + console.error('*** Please set the LAUNCHDARKLY_SDK_KEY env first'); + process.exit(1); +} + +const ldClient = init(sdkKey); + +const context: LDContext = { + kind: 'user', + key: 'example-user-key', + name: 'Sandy', +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Build a provider-specific agent for this node. +// In a real implementation you would use node.getConfig() to read the agent's +// instructions/model and wire them into your framework (e.g. OpenAI Agents SDK, +// LangGraph, CrewAI). +function buildAgent(node: AgentGraphNode): string { + return ``; +} + +// --------------------------------------------------------------------------- +// Forward traversal — use when your framework builds parents before children. +// +// Each node receives the agents built by its ancestors via executionContext, +// so a parent can be passed to its children as a handoff target. +// +// Example frameworks: OpenAI Agents SDK (register tools/handoffs on the +// parent, then attach child agents). +// --------------------------------------------------------------------------- +function forwardTraversalExample(graph: ReturnType): void { + console.log('\n--- Forward traversal (root → terminals) ---'); + + graph.traverse((node: AgentGraphNode, executionContext: Record) => { + const agent = buildAgent(node); + + // Edges leaving this node tell you which agents this one can hand off to. + // Those child agents will be built in subsequent iterations and available + // in executionContext by the time they run. + const childKeys = node.getEdges().map((e) => e.key); + const ready = childKeys.filter((k) => executionContext[k]); + console.log( + ` built ${agent} children: [${childKeys.join(', ') || 'none'}] pre-built: [${ready.join(', ') || 'none'}]`, + ); + + // Store the built agent so descendants can reference it. + return agent; + }); +} + +// --------------------------------------------------------------------------- +// Reverse traversal — use when your framework builds children before parents. +// +// Each node receives already-built descendant agents via executionContext, +// so a child can be attached to its parent as a tool or sub-agent. +// +// Example frameworks: LangGraph (define leaf nodes first, then compose them +// into parent nodes as edges in the graph). +// --------------------------------------------------------------------------- +function reverseTraversalExample(graph: ReturnType): void { + console.log('\n--- Reverse traversal (terminals → root) ---'); + + graph.reverseTraverse((node: AgentGraphNode, executionContext: Record) => { + const agent = buildAgent(node); + + // Children of this node are guaranteed to already be in executionContext. + const childKeys = node.getEdges().map((e) => e.key); + const builtChildren = childKeys.map((k) => executionContext[k]).filter(Boolean); + console.log(` built ${agent} attaching children: [${builtChildren.join(', ') || 'none'}]`); + + return agent; + }); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main() { + try { + await ldClient.waitForInitialization({ timeout: 10 }); + console.log('*** SDK successfully initialized'); + } catch (error) { + console.log(`*** SDK failed to initialize: ${error}`); + process.exit(1); + } + + const aiClient = initAi(ldClient); + + const graph = await aiClient.agentGraph(GRAPH_KEY, context); + + if (!graph.enabled) { + console.log(`\n*** Graph "${GRAPH_KEY}" is not enabled or could not be fetched.`); + process.exit(0); + } + + console.log(`\n=== Graph: ${GRAPH_KEY} ===`); + console.log(`Root : ${graph.rootNode().getKey()}`); + console.log( + `Terminals: ${ + graph + .terminalNodes() + .map((n) => n.getKey()) + .join(', ') || '(none — cyclic graph)' + }`, + ); + + forwardTraversalExample(graph); + reverseTraversalExample(graph); + + // Create a tracker to record this graph invocation in LaunchDarkly. + // Call trackInvocationSuccess() or trackInvocationFailure() when done. + const tracker = graph.createTracker(); + tracker.trackInvocationSuccess(); + + await ldClient.close(); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/packages/sdk/server-ai/examples/agent-graph-traversal/tsconfig.json b/packages/sdk/server-ai/examples/agent-graph-traversal/tsconfig.json new file mode 100644 index 0000000000..6916599c7d --- /dev/null +++ b/packages/sdk/server-ai/examples/agent-graph-traversal/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "node", + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "sourceMap": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 65eb87a1a9..9bf9c2ffc3 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -21,11 +21,13 @@ import { LDMessage, } from './api/config'; import { LDAIConfigFlagValue, LDAIConfigUtils } from './api/config/LDAIConfigUtils'; +import { AgentGraphDefinition, LDAgentGraphFlagValue, LDGraphTracker } from './api/graph'; import { Judge } from './api/judge/Judge'; import { LDAIClient } from './api/LDAIClient'; import { AIProviderFactory, SupportedAIProvider } from './api/providers'; import { LDAIConfigTrackerImpl } from './LDAIConfigTrackerImpl'; import { LDClientMin } from './LDClientMin'; +import { LDGraphTrackerImpl } from './LDGraphTrackerImpl'; import { aiSdkLanguage, aiSdkName, aiSdkVersion } from './sdkInfo'; /** @@ -38,6 +40,7 @@ const TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config'; const TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge'; const TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config'; const TRACK_USAGE_AGENT_CONFIGS = '$ld:ai:usage:agent-configs'; +const TRACK_USAGE_AGENT_GRAPH = '$ld:ai:usage:agent-graph'; const INIT_TRACK_CONTEXT: LDContext = { kind: 'ld_ai', @@ -393,4 +396,119 @@ export class LDAIClientImpl implements LDAIClient { createTracker(token: string, context: LDContext): LDAIConfigTracker { return LDAIConfigTrackerImpl.fromResumptionToken(token, this._ldClient, context); } + + async agentGraph( + graphKey: string, + context: LDContext, + variables?: Record, + ): Promise { + this._ldClient.track(TRACK_USAGE_AGENT_GRAPH, context, graphKey, 1); + + const defaultGraphValue: LDAgentGraphFlagValue = { root: '' }; + const graphFlagValue = (await this._ldClient.variation( + graphKey, + context, + defaultGraphValue, + )) as LDAgentGraphFlagValue; + + // eslint-disable-next-line no-underscore-dangle + const variationKey = graphFlagValue._ldMeta?.variationKey; + // eslint-disable-next-line no-underscore-dangle + const version = graphFlagValue._ldMeta?.version ?? 1; + const ldClient = this._ldClient; + const trackerFactory = () => + new LDGraphTrackerImpl(ldClient, randomUUID(), graphKey, variationKey, version, context); + + const disabled = new AgentGraphDefinition(graphFlagValue, {}, false, trackerFactory); + + // eslint-disable-next-line no-underscore-dangle + if (graphFlagValue._ldMeta?.enabled === false) { + this._logger?.debug(`agentGraph: graph "${graphKey}" is disabled.`); + return disabled; + } + + if (!graphFlagValue.root) { + this._logger?.debug(`agentGraph: graph "${graphKey}" is not fetchable or has no root node.`); + return disabled; + } + + const allKeys = AgentGraphDefinition.collectAllKeys(graphFlagValue); + const reachableKeys = this._collectReachableKeys(graphFlagValue); + + const unreachableKey = [...allKeys].find((key) => !reachableKeys.has(key)); + if (unreachableKey) { + this._logger?.debug( + `agentGraph: graph "${graphKey}" has unconnected node "${unreachableKey}" that is not reachable from the root.`, + ); + return disabled; + } + + const agentConfigs: Record = {}; + const fetchResults = await Promise.all( + [...allKeys].map(async (key) => { + const config = await this._agentConfigInternal(key, context, graphKey, variables); + return { key, config }; + }), + ); + + const disabledResult = fetchResults.find(({ config }) => !config.enabled); + if (disabledResult) { + this._logger?.debug( + `agentGraph: agent config "${disabledResult.key}" in graph "${graphKey}" is not enabled or could not be fetched.`, + ); + return disabled; + } + fetchResults.forEach(({ key, config }) => { + agentConfigs[key] = config; + }); + + const nodes = AgentGraphDefinition.buildNodes(graphFlagValue, agentConfigs); + return new AgentGraphDefinition(graphFlagValue, nodes, true, trackerFactory); + } + + createGraphTracker(token: string, context: LDContext): LDGraphTracker { + return LDGraphTrackerImpl.fromResumptionToken(token, this._ldClient, context); + } + + /** + * Fetches a single agent config without tracking usage (used internally by agentGraph). + */ + private async _agentConfigInternal( + key: string, + context: LDContext, + graphKey?: string, + variables?: Record, + ): Promise { + const config = await this._evaluate( + key, + context, + disabledAIConfig, + 'agent', + variables, + graphKey, + ); + return config as LDAIAgentConfig; + } + + /** + * Returns the set of all node keys reachable from the root via BFS. + */ + private _collectReachableKeys(graph: LDAgentGraphFlagValue): Set { + const visited = new Set(); + const queue: string[] = [graph.root]; + visited.add(graph.root); + + while (queue.length > 0) { + const key = queue.shift()!; + const edges = graph.edges?.[key] ?? []; + edges.forEach((edge) => { + if (!visited.has(edge.key)) { + visited.add(edge.key); + queue.push(edge.key); + } + }); + } + + return visited; + } } diff --git a/packages/sdk/server-ai/src/LDGraphTrackerImpl.ts b/packages/sdk/server-ai/src/LDGraphTrackerImpl.ts index d1f0602f50..6accab1959 100644 --- a/packages/sdk/server-ai/src/LDGraphTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDGraphTrackerImpl.ts @@ -1,65 +1,129 @@ -import { LDContext } from '@launchdarkly/js-server-sdk-common'; +import type { LDContext } from '@launchdarkly/js-server-sdk-common'; -import { LDGraphMetricSummary, LDGraphTracker } from './api/graph/LDGraphTracker'; -import { LDJudgeResult } from './api/judge/types'; -import { LDTokenUsage } from './api/metrics'; -import { LDClientMin } from './LDClientMin'; +import type { LDGraphTracker } from './api/graph/LDGraphTracker'; +import type { LDGraphMetricSummary, LDGraphTrackData } from './api/graph/types'; +import type { LDJudgeResult } from './api/judge/types'; +import type { LDTokenUsage } from './api/metrics'; +import type { LDClientMin } from './LDClientMin'; +/** + * Concrete implementation of {@link LDGraphTracker}. + * + * Construct directly or reconstruct from a resumption token via + * {@link LDGraphTrackerImpl.fromResumptionToken}. + */ export class LDGraphTrackerImpl implements LDGraphTracker { - private _trackedMetrics: LDGraphMetricSummary = {}; + private _summary: LDGraphMetricSummary = {}; constructor( - private _ldClient: LDClientMin, - private _graphKey: string, - private _variationKey: string, - private _version: number, - private _context: LDContext, + private readonly _ldClient: LDClientMin, + private readonly _runId: string, + private readonly _graphKey: string, + private readonly _variationKey: string | undefined, + private readonly _version: number, + private readonly _context: LDContext, ) {} - getTrackData(): { - variationKey: string; - graphKey: string; - version: number; - } { - return { - variationKey: this._variationKey, + /** + * Reconstructs an {@link LDGraphTrackerImpl} from a resumption token, preserving + * the original `runId` so all events continue to be correlated under the same run. + * + * **Security note:** The token contains the flag variation key and version. + * Do not pass the raw token to untrusted clients. + * + * @param token URL-safe Base64-encoded token produced by {@link LDGraphTrackerImpl.resumptionToken}. + * @param ldClient LaunchDarkly client instance. + * @param context LDContext for the new tracker. + */ + static fromResumptionToken( + token: string, + ldClient: LDClientMin, + context: LDContext, + ): LDGraphTrackerImpl { + const json = Buffer.from(token, 'base64url').toString('utf8'); + const data = JSON.parse(json) as LDGraphTrackData; + return new LDGraphTrackerImpl( + ldClient, + data.runId, + data.graphKey, + data.variationKey, + data.version, + context, + ); + } + + getTrackData(): LDGraphTrackData { + const data: LDGraphTrackData = { + runId: this._runId, graphKey: this._graphKey, version: this._version, }; + if (this._variationKey !== undefined) { + data.variationKey = this._variationKey; + } + return data; + } + + getSummary(): LDGraphMetricSummary { + return { ...this._summary }; + } + + get resumptionToken(): string { + // Keys must appear in exact spec-defined order: + // runId, graphKey, variationKey (omitted if absent), version + const parts: string[] = [ + `"runId":${JSON.stringify(this._runId)}`, + `"graphKey":${JSON.stringify(this._graphKey)}`, + ]; + if (this._variationKey !== undefined) { + parts.push(`"variationKey":${JSON.stringify(this._variationKey)}`); + } + parts.push(`"version":${this._version}`); + const json = `{${parts.join(',')}}`; + return Buffer.from(json).toString('base64url'); } trackInvocationSuccess(): void { - if (this._trackedMetrics.success !== undefined) { + if (this._summary.success !== undefined) { + this._ldClient.logger?.warn( + 'LDGraphTracker: invocation success/failure already recorded for this run — dropping duplicate call.', + ); return; } - this._trackedMetrics.success = true; + this._summary.success = true; this._ldClient.track('$ld:ai:graph:invocation_success', this._context, this.getTrackData(), 1); } trackInvocationFailure(): void { - if (this._trackedMetrics.success !== undefined) { + if (this._summary.success !== undefined) { + this._ldClient.logger?.warn( + 'LDGraphTracker: invocation success/failure already recorded for this run — dropping duplicate call.', + ); return; } - this._trackedMetrics.success = false; + this._summary.success = false; this._ldClient.track('$ld:ai:graph:invocation_failure', this._context, this.getTrackData(), 1); } trackLatency(durationMs: number): void { - if (this._trackedMetrics.durationMs !== undefined) { + if (this._summary.durationMs !== undefined) { + this._ldClient.logger?.warn( + 'LDGraphTracker: trackLatency already called for this run — dropping duplicate call.', + ); return; } - this._trackedMetrics.durationMs = durationMs; + this._summary.durationMs = durationMs; this._ldClient.track('$ld:ai:graph:latency', this._context, this.getTrackData(), durationMs); } trackTotalTokens(tokens: LDTokenUsage): void { - if (this._trackedMetrics.tokens !== undefined) { - return; - } - if (tokens.total <= 0) { + if (this._summary.tokens !== undefined) { + this._ldClient.logger?.warn( + 'LDGraphTracker: trackTotalTokens already called for this run — dropping duplicate call.', + ); return; } - this._trackedMetrics.tokens = tokens; + this._summary.tokens = { ...tokens }; this._ldClient.track( '$ld:ai:graph:total_tokens', this._context, @@ -69,10 +133,13 @@ export class LDGraphTrackerImpl implements LDGraphTracker { } trackPath(path: string[]): void { - if (this._trackedMetrics.path !== undefined) { + if (this._summary.path !== undefined) { + this._ldClient.logger?.warn( + 'LDGraphTracker: trackPath already called for this run — dropping duplicate call.', + ); return; } - this._trackedMetrics.path = path; + this._summary.path = [...path]; this._ldClient.track('$ld:ai:graph:path', this._context, { ...this.getTrackData(), path }, 1); } @@ -115,8 +182,4 @@ export class LDGraphTrackerImpl implements LDGraphTracker { 1, ); } - - getSummary(): LDGraphMetricSummary { - return { ...this._trackedMetrics }; - } } diff --git a/packages/sdk/server-ai/src/api/LDAIClient.ts b/packages/sdk/server-ai/src/api/LDAIClient.ts index fd93ca92a5..5dfec98072 100644 --- a/packages/sdk/server-ai/src/api/LDAIClient.ts +++ b/packages/sdk/server-ai/src/api/LDAIClient.ts @@ -11,6 +11,7 @@ import { LDAIJudgeConfig, LDAIJudgeConfigDefault, } from './config'; +import { AgentGraphDefinition, LDGraphTracker } from './graph'; import { Judge } from './judge/Judge'; import { SupportedAIProvider } from './providers'; @@ -337,4 +338,55 @@ export interface LDAIClient { * @returns A reconstructed AIConfigTracker with the original runId preserved. */ createTracker(token: string, context: LDContext): LDAIConfigTracker; + + /** + * Fetches an agent graph configuration from LaunchDarkly and returns an + * {@link AgentGraphDefinition}. + * + * When the graph is enabled the method validates that: + * - The graph flag can be evaluated. + * - A single root node is present. + * - All nodes in the graph are reachable from the root (no disconnected nodes). + * - Every referenced agent config can be fetched and is enabled. + * + * If any validation check fails, the returned definition has + * {@link AgentGraphDefinition.enabled | enabled} set to `false` with an empty + * node collection. When the logger level is DEBUG, a message describing the + * failure is emitted. + * + * @param graphKey The LaunchDarkly flag key for the agent graph configuration. + * @param context The LaunchDarkly context used for flag evaluation and tracking. + * @param variables Optional key-value pairs used for Mustache template interpolation + * in each node's agent config instructions. Applied uniformly to all nodes. + * + * @returns A promise that resolves to an {@link AgentGraphDefinition}. Check + * {@link AgentGraphDefinition.enabled | enabled} before traversing. + * + * @example + * ```typescript + * const graph = await aiClient.agentGraph('my-agent-graph', context, { userName: 'Sandy' }); + * if (graph.enabled) { + * graph.traverse((node, ctx) => { + * // build your provider-specific node here + * }); + * } + * ``` + */ + agentGraph( + graphKey: string, + context: LDContext, + variables?: Record, + ): Promise; + + /** + * Reconstructs an {@link LDGraphTracker} from a resumption token, preserving + * the original `runId` so events from a resumed session are correlated correctly. + * + * **Security note:** The token encodes the flag variation key and version. + * Keep it server-side; do not expose it to untrusted clients. + * + * @param token URL-safe Base64-encoded token from {@link LDGraphTracker.resumptionToken}. + * @param context LDContext to associate with the reconstructed tracker. + */ + createGraphTracker(token: string, context: LDContext): LDGraphTracker; } diff --git a/packages/sdk/server-ai/src/api/graph/AgentGraphDefinition.ts b/packages/sdk/server-ai/src/api/graph/AgentGraphDefinition.ts new file mode 100644 index 0000000000..c5113b53a5 --- /dev/null +++ b/packages/sdk/server-ai/src/api/graph/AgentGraphDefinition.ts @@ -0,0 +1,253 @@ +import type { LDAIAgentConfig } from '../config'; +import { AgentGraphNode } from './AgentGraphNode'; +import type { LDGraphTracker } from './LDGraphTracker'; +import type { LDAgentGraphFlagValue, LDGraphEdge } from './types'; + +/** + * Callback function signature for graph traversal methods. + */ +export type TraversalFn = ( + node: AgentGraphNode, + executionContext: Record, +) => unknown; + +/** + * Encapsulates an agent graph configuration and its pre-built node collection. + * + * Provides graph-level orchestration including relationship queries (parent/child), + * breadth-first traversal in both forward and reverse directions, and graph tracker creation. + * + * Obtain an instance via {@link LDAIClient.agentGraph}. When the graph is disabled + * or invalid, the returned instance has {@link enabled} set to `false` and an + * empty node collection. + */ +export class AgentGraphDefinition { + constructor( + private readonly _agentGraph: LDAgentGraphFlagValue, + private readonly _nodes: Record, + readonly enabled: boolean, + private readonly _createTracker: () => LDGraphTracker, + ) {} + + /** + * Builds a node map from a raw agent graph flag value and a map of pre-fetched agent configs. + * + * @param graph Raw graph flag value from LaunchDarkly. + * @param agentConfigs Map of agent config key to resolved LDAIAgentConfig. + * @returns Record mapping agent config keys to AgentGraphNode instances. + */ + static buildNodes( + graph: LDAgentGraphFlagValue, + agentConfigs: Record, + ): Record { + const nodes: Record = {}; + const allKeys = AgentGraphDefinition.collectAllKeys(graph); + + allKeys.forEach((key) => { + const config = agentConfigs[key]; + if (!config) { + return; + } + const outgoingEdges: LDGraphEdge[] = graph.edges?.[key] ?? []; + nodes[key] = new AgentGraphNode(key, config, outgoingEdges); + }); + + return nodes; + } + + /** + * Returns the children of the node identified by `nodeKey`. + * + * @param nodeKey The agent config key of the parent node. + */ + getChildNodes(nodeKey: string): AgentGraphNode[] { + const node = this._nodes[nodeKey]; + if (!node) { + return []; + } + return node + .getEdges() + .map((edge) => this._nodes[edge.key]) + .filter((n): n is AgentGraphNode => n !== undefined); + } + + /** + * Returns all nodes that have a direct edge to the node identified by `nodeKey`. + * + * @param nodeKey The agent config key of the child node. + */ + getParentNodes(nodeKey: string): AgentGraphNode[] { + return Object.values(this._nodes).filter((node) => + node.getEdges().some((edge) => edge.key === nodeKey), + ); + } + + /** + * Returns all terminal nodes (nodes with no outgoing edges). + */ + terminalNodes(): AgentGraphNode[] { + return Object.values(this._nodes).filter((node) => node.isTerminal()); + } + + /** + * Returns the root node of the graph. + */ + rootNode(): AgentGraphNode { + return this._nodes[this._agentGraph.root]; + } + + /** + * Returns the node with the given key, or `null` if not found. + * + * @param nodeKey The agent config key to look up. + */ + getNode(nodeKey: string): AgentGraphNode | null { + return this._nodes[nodeKey] ?? null; + } + + /** + * Returns the underlying raw graph configuration from LaunchDarkly. + */ + getConfig(): LDAgentGraphFlagValue { + return this._agentGraph; + } + + /** + * Returns a new {@link LDGraphTracker} for this graph invocation. + * + * Call this once per invocation. Each call produces a tracker with a fresh `runId` + * that groups all events for that invocation. + */ + createTracker(): LDGraphTracker { + return this._createTracker(); + } + + /** + * Traverses the graph breadth-first from the root to all terminal nodes. + * + * Nodes at the same depth are processed before advancing to the next depth. + * The value returned by `fn` is stored in the mutable `executionContext` under + * the node's key, making upstream results available to downstream nodes. + * + * Cyclic graphs are handled safely — each node is visited at most once. + * + * @param fn Callback invoked for each node. Its return value is added to + * `executionContext` keyed by the node's config key. + * @param initialExecutionContext Optional initial context to seed the traversal. + */ + traverse(fn: TraversalFn, initialExecutionContext: Record = {}): void { + const root = this.rootNode(); + if (!root) { + return; + } + + const executionContext = { ...initialExecutionContext }; + const visited = new Set(); + const queue: AgentGraphNode[] = [root]; + visited.add(root.getKey()); + + while (queue.length > 0) { + const node = queue.shift()!; + const result = fn(node, executionContext); + executionContext[node.getKey()] = result; + + node.getEdges().forEach((edge) => { + if (!visited.has(edge.key)) { + const child = this._nodes[edge.key]; + if (child) { + visited.add(edge.key); + queue.push(child); + } + } + }); + } + } + + /** + * Traverses the graph from terminal nodes up to the root. + * + * Uses BFS upward via parent edges so that each node is processed only after + * all of its reachable descendants have been processed. The root is always + * visited last. Cyclic graphs are handled safely — each node is visited at + * most once; if the graph has no terminal nodes, this method returns without + * invoking `fn`. + * + * **Ordering note:** Within a single BFS level (nodes at the same depth from a + * terminal) the visit order is not strictly guaranteed. The guarantee is only + * that a node is visited before any of its ancestors — not that siblings at the + * same depth are visited in a specific order relative to each other. + * + * The value returned by `fn` is stored in the mutable `executionContext` under + * the node's key. + * + * @param fn Callback invoked for each node. Its return value is added to + * `executionContext` keyed by the node's config key. + * @param initialExecutionContext Optional initial context to seed the traversal. + */ + reverseTraverse(fn: TraversalFn, initialExecutionContext: Record = {}): void { + const terminals = this.terminalNodes(); + if (terminals.length === 0) { + return; + } + + const executionContext = { ...initialExecutionContext }; + const rootKey = this._agentGraph.root; + const visited = new Set(); + let queue: AgentGraphNode[] = terminals; + + while (queue.length > 0) { + const nextQueue: AgentGraphNode[] = []; + + queue.forEach((node) => { + const key = node.getKey(); + if (visited.has(key)) { + return; + } + visited.add(key); + + // Defer the root so it is always processed last + if (key === rootKey) { + return; + } + + const result = fn(node, executionContext); + executionContext[key] = result; + + this.getParentNodes(key).forEach((parent) => { + if (!visited.has(parent.getKey())) { + nextQueue.push(parent); + } + }); + }); + + queue = nextQueue; + } + + // Root is always last — only invoke if it was reached during traversal + const root = this._nodes[rootKey]; + if (root && visited.has(rootKey)) { + const result = fn(root, executionContext); + executionContext[rootKey] = result; + } + } + + /** + * Collects every unique node key referenced in the graph (root + all edge sources + * and targets). + */ + static collectAllKeys(graph: LDAgentGraphFlagValue): Set { + const keys = new Set(); + keys.add(graph.root); + + if (graph.edges) { + Object.entries(graph.edges).forEach(([sourceKey, edges]) => { + keys.add(sourceKey); + edges.forEach((edge) => { + keys.add(edge.key); + }); + }); + } + + return keys; + } +} diff --git a/packages/sdk/server-ai/src/api/graph/AgentGraphNode.ts b/packages/sdk/server-ai/src/api/graph/AgentGraphNode.ts new file mode 100644 index 0000000000..598bfbf0c1 --- /dev/null +++ b/packages/sdk/server-ai/src/api/graph/AgentGraphNode.ts @@ -0,0 +1,46 @@ +import type { LDAIAgentConfig } from '../config'; +import type { LDGraphEdge } from './types'; + +/** + * Represents a single node within an agent graph. + * + * Each node wraps an {@link LDAIAgentConfig} and carries the outgoing edges + * to its children. Use the node's tracker (via `getConfig().tracker`) to record + * node-level metrics against the underlying agent config. + */ +export class AgentGraphNode { + constructor( + private readonly _key: string, + private readonly _config: LDAIAgentConfig, + private readonly _edges: LDGraphEdge[], + ) {} + + /** + * Returns the agent config key that identifies this node in the graph. + */ + getKey(): string { + return this._key; + } + + /** + * Returns the underlying AIAgentConfig for this node. + * Use `getConfig().tracker` to record node-level metrics. + */ + getConfig(): LDAIAgentConfig { + return this._config; + } + + /** + * Returns the outgoing edges from this node to its children. + */ + getEdges(): LDGraphEdge[] { + return this._edges; + } + + /** + * Returns `true` if this node has no outgoing edges (i.e., it is a terminal/leaf node). + */ + isTerminal(): boolean { + return this._edges.length === 0; + } +} diff --git a/packages/sdk/server-ai/src/api/graph/LDGraphTracker.ts b/packages/sdk/server-ai/src/api/graph/LDGraphTracker.ts index 9ce432d1db..25afc9b2ce 100644 --- a/packages/sdk/server-ai/src/api/graph/LDGraphTracker.ts +++ b/packages/sdk/server-ai/src/api/graph/LDGraphTracker.ts @@ -1,110 +1,120 @@ -import { LDJudgeResult } from '../judge/types'; -import { LDTokenUsage } from '../metrics'; +import type { LDJudgeResult } from '../judge/types'; +import type { LDTokenUsage } from '../metrics'; +import type { LDGraphMetricSummary, LDGraphTrackData } from './types'; /** - * Metrics tracked at the graph level. + * Tracks graph-level and edge-level metrics for an agent graph invocation. + * + * Graph-level methods enforce at-most-once semantics: calling the same method + * twice on a tracker instance drops the second call and emits a warning. + * Edge-level methods (trackRedirect, trackHandoffSuccess, trackHandoffFailure) + * are multi-fire and are not subject to this constraint. + * + * @example + * ```typescript + * const tracker = graphDefinition.createTracker(); + * try { + * // ... execute graph ... + * tracker.trackInvocationSuccess(); + * tracker.trackLatency(durationMs); + * } catch { + * tracker.trackInvocationFailure(); + * } + * ``` */ -export interface LDGraphMetricSummary { +export interface LDGraphTracker { /** - * True if the graph invocation succeeded, false if it failed, absent if not tracked. + * Returns tracking metadata to be included in every LDClient.track call. */ - success?: boolean; + getTrackData(): LDGraphTrackData; /** - * Total graph execution duration in milliseconds, if tracked. + * Returns a snapshot of all graph-level metrics tracked so far. */ - durationMs?: number; + getSummary(): LDGraphMetricSummary; /** - * Aggregated token usage across the entire graph invocation, if tracked. + * A URL-safe Base64-encoded (RFC 4648, no padding) token encoding the tracker's + * identity. Pass this token to {@link LDGraphTrackerImpl.fromResumptionToken} to + * reconstruct the tracker across process boundaries, preserving the original runId. + * + * **Security note:** The token contains the flag variation key and version. If passed + * to an untrusted client (e.g., a browser) this could expose feature-flag targeting + * details. Keep the token server-side and use an opaque reference in client-facing APIs. */ - tokens?: LDTokenUsage; + readonly resumptionToken: string; - /** - * Execution path through the graph as an array of config keys, if tracked. - */ - path?: string[]; -} + // ------------------------------------------------------------------------- + // Graph-level tracking methods (at-most-once per tracker instance) + // ------------------------------------------------------------------------- -/** - * Tracker for graph-level and edge-level metrics in AI agent graph operations. - * - * Node-level metrics are tracked via each node's {@link LDAIConfigTracker}. - */ -export interface LDGraphTracker { /** - * Get the data for tracking. - */ - getTrackData(): { - variationKey: string; - graphKey: string; - version: number; - }; - - /** - * Track a successful graph invocation. - * - * At-most-once per tracker instance. Subsequent calls are dropped. + * Tracks a successful graph invocation. + * Emits event `$ld:ai:graph:invocation_success` with metric value `1`. + * At-most-once: subsequent calls are dropped with a warning. */ trackInvocationSuccess(): void; /** - * Track an unsuccessful graph invocation. - * - * At-most-once per tracker instance. Subsequent calls are dropped. + * Tracks an unsuccessful graph invocation. + * Emits event `$ld:ai:graph:invocation_failure` with metric value `1`. + * At-most-once: subsequent calls are dropped with a warning. */ trackInvocationFailure(): void; /** - * Track the total latency of graph execution. - * - * At-most-once per tracker instance. Subsequent calls are dropped. + * Tracks the total latency of the graph execution in milliseconds. + * Emits event `$ld:ai:graph:latency` with the duration as the metric value. + * At-most-once: subsequent calls are dropped with a warning. * * @param durationMs Duration in milliseconds. */ trackLatency(durationMs: number): void; /** - * Track aggregated token usage across the entire graph invocation. - * - * At-most-once per tracker instance. Subsequent calls are dropped. + * Tracks aggregate token usage across the entire graph invocation. + * Emits event `$ld:ai:graph:total_tokens` with the total token count as the metric value. + * At-most-once: subsequent calls are dropped with a warning. * * @param tokens Token usage information. */ trackTotalTokens(tokens: LDTokenUsage): void; /** - * Track the execution path through the graph. + * Tracks the execution path through the graph. + * Emits event `$ld:ai:graph:path` with metric value `1`. + * The data payload includes the path array in addition to standard track data. + * At-most-once: subsequent calls are dropped with a warning. * - * At-most-once per tracker instance. Subsequent calls are dropped. - * - * @param path Array of config keys representing the sequence of nodes executed. + * @param path An ordered array of agent config keys representing the execution path. */ trackPath(path: string[]): void; /** - * Track a judge evaluation result for the final graph output. - * - * No event is emitted when the result was not sampled (result.sampled is false). + * Tracks a judge evaluation result for the final graph output. + * Emits one LDClient.track call when the result was sampled and successful. + * Not subject to at-most-once constraints. * * @param result Judge result containing score, reasoning, and metadata. */ trackJudgeResult(result: LDJudgeResult): void; + // ------------------------------------------------------------------------- + // Edge-level tracking methods (multi-fire, not at-most-once) + // ------------------------------------------------------------------------- + /** - * Track when a node redirects to a different target than originally specified. - * - * May be called multiple times. + * Tracks when a node redirects to a different target than originally specified. + * Emits event `$ld:ai:graph:redirect` with metric value `1`. * * @param sourceKey Config key of the source node. - * @param redirectedTarget Config key of the target node that was redirected to. + * @param redirectedTarget Config key of the actual target node. */ trackRedirect(sourceKey: string, redirectedTarget: string): void; /** - * Track a successful handoff between nodes. - * - * May be called multiple times. + * Tracks a successful handoff between two nodes. + * Emits event `$ld:ai:graph:handoff_success` with metric value `1`. * * @param sourceKey Config key of the source node. * @param targetKey Config key of the target node. @@ -112,17 +122,11 @@ export interface LDGraphTracker { trackHandoffSuccess(sourceKey: string, targetKey: string): void; /** - * Track a failed handoff between nodes. - * - * May be called multiple times. + * Tracks a failed handoff between two nodes. + * Emits event `$ld:ai:graph:handoff_failure` with metric value `1`. * * @param sourceKey Config key of the source node. * @param targetKey Config key of the target node. */ trackHandoffFailure(sourceKey: string, targetKey: string): void; - - /** - * Get a summary of the tracked graph-level metrics. - */ - getSummary(): LDGraphMetricSummary; } diff --git a/packages/sdk/server-ai/src/api/graph/index.ts b/packages/sdk/server-ai/src/api/graph/index.ts index 536e630115..9d899029d5 100644 --- a/packages/sdk/server-ai/src/api/graph/index.ts +++ b/packages/sdk/server-ai/src/api/graph/index.ts @@ -1 +1,4 @@ +export * from './types'; export * from './LDGraphTracker'; +export * from './AgentGraphNode'; +export * from './AgentGraphDefinition'; diff --git a/packages/sdk/server-ai/src/api/graph/types.ts b/packages/sdk/server-ai/src/api/graph/types.ts new file mode 100644 index 0000000000..1b578fecba --- /dev/null +++ b/packages/sdk/server-ai/src/api/graph/types.ts @@ -0,0 +1,88 @@ +import { LDTokenUsage } from '../metrics'; + +/** + * Represents a directed edge in an agent graph, connecting a source node to a target node. + */ +export interface LDGraphEdge { + /** + * The key of the target AIAgentConfig node. + */ + key: string; + + /** + * Optional handoff options that customize how data flows between nodes. + */ + handoff?: Record; +} + +/** + * Raw flag value for an agent graph configuration as returned by LaunchDarkly. + * This represents the data structure delivered by LaunchDarkly for graph configurations. + */ +export interface LDAgentGraphFlagValue { + _ldMeta?: { + variationKey?: string; + version?: number; + enabled?: boolean; + }; + + /** + * The key of the root AIAgentConfig in the graph. + */ + root: string; + + /** + * Object mapping source agent config keys to arrays of target edges. + */ + edges?: Record; +} + +/** + * Accumulated graph-level metrics collected by an LDGraphTracker. + */ +export interface LDGraphMetricSummary { + /** + * Whether the graph invocation succeeded. Absent if not yet tracked. + */ + success?: boolean; + + /** + * Total graph execution duration in milliseconds. Absent if not yet tracked. + */ + durationMs?: number; + + /** + * Aggregate token usage across the entire graph invocation. Absent if not yet tracked. + */ + tokens?: LDTokenUsage; + + /** + * Execution path through the graph as an array of config keys. Absent if not yet tracked. + */ + path?: string[]; +} + +/** + * Tracking metadata returned by {@link LDGraphTracker.getTrackData}. + */ +export interface LDGraphTrackData { + /** + * UUID v4 uniquely identifying this tracker and all events it emits. + */ + runId: string; + + /** + * The graph configuration key. + */ + graphKey: string; + + /** + * The variation key. Absent when a default config was used rather than a real flag evaluation. + */ + variationKey?: string; + + /** + * The version of the flag variation. + */ + version: number; +} diff --git a/release-please-config.json b/release-please-config.json index a667b4b38c..8aada61d70 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -167,6 +167,11 @@ "type": "json", "path": "/packages/sdk/server-ai/examples/chat-observability/package.json", "jsonpath": "$.dependencies['@launchdarkly/node-server-sdk']" + }, + { + "type": "json", + "path": "/packages/sdk/server-ai/examples/agent-graph-traversal/package.json", + "jsonpath": "$.dependencies['@launchdarkly/node-server-sdk']" } ] }, @@ -256,6 +261,11 @@ "type": "json", "path": "examples/chat-observability/package.json", "jsonpath": "$.dependencies['@launchdarkly/server-sdk-ai']" + }, + { + "type": "json", + "path": "examples/agent-graph-traversal/package.json", + "jsonpath": "$.dependencies['@launchdarkly/server-sdk-ai']" } ] },