feat(agent): OpenUI bindings — library/fragment builders, toUIOutput, getUiStream() (DEV-773)

packages/agent/src/index.ts

@@ -182,6 +182,41 @@ export {
   buildNextTurnParamsContext,
   executeNextTurnParamsFunctions,
 } from './lib/next-turn-params.js';
+export type {
+  ComponentDefinition,
+  CreateLibraryOptions,
+  FragmentArg,
+  FragmentBuilder,
+  FragmentNode,
+  OpenUiPlugin,
+  OpenUiWireComponent,
+  PropSignature,
+  UiDocumentEvent,
+  UiExpr,
+  UiFragment,
+  UiFragmentEvent,
+  UiLibrary,
+  UiLiteralValue,
+  UiStatementEvent,
+  UiStreamEvent,
+} from './lib/openui/index.js';
+// OpenUI (generative UI) bindings: library model, fragment builder, plugin helper
+export {
+  componentProps,
+  createLibrary,
+  defineComponent,
+  fragment,
+  OPENUI_BUILTIN_COMPONENTS,
+  OPENUI_LANG_DIALECT,
+  OPENUI_ROOT_REF,
+  OPENUI_WIRE_EVENT,
+  openui,
+  serializeExpr,
+  translateUiEvent,
+  uiBuiltin,
+  uiRef,
+  uiState,
+} from './lib/openui/index.js';
 // Stop condition helpers
 export {
   finishReasonIs,
@@ -248,8 +283,10 @@ export type {
   ToolResultEvent,
   ToolResultItem,
   ToolStreamEvent,
+  ToolUiFragmentEvent,
   ToolWithExecute,
   ToolWithGenerator,
+  ToUIOutputFunction,
   TurnContext,
   TurnEndEvent,
   TurnStartEvent,
@@ -272,6 +309,7 @@ export {
   isToolCallOutputEvent,
   isToolPreliminaryResultEvent,
   isToolResultEvent,
+  isToolUiFragmentEvent,
   isTurnEndEvent,
   isTurnStartEvent,
   ToolType,

packages/agent/src/lib/model-result.ts

@@ -31,6 +31,8 @@ import {
   applyNextTurnParamsToRequest,
   executeNextTurnParamsFunctions,
 } from './next-turn-params.js';
+import type { UiStreamEvent } from './openui/ui-stream.js';
+import { translateUiEvent } from './openui/ui-stream.js';
 import { ReusableReadableStream } from './reusable-stream.js';
 import { isStopConditionMet } from './stop-conditions.js';
 import type { ItemInProgress, StreamableOutputItem } from './stream-transformers.js';
@@ -78,6 +80,7 @@ import type {
   ToolContextMapWithShared,
   ToolResultItem,
   ToolStreamEvent,
+  ToolUiFragmentEvent,
   TurnContext,
   TurnEndEvent,
   TurnStartEvent,
@@ -2670,6 +2673,8 @@ export class ModelResult<
         output: executedOutput,
         timestamp: Date.now(),
       } satisfies ToolCallOutputEvent);
+
+      await this.broadcastUiFragment(value);
     }
 
     return {
@@ -2678,6 +2683,54 @@ export class ModelResult<
     };
   }
 
+  /**
+   * Compute and broadcast a tool-authored OpenUI fragment for a successful
+   * execution. Render-only: the fragment never reaches the model, so a
+   * throwing `toUIOutput` degrades to "no fragment" instead of failing the
+   * round — the model-facing output has already been pushed.
+   */
+  private async broadcastUiFragment(value: {
+    toolCall: ParsedToolCall<Tool>;
+    tool: Tool;
+    result: {
+      result: unknown;
+      error?: Error;
+    };
+  }): Promise<void> {
+    if (
+      value.result.error ||
+      !isAutoResolvableTool(value.tool) ||
+      !value.tool.function.toUIOutput
+    ) {
+      return;
+    }
+    const rawArgs: unknown = value.toolCall.arguments;
+    if (!isRecord(rawArgs)) {
+      return;
+    }
+    try {
+      const fragment = await value.tool.function.toUIOutput({
+        output: value.result.result,
+        input: rawArgs,
+      });
+      if (!fragment) {
+        return;
+      }
+      this.turnBroadcaster?.push({
+        type: 'tool.ui_fragment' as const,
+        toolCallId: value.toolCall.id,
+        toolName: value.toolCall.name,
+        fragment: {
+          dialect: fragment.dialect,
+          source: fragment.source,
+        },
+        timestamp: Date.now(),
+      } satisfies ToolUiFragmentEvent);
+    } catch {
+      // Fragment construction failed — drop it; rendering is best-effort.
+    }
+  }
+
   /**
    * Resolve async functions for the current turn.
    * Updates the resolved request with turn-specific parameter values.
@@ -4592,6 +4645,53 @@ export class ModelResult<
     }.call(this);
   }
 
+  /**
+   * Stream OpenUI events from all turns: completed OpenUI Lang statements
+   * authored by the model (`response.openui.*` wire events from the `openui`
+   * plugin) and tool-authored fragments (`tool.ui_fragment` events produced
+   * by tools declaring `toUIOutput`).
+   *
+   * Wire events not yet in the SDK's stream-event union arrive through its
+   * forward-compat catch-all; translation reads the raw payload, so this
+   * stream works both before and after the SDK regen picks them up.
+   */
+  getUiStream(): AsyncIterableIterator<UiStreamEvent> {
+    return async function* (this: ModelResult<TTools, TShared>) {
+      await this.initStreamGuarded();
+
+      if (!this.options.tools?.length) {
+        let streamFailed = false;
+        try {
+          if (this.reusableStream) {
+            for await (const event of this.reusableStream.createConsumer()) {
+              const uiEvent = translateUiEvent(event);
+              if (uiEvent) {
+                yield uiEvent;
+              }
+            }
+          }
+        } catch (error) {
+          streamFailed = true;
+          throw error;
+        } finally {
+          await this.finishHooksSessionForStream(streamFailed ? 'error' : 'complete');
+        }
+        return;
+      }
+
+      const { consumer, executionPromise } = this.startTurnBroadcasterExecution();
+
+      for await (const event of consumer) {
+        const uiEvent = translateUiEvent(event);
+        if (uiEvent) {
+          yield uiEvent;
+        }
+      }
+
+      await executionPromise;
+    }.call(this);
+  }
+
   /**
    * Stream tool call argument deltas and preliminary results from all turns.
    * Preliminary results are streamed in REAL-TIME as generator tools yield.

packages/agent/src/lib/openui/document.ts

@@ -0,0 +1,84 @@
+/**
+ * OpenUI Lang expression model + serialization.
+ *
+ * OpenUI Lang is a line-oriented assignment language: one statement per line,
+ * `name = expression`. The SDK only *authors* OpenUI Lang (tool-authored
+ * fragments, wire-format libraries) — parsing, validation, and prompt
+ * injection are API-side responsibilities. This module is therefore the
+ * minimal expression tree and serializer shared by the fragment builder.
+ */
+
+/** The OpenUI Lang dialect this package emits. */
+export const OPENUI_LANG_DIALECT = 'openui-lang/0.5';
+
+/** The reserved assignment ref that designates the document root. */
+export const OPENUI_ROOT_REF = 'root';
+
+export type UiLiteralValue = string | number | boolean | null;
+
+/** Expression tree for one assignment's right-hand side. */
+export type UiExpr =
+  | {
+      kind: 'literal';
+      value: UiLiteralValue;
+    }
+  | {
+      kind: 'ref';
+      name: string;
+    }
+  | {
+      kind: 'state-ref';
+      name: string;
+    }
+  | {
+      kind: 'member';
+      base: UiExpr;
+      path: string[];
+    }
+  | {
+      kind: 'array';
+      items: UiExpr[];
+    }
+  | {
+      kind: 'object';
+      entries: Array<{
+        key: string;
+        value: UiExpr;
+      }>;
+    }
+  | {
+      kind: 'call';
+      fn: string;
+      builtin: boolean;
+      args: UiExpr[];
+    };
+
+/**
+ * A renderable piece of UI: the dialect it's expressed in plus its serialized
+ * OpenUI Lang source. This is the shape carried on `tool.ui_fragment` stream
+ * events and (for server tools) `response.openui.fragment` wire events.
+ */
+export interface UiFragment {
+  dialect: string;
+  source: string;
+}
+
+/** Serialize an expression to OpenUI Lang source. */
+export function serializeExpr(expr: UiExpr): string {
+  switch (expr.kind) {
+    case 'literal':
+      return typeof expr.value === 'string' ? JSON.stringify(expr.value) : String(expr.value);
+    case 'ref':
+      return expr.name;
+    case 'state-ref':
+      return `$${expr.name}`;
+    case 'member':
+      return `${serializeExpr(expr.base)}.${expr.path.join('.')}`;
+    case 'array':
+      return `[${expr.items.map(serializeExpr).join(', ')}]`;
+    case 'object':
+      return `{${expr.entries.map((e) => `${e.key}: ${serializeExpr(e.value)}`).join(', ')}}`;
+    case 'call':
+      return `${expr.builtin ? '@' : ''}${expr.fn}(${expr.args.map(serializeExpr).join(', ')})`;
+  }
+}