feat(agent): OpenUI bindings — library/fragment builders, toUIOutput, getUiStream() (DEV-773)

.changeset/openui-bindings.md

@@ -0,0 +1,76 @@
+---
+'@openrouter/agent': minor
+---
+
+OpenUI bindings: a component-library model (`defineComponent`, `createLibrary`, `componentProps`), a typed fragment builder (`fragment`, `uiRef`, `uiState`, `uiBuiltin`), the `openui` plugin helper, `serializeExpr`/`OPENUI_LANG_DIALECT` for emitting OpenUI Lang, a `toUIOutput` tool option that renders a tool's result as UI, and `ModelResult.getUiStream()` for consuming fragments as they arrive.
+
+A tool declares how its output renders, and the caller streams the fragments:
+
+```ts
+import {
+  callModel,
+  createLibrary,
+  defineComponent,
+  fragment,
+  openui,
+  tool,
+} from '@openrouter/agent';
+import { z } from 'zod/v4';
+
+const library = createLibrary([
+  defineComponent({
+    name: 'Card',
+    description: 'Container with a title',
+    props: z.object({
+      title: z.string(),
+      children: z.array(z.unknown()).optional(),
+    }),
+  }),
+  defineComponent({
+    name: 'Text',
+    props: z.object({
+      value: z.string(),
+    }),
+  }),
+]);
+
+const ui = fragment(library);
+
+const weather = tool({
+  name: 'weather',
+  inputSchema: z.object({
+    city: z.string(),
+  }),
+  outputSchema: z.object({
+    summary: z.string(),
+  }),
+  execute: ({ city }) => ({
+    summary: `Clear in ${city}`,
+  }),
+  // Renders the tool's result instead of leaving the model to describe it.
+  toUIOutput: ({ input, output }) =>
+    ui.Card(input.city, [
+      ui.Text(output.summary),
+    ]),
+});
+
+const result = callModel(client, {
+  model: 'anthropic/claude-sonnet-4.5',
+  input: 'What is the weather in Lisbon?',
+  tools: [
+    weather,
+  ],
+  plugins: [
+    // `as never` until the SDK regen adds `openui` to its plugin union; the
+    // wire shape is already accepted by the API.
+    openui(library) as never,
+  ],
+});
+
+for await (const event of result.getUiStream()) {
+  if (event.type === 'fragment') {
+    // source: 'root = Card("Lisbon", [Text("Clear in Lisbon")])'
+    console.log(event.source);
+  }
+}
+```

packages/agent/src/index.ts

@@ -182,6 +182,41 @@ export {
   buildNextTurnParamsContext,
   executeNextTurnParamsFunctions,
 } from './lib/next-turn-params.js';
+export type {
+  ComponentDefinition,
+  CreateLibraryOptions,
+  FragmentArg,
+  FragmentBuilder,
+  FragmentNode,
+  OpenUiPlugin,
+  OpenUiWireComponent,
+  PropSignature,
+  UiDocumentEvent,
+  UiExpr,
+  UiFragment,
+  UiFragmentEvent,
+  UiLibrary,
+  UiLiteralValue,
+  UiStatementEvent,
+  UiStreamEvent,
+} from './lib/openui/index.js';
+// OpenUI (generative UI) bindings: library model, fragment builder, plugin helper
+export {
+  componentProps,
+  createLibrary,
+  defineComponent,
+  fragment,
+  OPENUI_BUILTIN_COMPONENTS,
+  OPENUI_LANG_DIALECT,
+  OPENUI_ROOT_REF,
+  OPENUI_WIRE_EVENT,
+  openui,
+  serializeExpr,
+  translateUiEvent,
+  uiBuiltin,
+  uiRef,
+  uiState,
+} from './lib/openui/index.js';
 // Stop condition helpers
 export {
   finishReasonIs,
@@ -248,8 +283,10 @@ export type {
   ToolResultEvent,
   ToolResultItem,
   ToolStreamEvent,
+  ToolUiFragmentEvent,
   ToolWithExecute,
   ToolWithGenerator,
+  ToUIOutputFunction,
   TurnContext,
   TurnEndEvent,
   TurnStartEvent,
@@ -272,6 +309,7 @@ export {
   isToolCallOutputEvent,
   isToolPreliminaryResultEvent,
   isToolResultEvent,
+  isToolUiFragmentEvent,
   isTurnEndEvent,
   isTurnStartEvent,
   ToolType,

packages/agent/src/lib/model-result.ts

@@ -33,7 +33,7 @@ import {
 } from './next-turn-params.js';
 import { ReusableReadableStream } from './reusable-stream.js';
 import { isStopConditionMet } from './stop-conditions.js';
-import type { ItemInProgress, StreamableOutputItem } from './stream-transformers.js';
+import type { ItemInProgress, StreamableOutputItem, UiStreamEvent } from './stream-transformers.js';
 import {
   buildItemsStream,
   buildResponsesMessageStream,
@@ -47,6 +47,7 @@ import {
   extractToolDeltas,
   itemsStreamHandlers,
   streamTerminationEvents,
+  translateUiEvent,
 } from './stream-transformers.js';
 import {
   hasTypeProperty,
@@ -78,6 +79,7 @@ import type {
   ToolContextMapWithShared,
   ToolResultItem,
   ToolStreamEvent,
+  ToolUiFragmentEvent,
   TurnContext,
   TurnEndEvent,
   TurnStartEvent,
@@ -2670,6 +2672,8 @@ export class ModelResult<
         output: executedOutput,
         timestamp: Date.now(),
       } satisfies ToolCallOutputEvent);
+
+      await this.broadcastUiFragment(value);
     }
 
     return {
@@ -2678,6 +2682,54 @@ export class ModelResult<
     };
   }
 
+  /**
+   * Compute and broadcast a tool-authored OpenUI fragment for a successful
+   * execution. Render-only: the fragment never reaches the model, so a
+   * throwing `toUIOutput` degrades to "no fragment" instead of failing the
+   * round — the model-facing output has already been pushed.
+   */
+  private async broadcastUiFragment(value: {
+    toolCall: ParsedToolCall<Tool>;
+    tool: Tool;
+    result: {
+      result: unknown;
+      error?: Error;
+    };
+  }): Promise<void> {
+    if (
+      value.result.error ||
+      !isAutoResolvableTool(value.tool) ||
+      !value.tool.function.toUIOutput
+    ) {
+      return;
+    }
+    const rawArgs: unknown = value.toolCall.arguments;
+    if (!isRecord(rawArgs)) {
+      return;
+    }
+    try {
+      const fragment = await value.tool.function.toUIOutput({
+        output: value.result.result,
+        input: rawArgs,
+      });
+      if (!fragment) {
+        return;
+      }
+      this.turnBroadcaster?.push({
+        type: 'tool.ui_fragment' as const,
+        toolCallId: value.toolCall.id,
+        toolName: value.toolCall.name,
+        fragment: {
+          dialect: fragment.dialect,
+          source: fragment.source,
+        },
+        timestamp: Date.now(),
+      } satisfies ToolUiFragmentEvent);
+    } catch {
+      // Fragment construction failed — drop it; rendering is best-effort.
+    }
+  }
+
   /**
    * Resolve async functions for the current turn.
    * Updates the resolved request with turn-specific parameter values.
@@ -4592,6 +4644,53 @@ export class ModelResult<
     }.call(this);
   }
 
+  /**
+   * Stream OpenUI events from all turns: completed OpenUI Lang statements
+   * authored by the model (`response.openui.*` wire events from the `openui`
+   * plugin) and tool-authored fragments (`tool.ui_fragment` events produced
+   * by tools declaring `toUIOutput`).
+   *
+   * Wire events not yet in the SDK's stream-event union arrive through its
+   * forward-compat catch-all; translation reads the raw payload, so this
+   * stream works both before and after the SDK regen picks them up.
+   */
+  getUiStream(): AsyncIterableIterator<UiStreamEvent> {
+    return async function* (this: ModelResult<TTools, TShared>) {
+      await this.initStreamGuarded();
+
+      if (!this.options.tools?.length) {
+        let streamFailed = false;
+        try {
+          if (this.reusableStream) {
+            for await (const event of this.reusableStream.createConsumer()) {
+              const uiEvent = translateUiEvent(event);
+              if (uiEvent) {
+                yield uiEvent;
+              }
+            }
+          }
+        } catch (error) {
+          streamFailed = true;
+          throw error;
+        } finally {
+          await this.finishHooksSessionForStream(streamFailed ? 'error' : 'complete');
+        }
+        return;
+      }
+
+      const { consumer, executionPromise } = this.startTurnBroadcasterExecution();
+
+      for await (const event of consumer) {
+        const uiEvent = translateUiEvent(event);
+        if (uiEvent) {
+          yield uiEvent;
+        }
+      }
+
+      await executionPromise;
+    }.call(this);
+  }
+
   /**
    * Stream tool call argument deltas and preliminary results from all turns.
    * Preliminary results are streamed in REAL-TIME as generator tools yield.

packages/agent/src/lib/openui/document.ts

@@ -0,0 +1,114 @@
+/**
+ * OpenUI Lang expression model + serialization.
+ *
+ * OpenUI Lang is a line-oriented assignment language: one statement per line,
+ * `name = expression`. The SDK only *authors* OpenUI Lang (tool-authored
+ * fragments, wire-format libraries) — parsing, validation, and prompt
+ * injection are API-side responsibilities. This module is therefore the
+ * minimal expression tree and serializer shared by the fragment builder.
+ */
+
+/** The OpenUI Lang dialect this package emits. */
+export const OPENUI_LANG_DIALECT = 'openui-lang/0.5';
+
+/** The reserved assignment ref that designates the document root. */
+export const OPENUI_ROOT_REF = 'root';
+
+export type UiLiteralValue = string | number | boolean | null;
+
+/** Expression tree for one assignment's right-hand side. */
+export type UiExpr =
+  | {
+      kind: 'literal';
+      value: UiLiteralValue;
+    }
+  | {
+      kind: 'ref';
+      name: string;
+    }
+  | {
+      kind: 'state-ref';
+      name: string;
+    }
+  | {
+      kind: 'member';
+      base: UiExpr;
+      path: string[];
+    }
+  | {
+      kind: 'array';
+      items: UiExpr[];
+    }
+  | {
+      kind: 'object';
+      entries: Array<{
+        key: string;
+        value: UiExpr;
+      }>;
+    }
+  | {
+      kind: 'call';
+      fn: string;
+      builtin: boolean;
+      args: UiExpr[];
+    };
+
+/**
+ * A renderable piece of UI: the dialect it's expressed in plus its serialized
+ * OpenUI Lang source. This is the shape carried on `tool.ui_fragment` stream
+ * events and (for server tools) `response.openui.fragment` wire events.
+ */
+export interface UiFragment {
+  dialect: string;
+  source: string;
+}
+
+/**
+ * Bare-identifier object keys, which the grammar accepts unquoted. Anything
+ * else — spaces, quotes, punctuation, a leading digit, the empty string — must
+ * be quoted or the emitted source does not parse.
+ */
+const BARE_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+/**
+ * Object keys reach here from arbitrary tool-authored objects via `toExpr`, so
+ * they cannot be assumed to be identifiers. The parser accepts a quoted key
+ * (`parseObject` branches on `"`), so quoting the rest round-trips.
+ */
+function serializeKey(key: string): string {
+  return BARE_KEY.test(key) ? key : JSON.stringify(key);
+}
+
+/**
+ * Numbers that have no OpenUI Lang literal: `String(NaN)` is `NaN` and
+ * `String(Infinity)` is `Infinity`, both of which serialize as bare identifiers
+ * and would parse back as refs to undefined names (or fail outright). JSON has
+ * the same hole and resolves it as `null`; do the same rather than emit source
+ * that cannot round-trip.
+ */
+function serializeNumber(value: number): string {
+  return Number.isFinite(value) ? String(value) : 'null';
+}
+
+/** Serialize an expression to OpenUI Lang source. */
+export function serializeExpr(expr: UiExpr): string {
+  switch (expr.kind) {
+    case 'literal':
+      if (typeof expr.value === 'string') {
+        return JSON.stringify(expr.value);
+      }
+      return typeof expr.value === 'number' ? serializeNumber(expr.value) : String(expr.value);
+    case 'ref':
+      return expr.name;
+    case 'state-ref':
+      return `$${expr.name}`;
+    case 'member':
+      return `${serializeExpr(expr.base)}.${expr.path.join('.')}`;
+    case 'array':
+      return `[${expr.items.map(serializeExpr).join(', ')}]`;
+    case 'object':
+      return `{${expr.entries.map((e) => `${serializeKey(e.key)}: ${serializeExpr(e.value)}`).join(', ')}}`;
+    case 'call':
+      return `${expr.builtin ? '@' : ''}${expr.fn}(${expr.args.map(serializeExpr).join(', ')})`;
+  }
+}