Validate machine input against its schema; add chat-with-pdf example

.changeset/validate-machine-input.md

@@ -0,0 +1,31 @@
+---
+"@statelyai/agent": minor
+---
+
+**Machine input is validated against its schema, and defaulted fields are optional at the call site.**
+
+XState's `schemas` are types only — it never validates, and it resolves `schemas.input` to one type shared by `createActor`'s `input` option and the `context: ({ input })` factory. A field declared with a default was therefore both absent at runtime and required at the call site.
+
+`runAgent`/`createAgentActor` now validate `options.input` against the machine's declared input schema before the actor starts: defaults are filled and transforms applied, and the resolved value is what reaches the actor, the replayable event log, and the `run.start` trace — so a replay reproduces the run even if a default is computed. Invalid input throws an `AgentError` with code `invalid-machine-input` (like a mismatched resume snapshot; the actor never starts). Omitting `input` entirely still skips validation.
+
+```ts
+const agent = setupAgent({
+  schemas: createAgentSchemas({
+    context: z.object({ topic: z.string(), rounds: z.number() }),
+    input: z.object({ topic: z.string(), rounds: z.number().default(3) }),
+  }),
+});
+
+const machine = agent.createMachine({
+  // `rounds` arrives filled in — no `?? 3` restating the default here
+  context: ({ input }) => ({ topic: input.topic, rounds: input.rounds }),
+  // ...
+});
+
+// `rounds` is optional at the call site; `topic` (no default) is not
+await runAgent(machine, { input: { topic: "otters" } });
+```
+
+Standard Schema throughout — no validation library is referenced, so this works with whatever the machine was declared with.
+
+Types: `runAgent`'s `input` is now `AgentInputFrom<TMachine>`, which reads the schema's pre-validation side (`~standard.types.input`) while the context factory keeps the validated side. Machines reached through `.provide(...)` lose the brand and fall back to xstate's `InputFrom`. New exported type helpers: `AgentInputFrom`, `InferInput`.

examples/README.md

@@ -66,6 +66,7 @@ These use `setupAgent(...)` (or plain XState `setup(...)` plus `createTextLogic(
 - [`file-snapshot-store/index.ts`](file-snapshot-store/index.ts): durable HITL checkpoints in a file-backed snapshot store: each idle settle writes JSON to disk and a fresh `runAgent` call resumes across turns
 - [`machine-as-tool/index.ts`](machine-as-tool/index.ts): a whole agent machine embedded inside one tool call of a host harness: start/resume tools bridge a JSON-safe snapshot handle and read the typed interaction meta
 - [`rag/index.ts`](rag/index.ts): retrieve (typed plain actor, keyword scoring over a sample corpus) → grounded answer, with conversational memory in context
+- [`chat-with-pdf/index.ts`](chat-with-pdf/index.ts): agentcn's chat-with-PDF quiz recipe with the sequencing lifted out of `instructions.md` and into the machine — one question per entry into `asking`, a `refreshEvery` guard instead of "after 3-4 questions", `pagesCovered` fed back to retrieval as `excludePages`, `documentId` threaded from context, page hints read off the chunk, and an idle `choosingDocument` state when the library is ambiguous; voice and question formatting stay in the prompt
 - [`corrective-rag/index.ts`](corrective-rag/index.ts): LangGraph's CRAG tutorial as explicit states: retrieve → grade documents → conditional correction branch (rewrite query → web-search fallback) → grounded generate, every model call with a degrading `onError`
 - [`adaptive-rag/index.ts`](adaptive-rag/index.ts): LangGraph adaptive RAG as route → retrieve/search → grade → bounded rewrite → generate → groundedness/usefulness verification
 - [`deep-research/index.ts`](deep-research/index.ts): plan 2-4 complementary searches → one dynamically spawned researcher per query → coverage reflection → optional targeted follow-up → sourced report; search remains host-owned

examples/chat-with-pdf/index.test.ts

@@ -0,0 +1,251 @@
+import { describe, expect, test } from "vitest";
+import { runAgent } from "@statelyai/agent";
+import type { AgentRequestExecutor } from "@statelyai/agent";
+import {
+  chatWithPdfMachine,
+  idlePrompt,
+  queryPdfContent,
+  SAMPLE_LIBRARY,
+  type LearnerEvent,
+} from "./index.js";
+
+type MachineInput = {
+  documentId?: string | null;
+  topic?: string;
+  pageStart?: number | null;
+  pageEnd?: number | null;
+  maxQuestions?: number;
+  refreshEvery?: number;
+};
+
+interface PlayOptions {
+  input?: MachineInput;
+  /** Consumed in order on each idle settle. */
+  learnerEvents: LearnerEvent[];
+  /** Grade every answer as correct unless this says otherwise. */
+  grade?: (answer: string) => boolean;
+}
+
+interface PlayResult {
+  status: string;
+  output?: {
+    documentId: string;
+    correct: number;
+    answered: number;
+    pagesCovered: number[];
+    results: Array<{ pageNumber: number; prompt: string; correct: boolean }>;
+    exhausted: boolean;
+  };
+  /** Passages `writeQuestion` was given, in order. */
+  passages: string[];
+  /** Prompts `gradeAnswer` was given, in order. */
+  gradePrompts: string[];
+  /** Every idle label the run settled on. */
+  idleLabels: string[];
+}
+
+async function play(options: PlayOptions): Promise<PlayResult> {
+  const passages: string[] = [];
+  const gradePrompts: string[] = [];
+  const idleLabels: string[] = [];
+  let questionNumber = 0;
+
+  const generateText: AgentRequestExecutor = async (request) => {
+    if (request.system?.includes("Grade a quiz answer")) {
+      gradePrompts.push(request.prompt ?? "");
+      const answer = (request.prompt ?? "").match(/Learner's answer: (.*)/)?.[1] ?? "";
+      return {
+        output: {
+          correct: options.grade ? options.grade(answer) : true,
+          explanation: `graded "${answer}"`,
+        },
+      };
+    }
+    // writeQuestion
+    const passage = (request.prompt ?? "").match(/Passage:\n(.*)/)?.[1] ?? "";
+    passages.push(passage);
+    questionNumber += 1;
+    return {
+      output: {
+        type: "short-answer",
+        question: `Q${questionNumber} about: ${passage.slice(0, 24)}`,
+        choices: [],
+      },
+    };
+  };
+
+  const queue = [...options.learnerEvents];
+  let result = await runAgent(chatWithPdfMachine, {
+    input: options.input ?? {},
+    executors: { generateText },
+  });
+
+  while (result.status === "idle") {
+    idleLabels.push(idlePrompt(result.snapshot));
+    const event = queue.shift();
+    if (!event) break;
+    result = await runAgent(chatWithPdfMachine, {
+      snapshot: result.persistedSnapshot,
+      event,
+      executors: { generateText },
+    });
+  }
+
+  return {
+    status: result.status,
+    output: result.status === "done" ? (result.output as PlayResult["output"]) : undefined,
+    passages,
+    gradePrompts,
+    idleLabels,
+  };
+}
+
+const answers = (count: number): LearnerEvent[] =>
+  Array.from({ length: count }, (_, i) => ({ type: "ANSWER", text: `answer ${i + 1}` }));
+
+describe("chat-with-pdf quiz mode", () => {
+  test("one question per idle settle: the machine cannot pose two at once", async () => {
+    const result = await play({
+      input: { documentId: "statecharts", maxQuestions: 4, refreshEvery: 2 },
+      learnerEvents: answers(4),
+    });
+
+    expect(result.status).toBe("done");
+    // Four questions asked, four idle settles, four gradings. No batching.
+    expect(result.passages).toHaveLength(4);
+    expect(result.idleLabels).toHaveLength(4);
+    expect(result.output?.answered).toBe(4);
+    // Each idle label is exactly the one pending question.
+    for (const label of result.idleLabels) {
+      expect(label.match(/^Q\d+ about:/m)).not.toBeNull();
+      expect(label.match(/Hint: see page \d+/g)).toHaveLength(1);
+    }
+  });
+
+  test("covered pages are excluded by the query, so refreshed batches never repeat a page", async () => {
+    // 6 questions with refreshEvery 2 forces three retrievals.
+    const result = await play({
+      input: { documentId: "statecharts", maxQuestions: 6, refreshEvery: 2 },
+      learnerEvents: answers(6),
+    });
+
+    expect(result.status).toBe("done");
+    const pages = result.output!.pagesCovered;
+    expect(pages).toHaveLength(6);
+    expect(new Set(pages).size).toBe(6);
+    // Passages came from distinct pages too, not the same top-scoring chunk.
+    expect(new Set(result.passages).size).toBe(6);
+  });
+
+  test("documentId is threaded by the machine: every passage comes from the chosen document", async () => {
+    const result = await play({
+      input: { documentId: "retrieval", maxQuestions: 4, refreshEvery: 2 },
+      learnerEvents: answers(4),
+    });
+
+    const chosen = SAMPLE_LIBRARY.find((entry) => entry.id === "retrieval")!;
+    const owned = new Set(chosen.pages.map((page) => page.content));
+    expect(result.passages.length).toBeGreaterThan(0);
+    for (const passage of result.passages) {
+      expect(owned.has(passage)).toBe(true);
+    }
+    expect(result.output?.documentId).toBe("retrieval");
+  });
+
+  test("grading is grounded on the exact passage the question came from", async () => {
+    const result = await play({
+      input: { documentId: "statecharts", maxQuestions: 3, refreshEvery: 3 },
+      learnerEvents: answers(3),
+    });
+
+    expect(result.gradePrompts).toHaveLength(3);
+    result.gradePrompts.forEach((prompt, index) => {
+      expect(prompt).toContain(result.passages[index]!);
+      expect(prompt).toMatch(/Source passage \(page \d+\)/);
+    });
+  });
+
+  test("an ambiguous library stops at the document picker, and an unknown choice does not advance", async () => {
+    const result = await play({
+      input: { maxQuestions: 2, refreshEvery: 2 },
+      learnerEvents: [
+        { type: "SELECT_DOCUMENT", documentId: "not-a-document" },
+        { type: "SELECT_DOCUMENT", documentId: "Retrieval Systems" },
+        ...answers(2),
+      ],
+    });
+
+    expect(result.status).toBe("done");
+    // First settle is the picker; the rejected choice leaves the machine there.
+    expect(result.idleLabels[0]).toContain("Which document");
+    expect(result.idleLabels[1]).toContain("Which document");
+    // No retrieval ran against a guessed id.
+    expect(result.passages.length).toBe(2);
+    expect(result.output?.documentId).toBe("retrieval");
+  });
+
+  test("running out of fresh pages ends the session instead of repeating", async () => {
+    // The retrieval doc has 6 pages; ask for 8.
+    const result = await play({
+      input: { documentId: "retrieval", maxQuestions: 8, refreshEvery: 2 },
+      learnerEvents: answers(8),
+    });
+
+    expect(result.status).toBe("done");
+    expect(result.output?.exhausted).toBe(true);
+    expect(result.output?.answered).toBe(6);
+    expect(new Set(result.output!.pagesCovered).size).toBe(6);
+  });
+
+  test("SKIP advances the loop without scoring", async () => {
+    const result = await play({
+      input: { documentId: "statecharts", maxQuestions: 3, refreshEvery: 3 },
+      learnerEvents: [{ type: "SKIP" }, ...answers(2)],
+    });
+
+    expect(result.status).toBe("done");
+    expect(result.passages).toHaveLength(3);
+    // Three questions posed, only two graded.
+    expect(result.output?.answered).toBe(2);
+    expect(result.output?.pagesCovered).toHaveLength(3);
+  });
+
+  test("STOP ends with the score so far", async () => {
+    const result = await play({
+      input: { documentId: "statecharts", maxQuestions: 6, refreshEvery: 3 },
+      learnerEvents: [{ type: "ANSWER", text: "yes" }, { type: "STOP" }],
+    });
+
+    expect(result.status).toBe("done");
+    expect(result.output?.answered).toBe(1);
+    expect(result.output?.correct).toBe(1);
+  });
+
+  test("retrieval stratifies across the document instead of clustering", () => {
+    const chunks = queryPdfContent({
+      documentId: "statecharts",
+      topic: "",
+      pageStart: null,
+      pageEnd: null,
+      excludePages: [],
+      limit: 3,
+    });
+
+    expect(chunks).toHaveLength(3);
+    // Nine pages, thirds are 1-3 / 4-6 / 7-9: one page from each band.
+    expect(chunks.map((chunk) => chunk.pageNumber)).toEqual([1, 4, 7]);
+  });
+
+  test("page range is honored by the query", () => {
+    const chunks = queryPdfContent({
+      documentId: "statecharts",
+      topic: "",
+      pageStart: 4,
+      pageEnd: 6,
+      excludePages: [5],
+      limit: 5,
+    });
+
+    expect(chunks.map((chunk) => chunk.pageNumber)).toEqual([4, 6]);
+  });
+});