Repair voice turns that claim a knowledge gap without searching

apps/desk/scripts/voice-eval-worker.ts

@@ -10,6 +10,7 @@ import {
   voiceAgentSystemPrompt,
 } from '../src/voice/conversation'
 import { dedupAssistantText, dedupRepeatedSentences } from '../src/voice/dedup'
+import { claimsKnowledgeGap, prepareForcedSearchStep, repairUnsearchedKnowledgeGap } from '../src/voice/repair'
 import { ESCALATION_CATEGORIES } from '../src/voice/escalation'
 
 type EvalEnv = { AI: Ai }
@@ -168,8 +169,13 @@ export default {
       const toolCalls: { name: string; input: unknown }[] = []
       const errors: string[] = []
       let text = ''
-      // Mirrors production: the agent wraps fullStream in the same dedup.
-      for await (const part of dedupAssistantText(result.fullStream)) {
+      // Mirrors production: the agent wraps fullStream in the same
+      // knowledge-gap repair and dedup.
+      const repaired = repairUnsearchedKnowledgeGap(
+        result.fullStream,
+        () => streamText({ ...turnOptions, prepareStep: prepareForcedSearchStep }).fullStream,
+      )
+      for await (const part of dedupAssistantText(repaired)) {
         const p = part as { type: string; text?: string; toolName?: string; input?: unknown; error?: unknown }
         streamParts.push(p.type)
         if (p.type === 'text-delta') text += p.text ?? ''
@@ -179,13 +185,25 @@ export default {
       return Response.json({ text, toolCalls, streamParts, errors })
     }
 
-    const result = await generateText(turnOptions)
+    const collectToolCalls = (
+      steps: ReadonlyArray<{ toolCalls: ReadonlyArray<{ toolName: string; input: unknown } | null | undefined> }>,
+    ) => steps.flatMap((step) => step.toolCalls.flatMap((call) => (call ? [{
+      name: call.toolName,
+      input: call.input,
+    }] : [])))
+    let result = await generateText(turnOptions)
+    let repaired = false
+    // Mirrors production's stream repair: a turn that claims a knowledge gap
+    // without any tool call is re-run once with the search forced, and only
+    // the repaired turn is reported.
+    if (collectToolCalls(result.steps).length === 0 && claimsKnowledgeGap(result.text)) {
+      result = await generateText({ ...turnOptions, prepareStep: prepareForcedSearchStep })
+      repaired = true
+    }
     return Response.json({
       text: dedupRepeatedSentences(result.text),
-      toolCalls: result.steps.flatMap((step) => step.toolCalls.flatMap((call) => (call ? [{
-        name: call.toolName,
-        input: call.input,
-      }] : []))),
+      toolCalls: collectToolCalls(result.steps),
+      ...(repaired ? { repaired } : {}),
     })
   },
 }

apps/desk/src/voice/demo-agent.ts

@@ -51,6 +51,7 @@ import {
   type VoiceVerificationChallenge,
 } from './verification'
 import { dedupAssistantText } from './dedup'
+import { prepareForcedSearchStep, repairUnsearchedKnowledgeGap } from './repair'
 import {
   SHOPIFY_CUSTOMER_SESSION_COOKIE,
   shopifyCustomerConfigured,
@@ -356,7 +357,8 @@ export class AbleDeskAgent extends VoiceAgent<Env> {
     let contactCardRequested = false
     const settings = await loadWorkspaceSettings(this.env.DB)
     const workersAI = createWorkersAI({ binding: this.env.AI })
-    const result = streamText({
+    const invokeVoiceModel = (repairing: boolean) => streamText({
+      ...(repairing ? { prepareStep: prepareForcedSearchStep } : {}),
       model: workersAI(VOICE_AGENT_MODEL, {
         sessionAffinity: this.sessionAffinity,
         reasoning_effort: null,
@@ -478,11 +480,17 @@ export class AbleDeskAgent extends VoiceAgent<Env> {
       stopWhen: stepCountIs(4),
       abortSignal: context.signal,
     })
+    const result = invokeVoiceModel(false)
 
-    // At temperature 0 the model sometimes restates its pre-tool-call sentence
-    // verbatim after the tool result; the wrapper drops exact repeats within
+    // A turn that claims a knowledge gap without any tool call is re-run once
+    // with the help-centre search forced, and at temperature 0 the model
+    // sometimes restates its pre-tool-call sentence verbatim after the tool
+    // result; the wrappers repair the former and drop exact repeats within
     // the turn before they reach TTS and the transcript.
-    const stream = dedupAssistantText(result.fullStream)
+    const stream = dedupAssistantText(repairUnsearchedKnowledgeGap(
+      result.fullStream,
+      () => invokeVoiceModel(true).fullStream,
+    ))
     if (contact) return stream
 
     // Safety net for the anonymous branch: the model occasionally speaks the

apps/desk/src/voice/repair.ts

@@ -0,0 +1,88 @@
+/**
+ * Deterministic repair for a turn that claims a knowledge gap it never
+ * verified.
+ *
+ * The system prompt forbids saying anything is undocumented unless
+ * search_help_center returned no_match in the same turn, yet the voice model
+ * at temperature 0 still answers unfamiliar-product questions with "I don't
+ * have information on that" without searching. The broken shape is
+ * deterministic to detect — the turn made no tool call and the reply claims
+ * missing documentation — so the turn is re-run once with the help-centre
+ * search forced on the first step, and only the repaired turn reaches the
+ * caller.
+ *
+ * Scope redirects for off-topic questions ("I can only help with support…",
+ * "I'm not able to discuss…") assert what the assistant will not do, never
+ * that documentation is missing, so they are never re-run and an off-topic
+ * turn never gains a tool call. The detector keys on a claim of missing
+ * knowledge material: a negated possession verb reaching a documentation
+ * noun, a negated existence claim about one, or "not documented/covered".
+ */
+
+// "to answer"/"to guide" are excluded so redirect phrasings like "I don't
+// have the ability to answer that" read as refusals, not as missing answers.
+const NEGATED_POSSESSION = /\b(?:do(?:es)?\s?not|don['’]t|doesn['’]t|did\s?not|didn['’]t|could\s?not|couldn['’]t|can\s?not|can['’]t|cannot|unable\s+to)\b[^.!?]{0,24}?\b(?:have|find|locate|see|provide|offer|access)\b[^.!?]{0,50}?\b(?:info(?:rmation)?|document(?:ation|ed)?|article|(?<!to\s)guide|knowledge|(?<!to\s)answer|steps)\b/i
+// A bare "no answer" (a device that stays silent) is not a knowledge claim;
+// "answer" needs a documentation adjective here.
+const NEGATED_EXISTENCE = /\b(?:there\s+(?:is|are)\s+no|no)\s+(?:(?:documented|published|available|specific)\s+answers?|(?:documented|published|available|specific)?\s*(?:info(?:rmation)?|documentation|article|guide)s?)\b/i
+const NOT_DOCUMENTED = /(?:\bnot|n['’]t|\bnever)\s+(?:documented|covered)\b/i
+// A reply that names the topic as general/unrelated knowledge is a scope
+// redirect even when it also uses gap-like wording; redirects never repair.
+const SCOPE_REDIRECT = /\b(?:general|unrelated|off[\s-]?topic)\s+(?:questions?|topics?|knowledge)\b|\bprogramming\s+(?:languages?|topics?)\b/i
+
+/** True when the reply claims that documentation or knowledge is missing. */
+export function claimsKnowledgeGap(text: string): boolean {
+  if (SCOPE_REDIRECT.test(text)) return false
+  return NEGATED_POSSESSION.test(text) || NEGATED_EXISTENCE.test(text) || NOT_DOCUMENTED.test(text)
+}
+
+/**
+ * Per-step settings for the repaired run: the first step must call
+ * search_help_center; later steps are unconstrained so the model can answer
+ * from the result or continue with request_contact.
+ */
+export function prepareForcedSearchStep(
+  { stepNumber }: { stepNumber: number },
+): { toolChoice?: { type: 'tool'; toolName: 'search_help_center' } } {
+  return stepNumber === 0 ? { toolChoice: { type: 'tool', toolName: 'search_help_center' } } : {}
+}
+
+/**
+ * Wraps an AI SDK fullStream. Parts are buffered until the first tool call;
+ * a tool call proves the turn consulted a tool, so everything is released
+ * and the rest passes through live. If the turn ends with no tool call and
+ * the accumulated text claims a knowledge gap, the buffered turn is
+ * discarded and the rerun (which must force the search) is streamed in its
+ * place. Voice replies are capped at 120 output tokens, so the buffering
+ * window is small.
+ */
+export async function* repairUnsearchedKnowledgeGap<Part extends { type: string }>(
+  stream: AsyncIterable<Part>,
+  rerun: () => AsyncIterable<Part>,
+): AsyncIterable<Part> {
+  const buffered: Part[] = []
+  let text = ''
+  let toolCalled = false
+  for await (const part of stream) {
+    if (toolCalled) {
+      yield part
+      continue
+    }
+    const candidate = part as { type: string; text?: unknown }
+    if (candidate.type === 'tool-call') {
+      toolCalled = true
+      for (const held of buffered) yield held
+      buffered.length = 0
+      yield part
+      continue
+    }
+    if (candidate.type === 'text-delta' && typeof candidate.text === 'string') text += candidate.text
+    buffered.push(part)
+  }
+  if (toolCalled) return
+  if (!claimsKnowledgeGap(text)) {
+    for (const held of buffered) yield held
+    return
+  }
+  yield* rerun()
+}

apps/desk/test/voice-repair.test.ts

@@ -0,0 +1,127 @@
+import { describe, expect, it } from 'vitest'
+
+import { claimsKnowledgeGap, prepareForcedSearchStep, repairUnsearchedKnowledgeGap } from '../src/voice/repair'
+
+// Observed @cf/zai-org/glm-4.7-flash output at temperature 0: the broken
+// kbMiss shape — an unfamiliar-product question answered with a knowledge-gap
+// claim and no tool call.
+const UNSEARCHED_GAP_REPLIES = [
+  "I can only help with support for products, services, orders, or accounts. I don't have information on router settings. If you have a support case or a product issue, I can open a ticket for you. What support problem can I help with?",
+  "I can only help with support for products, services, orders, or accounts. I don't have information about adjusting print alignment on routers. If you have a support question about a router, I can open a ticket for the team to assist. Would you like me to do that?",
+  "I don't have a documented answer for adjusting print alignment on a router. If this is related to a specific product or service you're using, I can open a support ticket for you.",
+  "I don't have a documented guide for that. Add your name and email in the card below and I can open a ticket for the team.",
+  'That is not documented in our help articles, so I cannot walk you through it.',
+  'There is no article covering that, but I can open a ticket for the team.',
+  "I couldn't find a guide for that, but the team can help.",
+]
+
+// Observed off-topic scope redirects, plus non-KB capability statements that
+// must never trigger a forced search.
+const NON_GAP_REPLIES = [
+  "I can only help with support for Example Company's products, services, orders, or accounts. If you have a question about a product or service from us, I'm happy to assist. Otherwise, I can open a support ticket for you to discuss this further.",
+  "I can only help with support for Example Company's products, services, orders, or accounts. If you have a question about a product or service from us, I'd be happy to assist. Otherwise, I'm not able to answer general questions like this.",
+  "I can only help with support questions about products, services, orders, or accounts. If you have a support issue, please describe it and I'll assist.",
+  "I can only help with support for Example Company's products, services, orders, accounts, or existing support cases. I'm not able to discuss general programming topics or language comparisons. If you have a support question about a product or service, I'd be happy to assist.",
+  "I'm not able to discuss general programming language comparisons or opinions. If you have a support question, I'm happy to assist.",
+  "You'll get updates by email through your private case link — I can't check ticket status here.",
+  "I couldn't find that order for the email on this session. Double-check the number, or restart the chat with the email used at checkout. I can also open a support ticket for the team.",
+  "I'm having trouble checking orders right now. I can open a support ticket for the team to follow up.",
+  "Please don't share your password. I can help without it—what problem are you seeing?",
+  "A machine that won't start is frustrating. What kind of machine is it?",
+  'Clean the machine every 60 days using a citric acid cleaning solution, then run one full tank of fresh water through afterward. Never use vinegar, as it damages the pump seals.',
+  "I'm not able to answer that, but our support team can guide you through the next steps.",
+  'Happy to check. What is the order number from your confirmation email?',
+  "I don't have the ability to answer general questions like this.",
+  "Got it — there's no answer when you press the power button, correct?",
+  "I don't have anyone to guide you through that right now, but I can open a ticket.",
+]
+
+type Part = { type: string; text?: string; toolName?: string }
+
+async function collect(stream: AsyncIterable<Part>): Promise<Part[]> {
+  const parts: Part[] = []
+  for await (const part of stream) parts.push(part)
+  return parts
+}
+
+async function* parts(...items: Part[]): AsyncIterable<Part> {
+  yield* items
+}
+
+describe('knowledge-gap claim detection', () => {
+  it('matches every observed unsearched knowledge-gap reply', () => {
+    for (const reply of UNSEARCHED_GAP_REPLIES) {
+      expect(claimsKnowledgeGap(reply), reply).toBe(true)
+    }
+  })
+
+  it('never matches scope redirects or non-knowledge capability statements', () => {
+    for (const reply of NON_GAP_REPLIES) {
+      expect(claimsKnowledgeGap(reply), reply).toBe(false)
+    }
+  })
+})
+
+describe('forced search step', () => {
+  it('forces search_help_center only on the first step', () => {
+    expect(prepareForcedSearchStep({ stepNumber: 0 })).toEqual({
+      toolChoice: { type: 'tool', toolName: 'search_help_center' },
+    })
+    expect(prepareForcedSearchStep({ stepNumber: 1 })).toEqual({})
+  })
+})
+
+describe('unsearched knowledge-gap stream repair', () => {
+  it('passes a turn with a tool call through untouched and never reruns', async () => {
+    const source = parts(
+      { type: 'text-delta', text: 'Let me check. ' },
+      { type: 'tool-call', toolName: 'search_help_center' },
+      { type: 'text-delta', text: "I don't have a documented answer for that." },
+      { type: 'finish' },
+    )
+    const collected = await collect(repairUnsearchedKnowledgeGap(source, () => {
+      throw new Error('must not rerun')
+    }))
+    expect(collected.map((part) => part.type)).toEqual(['text-delta', 'tool-call', 'text-delta', 'finish'])
+  })
+
+  it('passes a text-only turn without a gap claim through untouched', async () => {
+    const source = parts(
+      { type: 'text-delta', text: 'What kind of machine is it?' },
+      { type: 'finish' },
+    )
+    const collected = await collect(repairUnsearchedKnowledgeGap(source, () => {
+      throw new Error('must not rerun')
+    }))
+    expect(collected.map((part) => part.type)).toEqual(['text-delta', 'finish'])
+  })
+
+  it('discards an unsearched gap-claim turn and streams the rerun instead', async () => {
+    const source = parts(
+      { type: 'text-delta', text: 'I can only help with support. ' },
+      { type: 'text-delta', text: "I don't have information on router settings." },
+      { type: 'finish' },
+    )
+    const collected = await collect(repairUnsearchedKnowledgeGap(source, () => parts(
+      { type: 'tool-call', toolName: 'search_help_center' },
+      { type: 'text-delta', text: "I don't have a documented answer for that. I can open a ticket." },
+      { type: 'finish' },
+    )))
+    expect(collected[0]).toEqual({ type: 'tool-call', toolName: 'search_help_center' })
+    const text = collected.filter((part) => part.type === 'text-delta').map((part) => part.text).join('')
+    expect(text).toBe("I don't have a documented answer for that. I can open a ticket.")
+  })
+
+  it('detects a gap claim split across text deltas', async () => {
+    const source = parts(
+      { type: 'text-delta', text: "I don't have inform" },
+      { type: 'text-delta', text: 'ation about that product.' },
+      { type: 'finish' },
+    )
+    const collected = await collect(repairUnsearchedKnowledgeGap(source, () => parts(
+      { type: 'tool-call', toolName: 'search_help_center' },
+      { type: 'finish' },
+    )))
+    expect(collected[0]?.type).toBe('tool-call')
+  })
+})