diff --git a/src/500-application/516-chat-with-your-factory/.env.template b/src/500-application/516-chat-with-your-factory/.env.template index 2f26b7ba..ee37e569 100644 --- a/src/500-application/516-chat-with-your-factory/.env.template +++ b/src/500-application/516-chat-with-your-factory/.env.template @@ -61,15 +61,16 @@ TEAMS_AUTH_AUDIENCES=, AZURE_SPEECH_REGION= AZURE_SPEECH_RESOURCE_ID= -# Azure Voice Live (preview — Foundry resource) +# Azure Voice Live (Foundry resource) # Server runtime reads VOICE_PROVIDER to decide whether to attach /api/voice-live bridge. # Keep this in sync with the client bundle provider selected by npm scripts # (build:client:* or start:* sets __SPEECH_PROVIDER__). # Required when VOICE_PROVIDER=voicelive and client is built for voicelive. # Auth: server-side DefaultAzureCredential (no key); identity needs "Cognitive Services User" on the resource +# gpt-realtime requires a supported Voice Live region; use East US 2 for the documented Azure setup. AZURE_VOICELIVE_RESOURCE= AZURE_VOICELIVE_MODEL=gpt-realtime -AZURE_VOICELIVE_API_VERSION=2025-10-01 +AZURE_VOICELIVE_API_VERSION=2026-04-10 # Voice provider selector (server runtime): "azure" (default), "webspeech", or "voicelive" # Client provider is chosen at build time via __SPEECH_PROVIDER__. diff --git a/src/500-application/516-chat-with-your-factory/README.md b/src/500-application/516-chat-with-your-factory/README.md index eae95fb6..b5e14dfc 100644 --- a/src/500-application/516-chat-with-your-factory/README.md +++ b/src/500-application/516-chat-with-your-factory/README.md @@ -1,7 +1,7 @@ --- title: Chat With Factory description: A voice-enabled AI agent web application for industrial environments powered by Azure AI Foundry Agents or Copilot Studio -ms.date: 2026-06-29 +ms.date: 2026-08-04 ms.topic: overview keywords: - chat with factory @@ -200,7 +200,7 @@ Create the Foundry resource, project, and model deployment: ```powershell $RG_NAME = "rg-chat-with-your-factory" -$LOCATION = "eastus" +$LOCATION = "eastus2" $AI_NAME = "chat-factory-ai" $PROJECT = "chat-factory-project" @@ -214,6 +214,10 @@ az cognitiveservices account create ` --location $LOCATION ` --custom-domain $AI_NAME +# East US 2 supports the default gpt-realtime model for Voice Live. +# Existing AI Services resources cannot change regions; create a replacement +# in East US 2 and update AZURE_VOICELIVE_RESOURCE when migrating from East US. + az cognitiveservices account deployment create ` --name $AI_NAME ` --resource-group $RG_NAME ` @@ -544,7 +548,7 @@ Each concern maps to a specific technology. No component serves double duty. The | Concern | Technology | Details | MVP Upgrade | |---------------------------|----------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------| | Speech-to-text (STT) | Configurable: Azure Speech SDK (default) or Browser Web Speech API or Voice Live | Build-time switch via `npm run build:client:azure`, `npm run build:client:webspeech`, or `npm run build:client:voicelive`. esbuild tree-shakes the unused providers at build time. Voice Live (`VOICE_PROVIDER=voicelive`) uses a server-proxied WebSocket bridge at `/api/voice-live` — see ADR 0001. | Server-side real-time STT via Azure Speech SDK. Browser streams raw audio to Express server via WebSocket; server runs Speech SDK for transcription. | -| Text-to-speech (TTS) | None today | Voice Live can produce TTS in principle, but the current bridge runs it as STT/VAD only (`modalities: ['text']`, `create_response: false`) — see ADR 0001. Agent responses render as text in all configurations. | Azure Speech neural TTS for optional read-aloud via `/api/tts` endpoint, or enable Voice Live audio output once topology supports it. | +| Text-to-speech (TTS) | Voice Live PCM16 output | The authoritative backend Markdown is persisted and rendered as text. A citation- and URL-free derivative is sent to Voice Live, and the browser plays the returned 24 kHz PCM16 audio. | Add distributed synthesis latency and interruption telemetry. | | AI / LLM backend | Copilot Studio (Agents SDK), Azure AI Foundry, or Copilot Studio (Direct Line) | Configurable via `AGENT_BACKEND` env var (`copilotstudio` default). Agents SDK uses `@microsoft/agents-copilotstudio-client` with OBO token exchange and streaming. Foundry uses Assistants-style threads with `createAndPoll`. Direct Line uses fire-and-forget POST with WebSocket relay for bot replies. | Foundry streaming via `createAndStream` + SSE for real-time token delivery. | | Server framework | Express v5 (TypeScript, ESM) | Serves static files, REST API routes, SSE broadcast, and `Permissions-Policy` header for iframe microphone access. | Same. Native async error handling. Managed Identity via `DefaultAzureCredential` in production. | | User authentication | Teams SSO + JWT validation (`jose`) | Validates AAD v2.0 tokens. Dev bypass via `SKIP_AUTH=true`. | Remove `SKIP_AUTH` bypass in production. | @@ -560,7 +564,7 @@ A single user interaction follows this path: 1. The user clicks the mic button or types a message. 2. For voice: the `useSpeech` barrel hook delegates to either the Web Speech API, Azure Speech SDK, or Voice Live hook based on the build-time `__SPEECH_PROVIDER__` constant. -3. For Web Speech and Azure Speech providers, the React client sends a POST to `/api/chat` with the message text. For Voice Live, the browser sends raw PCM16 audio over a WebSocket to `/api/voice-live`; the server bridge handles transcription and agent dispatch (via `dispatchChat`) — no `/api/chat` POST is made for voice turns. Agent responses arrive via SSE and render as text (no TTS playback today). +3. For Web Speech and Azure Speech providers, the React client sends a POST to `/api/chat` with the message text. For Voice Live, the browser sends raw PCM16 audio over a WebSocket to `/api/voice-live`; the server bridge handles transcription and agent dispatch (via `dispatchChat`), so no `/api/chat` POST is made for voice turns. Agent responses arrive as authoritative Markdown through SSE, while Voice Live returns a speech-safe PCM16 rendition for browser playback. 4. The Express server validates the JWT, checks the user's session ACL, stores the user message locally, and broadcasts it via SSE. 5. **Copilot Studio (Agents SDK) backend**: The server performs an OBO token exchange using MSAL, connects to the Copilot Studio agent via `@microsoft/agents-copilotstudio-client`, streams the response, stores it locally, broadcasts via SSE, and returns it in the HTTP response. 6. **Foundry backend**: The server forwards the message to the Foundry agent via `agentsClient.messages.create`, calls `createAndPoll`, retrieves the response, stores it locally, broadcasts via SSE, and returns it in the HTTP response. @@ -634,7 +638,11 @@ the result back to Foundry. 4. The agent response appears in the chat panel. > [!TIP] -> The default speech provider (Azure Speech SDK) works across all browsers. To use the browser Web Speech API instead (Chrome/Edge only), run `npm run start:webspeech`. To use Voice Live (preview), set `VOICE_PROVIDER=voicelive` and `AZURE_VOICELIVE_RESOURCE` in `.env`, then run `npm run start:voicelive`. See ADR 0001 for details. The text input fallback is always available. +> The default speech provider (Azure Speech SDK) works across all browsers. To use the browser Web Speech API instead (Chrome/Edge only), run `npm run start:webspeech`. +> +> To use Voice Live, create the AI Services resource in a [region that supports `gpt-realtime`](https://learn.microsoft.com/azure/ai-services/speech-service/regions?tabs=voice-live#regions), such as East US 2. Set `VOICE_PROVIDER=voicelive`, `AZURE_VOICELIVE_RESOURCE`, `AZURE_VOICELIVE_MODEL=gpt-realtime`, and `AZURE_VOICELIVE_API_VERSION=2026-04-10` in `.env`, then run `npm run start:voicelive`. +> +> Existing AI Services resources cannot change regions and must be replaced when migrating from East US. See ADR 0001 for details. The text input fallback is always available. > > Plain `npm start` always rebuilds the client with the **default Azure Speech SDK** provider, which will overwrite any voicelive / webspeech bundle you previously built. Use `npm run start:voicelive` or `npm run start:webspeech` to keep the matching client bundle in sync with the server in one step. diff --git a/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/templates/deployment.yaml b/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/templates/deployment.yaml index 58808f79..102b492d 100644 --- a/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/templates/deployment.yaml +++ b/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/templates/deployment.yaml @@ -15,7 +15,13 @@ spec: metadata: labels: {{- include "chat-with-your-factory.selectorLabels" . | nindent 8 }} + {{- with .Values.serviceAccount.name }} + azure.workload.identity/use: "true" + {{- end }} spec: + {{- with .Values.serviceAccount.name }} + serviceAccountName: {{ . | quote }} + {{- end }} {{- with .Values.imagePullSecrets }} imagePullSecrets: {{- toYaml . | nindent 8 }} diff --git a/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/values.yaml b/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/values.yaml index c48a5b1c..9b84c7c1 100644 --- a/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/values.yaml +++ b/src/500-application/516-chat-with-your-factory/charts/chat-with-your-factory/values.yaml @@ -19,6 +19,11 @@ imagePullSecrets: nameOverride: "" fullnameOverride: "" +# Externally managed Azure Workload Identity ServiceAccount. Leave name empty +# to render no ServiceAccount binding or workload identity pod label. +serviceAccount: + name: "" + # Dedicated namespace scopes RBAC, NetworkPolicies, and quotas to this workload. # Operators may override via --set namespace=. namespace: chat-with-your-factory @@ -39,6 +44,14 @@ resources: env: AGENT_BACKEND: "foundry" PORT: "3978" + # Runtime VOICE_PROVIDER=voicelive requires an image built with the Docker + # build argument SPEECH_PROVIDER=voicelive; SPEECH_PROVIDER is build-time only. + VOICE_PROVIDER: "azure" + # Required deployment override when VOICE_PROVIDER=voicelive. The default + # gpt-realtime model requires a supported Voice Live region such as East US 2. + AZURE_VOICELIVE_RESOURCE: "" + AZURE_VOICELIVE_MODEL: "gpt-realtime" + AZURE_VOICELIVE_API_VERSION: "2026-04-10" # Foundry backend (AGENT_BACKEND=foundry): point at the 085-ai-foundry component. # FOUNDRY_ENDPOINT is the Foundry project endpoint; FOUNDRY_AGENT_ID is the deployed agent. # Create the agent reproducibly with `npm run provision:agent` (needs FOUNDRY_MODEL_DEPLOYMENT). @@ -72,3 +85,5 @@ env: secrets: {} # DIRECT_LINE_SECRET: "" # TEAMS_APP_CLIENT_SECRET: "" + # Required outside development; inject externally with no committed default. + # RESUME_TOKEN_SECRET: "" diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/Dockerfile b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/Dockerfile index 5b567ac6..3052cac7 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/Dockerfile +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/Dockerfile @@ -3,9 +3,10 @@ FROM node:22-slim@sha256:813a7480f28fdadac1f7f5c824bcdad435b5bc1322a5968bbbdef8d WORKDIR /app ARG SPEECH_PROVIDER=azure +ARG NPM_REGISTRY=https://registry.npmjs.org/ COPY package.json package-lock.json ./ -RUN npm ci --ignore-scripts +RUN npm ci --ignore-scripts --registry="${NPM_REGISTRY}" COPY tsconfig.json tsconfig.server.json ./ COPY src/ src/ diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/package-lock.json b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/package-lock.json index 2b168029..4de03a7c 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/package-lock.json +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/package-lock.json @@ -2932,9 +2932,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2952,9 +2949,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2972,9 +2966,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2992,9 +2983,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3012,9 +3000,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -3032,9 +3017,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6636,9 +6618,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6660,9 +6639,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6684,9 +6660,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6708,9 +6681,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -8695,6 +8665,474 @@ } } }, + "node_modules/vitest/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha1-egGo0uwvuy2seK2tCbD6eB5Agr4=", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha1-cEvSl95tdi3lTqu+r79V9nVqvi8=", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha1-tUCifRTkr9BYSWpNvsTT9BTbEQo=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha1-0csWbTSw+/D+irRgpVlPJKN4cB4=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha1-EDSyZFf8iGNo/mG70J9lP2r6jlQ=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha1-ZVVqQyoeTXIDLYIYwZMvzKGkl3I=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha1-LmHgWS+QMNfj2uGO4l68U1kYrvY=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha1-yV7CiZWe+AecTcqBeh4sS+Zrm9M=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha1-wJoPZ5F1kqwN6JKpvk04FN69Kmw=", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha1-QLIhdd2gYYLz7oFBGGxf8wTEpxc=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha1-pYD5xnZ5eDOJHlGfx6EzfIr9jbM=", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha1-RkUs8yHcf56Rwvp4Cla7Vuec1os=", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha1-QhGzGE3WYI9T3LIuOfXTTuCIUsg=", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha1-aXhXwqYcubC2u2ZS5AwdxeHKjl0=", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha1-0ZKUPrFGpArExkl9DPe+NbmGvwg=", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha1-rOoDVtoODrwI+Xz3ucLkAeHmSNw=", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha1-bww84MtkxTS3DExF7LLBbTTjXf0=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha1-i813B3oNzjN4tXT+2ybSolO3PTY=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha1-5/sqAemcgwyU5mI82f77TI+1g0c=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha1-xSkJNy24uG4sVeBaiUADO1Zgo7I=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha1-xCe5vlpkwmL/mn63C1+7qt9EbGw=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha1-3JsUe6yi5sSzyFVxdB70hgpIkJc=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha1-zoZtEt8TwV5MmfBzo9Rm9uBkmzo=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha1-dGjjaS0B1inVlB5dg4F7uA+eObQ=", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha1-pbwAY/sryrbQ7WPyoVN5WLwmnsY=", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, + "node_modules/vitest/node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha1-EAZO5E9DR7kMmgK0Rrv4CpFjKxI=", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=18" + } + }, "node_modules/vitest/node_modules/@vitest/mocker": { "version": "4.1.9", "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.9.tgz", @@ -8722,6 +9160,50 @@ } } }, + "node_modules/vitest/node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha1-70W0Y0ycnZeilq6kEUpfmED5VXg=", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "peer": true, + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, "node_modules/vitest/node_modules/vite": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/vite/-/vite-8.1.2.tgz", diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/App.tsx b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/App.tsx index 2e6be78b..e4d692cc 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/App.tsx +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/App.tsx @@ -1,5 +1,6 @@ import { useState, useCallback, useRef, useEffect } from 'react' -import { FluentProvider, makeStyles, tokens } from '@fluentui/react-components' +import { Button, FluentProvider, makeStyles, tokens } from '@fluentui/react-components' +import { DeleteRegular } from '@fluentui/react-icons' import { ChatPanel } from './components/ChatPanel.js' import { VoiceInput } from './components/VoiceInput.js' import { TextInput } from './components/TextInput.js' @@ -7,12 +8,9 @@ import { SessionBar } from './components/SessionBar.js' import { useTeamsTheme } from './hooks/useTeamsTheme.js' import { useTeamsUser } from './hooks/useTeamsUser.js' import { useSessionMessages } from './hooks/useSessionMessages.js' -import type { Session, TranscriptMessage } from '../shared/types.js' -import { - addParticipantErrorCodeFromStatus, - isAddParticipantError, - toAddParticipantError, -} from '../shared/addParticipantErrors.js' +import { useSessions } from './hooks/useSessions.js' +import type { TranscriptMessage } from '../shared/types.js' +import { toAddParticipantError } from '../shared/addParticipantErrors.js' import { apiFetch } from './utils/apiFetch.js' declare const __SPEECH_PROVIDER__: string @@ -27,6 +25,30 @@ export interface Message { userId?: string } +function toUiMessages(messages: TranscriptMessage[]): Message[] { + return messages.map(message => ({ + id: message.id, + role: message.role, + text: message.text, + source: message.source ?? 'text', + timestamp: new Date(message.timestamp).getTime(), + displayName: message.displayName, + userId: message.userId, + })) +} + +function toTranscriptMessage(message: Message): TranscriptMessage { + return { + id: message.id, + role: message.role, + text: message.text, + timestamp: new Date(message.timestamp).toISOString(), + source: message.source, + displayName: message.displayName, + userId: message.userId, + } +} + const useStyles = makeStyles({ root: { display: 'flex', @@ -46,6 +68,19 @@ const useStyles = makeStyles({ borderTop: `1px solid ${tokens.colorNeutralStroke2}`, backgroundColor: tokens.colorNeutralBackground1, }, + continuityBar: { + display: 'flex', + alignItems: 'center', + gap: tokens.spacingHorizontalS, + padding: `${tokens.spacingVerticalXS} ${tokens.spacingHorizontalL}`, + borderBottom: `1px solid ${tokens.colorNeutralStroke2}`, + backgroundColor: tokens.colorNeutralBackground2, + color: tokens.colorNeutralForeground2, + fontSize: tokens.fontSizeBase200, + }, + continuityStatus: { + flexGrow: 1, + }, }) export function App() { @@ -57,46 +92,34 @@ export function App() { // Real Teams identity const currentUser = useTeamsUser() - const [sessions, setSessions] = useState([]) - const [activeSessionId, setActiveSessionId] = useState(null) - - // Fetch sessions when user resolves - useEffect(() => { - if (!currentUser) return - - // If we're in a Teams chat, check for a session linked to this chatId first - const chatIdParam = currentUser.chatId ? `?chatId=${encodeURIComponent(currentUser.chatId)}` : '' - apiFetch(`/api/sessions${chatIdParam}`) - .then(r => { if (!r.ok) throw new Error(`HTTP ${r.status}`); return r.json() }) - .then((data: Session[]) => { - setSessions(data) - const active = data.find(s => s.status === 'active') - setActiveSessionId(active?.id ?? data[0]?.id ?? null) - }) - .catch(console.error) - }, [currentUser]) + const { + sessions, + setSessions, + activeSessionId, + setActiveSessionId, + continuityState, + loadTranscript, + createSession, + addParticipant, + persistMessage, + persistSession, + clearLocalData, + } = useSessions(currentUser) - // Load transcript when active session changes + // Render the local transcript before replacing it with the reconciled result. useEffect(() => { if (!activeSessionId) { setMessages([]) return } - apiFetch(`/api/transcript/${activeSessionId}`) - .then(r => { if (!r.ok) throw new Error(`HTTP ${r.status}`); return r.json() }) - .then((data: TranscriptMessage[]) => { - setMessages(data.map(m => ({ - id: m.id, - role: m.role, - text: m.text, - source: m.source ?? 'text', - timestamp: new Date(m.timestamp).getTime(), - displayName: m.displayName, - userId: m.userId, - }))) - }) - .catch(console.error) - }, [activeSessionId, currentUser]) + let cancelled = false + void loadTranscript(activeSessionId, local => { + if (!cancelled) setMessages(toUiMessages(local)) + }).then(reconciled => { + if (!cancelled) setMessages(toUiMessages(reconciled)) + }) + return () => { cancelled = true } + }, [activeSessionId, loadTranscript]) // Track pending dispatches for voice loading indicator. We use a Set of // turnIds (rather than a bare counter) so that when an assistant SSE @@ -121,6 +144,7 @@ export function App() { useSessionMessages({ sessionId: activeSessionId, onMessage: useCallback((msg: TranscriptMessage) => { + if (activeSessionId) persistMessage(activeSessionId, msg) // Skip our own user messages — they're already in the list from sendMessage // Exception: in voicelive mode, voice user messages come only via SSE (bridge writes them) if (msg.role === 'user' && msg.userId === currentUser?.userId) { @@ -175,7 +199,7 @@ export function App() { userId: msg.userId, }] }) - }, [currentUser?.userId]), + }, [activeSessionId, currentUser?.userId, persistMessage]), }) // Auto-scroll @@ -185,60 +209,19 @@ export function App() { const handleNewSession = useCallback(async (): Promise => { if (!currentUser) return null - try { - const resp = await apiFetch('/api/sessions', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chatId: currentUser.chatId }), - }) - const session: Session = await resp.json() - setSessions(prev => [session, ...prev]) - setActiveSessionId(session.id) - setMessages([]) - return session.id - } catch (err) { - console.error('Failed to create session:', err) - return null - } - }, [currentUser]) + const session = await createSession(currentUser.chatId) + if (!session) return null + setMessages([]) + return session.id + }, [createSession, currentUser]) const handleAddParticipant = useCallback(async (userId: string, displayName: string): Promise => { if (!activeSessionId) { throw toAddParticipantError('NO_ACTIVE_SESSION', 'No active session') } - try { - const resp = await apiFetch(`/api/sessions/${activeSessionId}/participants`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ userId, displayName }), - }) - - if (!resp.ok) { - let serverMessage = 'Failed to add participant' - try { - const body = await resp.json() as { error?: string } - if (typeof body?.error === 'string' && body.error.trim().length > 0) { - serverMessage = body.error - } - } catch { - // non-JSON response, keep default message - } - - const code = addParticipantErrorCodeFromStatus(resp.status) - - throw toAddParticipantError(code, serverMessage, resp.status) - } - - const updated: Session = await resp.json() - setSessions(prev => prev.map(s => s.id === updated.id ? updated : s)) - } catch (err) { - if (isAddParticipantError(err)) { - throw err - } - throw toAddParticipantError('NETWORK', 'Network error while adding participant') - } - }, [activeSessionId, currentUser]) + await addParticipant(activeSessionId, userId, displayName) + }, [activeSessionId, addParticipant]) const sendMessage = useCallback(async (text: string, source: 'voice' | 'text') => { const trimmed = text.trim() @@ -251,20 +234,9 @@ export function App() { let sessionId = activeSessionId if (!sessionId) { if (!currentUser) return - try { - const resp = await apiFetch('/api/sessions', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ chatId: currentUser.chatId }), - }) - const session: Session = await resp.json() - setSessions(prev => [session, ...prev]) - setActiveSessionId(session.id) - sessionId = session.id - } catch (err) { - console.error('Failed to create session:', err) - return - } + const session = await createSession(currentUser.chatId) + if (!session) return + sessionId = session.id } const userMessageId = crypto.randomUUID() @@ -278,13 +250,20 @@ export function App() { userId: currentUser?.userId, } setMessages(prev => [...prev, userMessage]) + persistMessage(sessionId, toTranscriptMessage(userMessage)) setIsLoading(true) try { const resp = await apiFetch('/api/chat', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text: trimmed, sessionId, source, chatId: currentUser?.chatId }), + body: JSON.stringify({ + text: trimmed, + sessionId, + source, + chatId: currentUser?.chatId, + messageId: userMessageId, + }), }) if (!resp.ok) throw new Error(`Server error: ${resp.status}`) @@ -293,9 +272,12 @@ export function App() { // Update session title if auto-generated by server if (data.title) { - setSessions(prev => prev.map(s => - s.id === sessionId ? { ...s, title: data.title } : s - )) + setSessions(prev => prev.map(session => { + if (session.id !== sessionId) return session + const updated = { ...session, title: data.title } + persistSession(updated) + return updated + })) } if (data.text) { @@ -310,6 +292,7 @@ export function App() { if (prev.some(m => m.id === agentMessage.id)) return prev return [...prev, agentMessage] }) + persistMessage(sessionId, toTranscriptMessage(agentMessage)) setIsLoading(false) } // For async backends (DirectLine, Copilot Studio) isLoading stays true @@ -325,7 +308,14 @@ export function App() { setMessages(prev => [...prev, errorMessage]) setIsLoading(false) } - }, [activeSessionId, isLoading, currentUser]) + }, [activeSessionId, createSession, currentUser, isLoading, persistMessage, persistSession, setSessions]) + + const continuityMessage = { + loading: 'Device-local continuity: loading cache.', + 'local-fallback': 'Device-local continuity: showing cached data while server reconciliation is unavailable.', + 'server-reconciled': 'Device-local continuity: reconciled with the server.', + cleared: 'Device-local continuity: device cache cleared. Server sessions were not deleted.', + }[continuityState] const ensureSession = useCallback(async (): Promise => { if (activeSessionId) return activeSessionId @@ -342,6 +332,18 @@ export function App() { onNewSession={handleNewSession} onAddParticipant={handleAddParticipant} /> +
+ {continuityMessage} + +
sendMessage(text, 'text')} />
sendMessage(text, 'voice')} disabled={isLoading} sessionId={activeSessionId} onEnsureSession={ensureSession} onDispatch={handleVoiceDispatch} onDispatchError={handleVoiceDispatchError} /> diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/audio/pcmPlayback.test.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/audio/pcmPlayback.test.ts new file mode 100644 index 00000000..ca68a8e2 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/audio/pcmPlayback.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from 'vitest' + +import { + calculatePlaybackSchedule, + clearPlaybackSources, + decodeBase64Pcm16, +} from './pcmPlayback.js' + +function encodeBytes(bytes: number[]): string { + return Buffer.from(bytes).toString('base64') +} + +describe('decodeBase64Pcm16', () => { + it('decodes signed little-endian PCM16 boundary values', () => { + const samples = decodeBase64Pcm16(encodeBytes([0x00, 0x80, 0x00, 0x00, 0xff, 0x7f])) + + expect(Array.from(samples)).toEqual([-1, 0, 1]) + }) + + it('scales negative and positive samples independently', () => { + const samples = decodeBase64Pcm16(encodeBytes([0x00, 0xc0, 0x00, 0x40])) + + expect(samples[0]).toBe(-0.5) + expect(samples[1]).toBeCloseTo(16384 / 32767) + }) + + it('ignores an incomplete trailing byte', () => { + const samples = decodeBase64Pcm16(encodeBytes([0x01, 0x00, 0xff])) + + expect(samples).toHaveLength(1) + expect(samples[0]).toBeCloseTo(1 / 32767) + }) +}) + +describe('calculatePlaybackSchedule', () => { + it('starts immediately when no prior audio remains scheduled', () => { + expect(calculatePlaybackSchedule(10, 8, 24_000, 24_000)).toEqual({ + startTime: 10, + endTime: 11, + }) + }) + + it('queues audio after the prior scheduled segment', () => { + expect(calculatePlaybackSchedule(10, 12, 12_000, 24_000)).toEqual({ + startTime: 12, + endTime: 12.5, + }) + }) +}) + +describe('clearPlaybackSources', () => { + it('stops, disconnects, and clears every source before resetting the schedule', () => { + const first = { stop: vi.fn(), disconnect: vi.fn() } + const second = { + stop: vi.fn(() => { throw new Error('already stopped') }), + disconnect: vi.fn(() => { throw new Error('already disconnected') }), + } + const sources = new Set([first, second]) + const resetSchedule = vi.fn() + + clearPlaybackSources(sources, resetSchedule) + + expect(first.stop).toHaveBeenCalledOnce() + expect(first.disconnect).toHaveBeenCalledOnce() + expect(second.stop).toHaveBeenCalledOnce() + expect(second.disconnect).toHaveBeenCalledOnce() + expect(sources.size).toBe(0) + expect(resetSchedule).toHaveBeenCalledOnce() + }) +}) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/audio/pcmPlayback.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/audio/pcmPlayback.ts new file mode 100644 index 00000000..537f88e6 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/audio/pcmPlayback.ts @@ -0,0 +1,60 @@ +export interface PlaybackSchedule { + startTime: number + endTime: number +} + +export interface PlaybackSource { + stop: () => void + disconnect: () => void +} + +export function clearPlaybackSources( + sources: Set, + resetSchedule: () => void, +): void { + const scheduledSources = Array.from(sources) + sources.clear() + + for (const source of scheduledSources) { + try { + source.stop() + } catch { + // The source may already have ended or been stopped. + } + + try { + source.disconnect() + } catch { + // Disconnect is best-effort during repeated teardown. + } + } + + resetSchedule() +} + +export function decodeBase64Pcm16(base64: string): Float32Array { + const binary = atob(base64) + const sampleCount = Math.floor(binary.length / 2) + const samples = new Float32Array(sampleCount) + + for (let index = 0; index < sampleCount; index += 1) { + const offset = index * 2 + const value = (binary.charCodeAt(offset) | (binary.charCodeAt(offset + 1) << 8)) << 16 >> 16 + samples[index] = value < 0 ? value / 32768 : value / 32767 + } + + return samples +} + +export function calculatePlaybackSchedule( + currentTime: number, + priorScheduledEnd: number, + sampleCount: number, + sampleRate: number, +): PlaybackSchedule { + const startTime = Math.max(currentTime, priorScheduledEnd) + return { + startTime, + endTime: startTime + sampleCount / sampleRate, + } +} \ No newline at end of file diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useSessions.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useSessions.ts new file mode 100644 index 00000000..8f36429c --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useSessions.ts @@ -0,0 +1,275 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import type { Dispatch, SetStateAction } from 'react' +import type { + DeviceContinuityState, + Session, + TranscriptMessage, + UserContext, +} from '../../shared/types.js' +import { + addParticipantErrorCodeFromStatus, + isAddParticipantError, + toAddParticipantError, +} from '../../shared/addParticipantErrors.js' +import { cleanLogging } from '../../shared/logging.js' +import { IndexedDbSessionStore } from '../storage/indexedDbSessionStore.js' +import { apiFetch } from '../utils/apiFetch.js' + +function timestampToMilliseconds(value: unknown): number { + if (typeof value === 'number') return value + if (typeof value !== 'string') return 0 + const parsed = Date.parse(value) + if (!Number.isNaN(parsed)) return parsed + const numeric = Number(value) + return Number.isFinite(numeric) ? numeric : 0 +} + +function mergeSessions(local: Session[], server: Session[]): Session[] { + const sessions = new Map(local.map(session => [session.id, session])) + for (const session of server) sessions.set(session.id, session) + return [...sessions.values()].sort((left, right) => + timestampToMilliseconds(right.lastActivityAt) - timestampToMilliseconds(left.lastActivityAt), + ) +} + +function mergeMessages(local: TranscriptMessage[], server: TranscriptMessage[]): TranscriptMessage[] { + const messages = new Map(local.map(message => [message.id, message])) + for (const message of server) messages.set(message.id, message) + return [...messages.values()].sort((left, right) => + timestampToMilliseconds(left.timestamp) - timestampToMilliseconds(right.timestamp), + ) +} + +export interface UseSessionsResult { + sessions: Session[] + setSessions: Dispatch> + activeSessionId: string | null + setActiveSessionId: Dispatch> + continuityState: DeviceContinuityState + loadTranscript: ( + sessionId: string, + onLocal: (messages: TranscriptMessage[]) => void, + ) => Promise + createSession: (chatId?: string | null) => Promise + addParticipant: (sessionId: string, userId: string, displayName: string) => Promise + persistMessage: (sessionId: string, message: TranscriptMessage) => void + persistSession: (session: Session) => void + deleteLocalSession: (sessionId: string) => Promise + clearLocalData: () => Promise +} + +export function useSessions(currentUser: UserContext | null): UseSessionsResult { + const [sessions, setSessions] = useState([]) + const [activeSessionId, setActiveSessionId] = useState(null) + const [continuityState, setContinuityState] = useState('loading') + const localCacheReadyRef = useRef(false) + const ownerMaintenanceRef = useRef>(Promise.resolve()) + const storeRef = useRef(null) + if (!storeRef.current) storeRef.current = new IndexedDbSessionStore() + const store = storeRef.current + + const persistSession = useCallback((session: Session) => { + if (!localCacheReadyRef.current) return + void store.saveSession(session).catch(error => { + cleanLogging.Warn('device-continuity', 'Session write failed', error) + }) + }, [store]) + + const persistMessage = useCallback((sessionId: string, message: TranscriptMessage) => { + if (!localCacheReadyRef.current) return + void store.saveMessage(sessionId, message).catch(error => { + cleanLogging.Warn('device-continuity', 'Message write failed', error) + }) + }, [store]) + + useEffect(() => { + localCacheReadyRef.current = false + setSessions([]) + setActiveSessionId(null) + setContinuityState('loading') + if (!currentUser) return + + let cancelled = false + void (async () => { + let localCacheReady = false + const ownerMaintenance = ownerMaintenanceRef.current.then(async () => { + await store.assertOwner(currentUser.userId) + await store.purgeExpired() + }) + ownerMaintenanceRef.current = ownerMaintenance.catch(() => {}) + try { + await ownerMaintenance + if (cancelled) return + localCacheReady = true + localCacheReadyRef.current = true + } catch (error) { + cleanLogging.Warn('device-continuity', 'Local maintenance failed', error) + } + if (cancelled) return + + let local: Session[] = [] + if (localCacheReady) { + try { + local = await store.list(currentUser.userId) + } catch (error) { + cleanLogging.Warn('device-continuity', 'Local session hydration failed', error) + } + } + if (cancelled) return + setSessions(local) + setActiveSessionId(local.find(session => session.status === 'active')?.id ?? local[0]?.id ?? null) + setContinuityState('local-fallback') + + const chatIdQuery = currentUser.chatId + ? `?chatId=${encodeURIComponent(currentUser.chatId)}` + : '' + try { + const response = await apiFetch(`/api/sessions${chatIdQuery}`) + if (!response.ok) throw new Error(`HTTP ${response.status}`) + const server = await response.json() as Session[] + if (cancelled) return + const serverIds = new Set(server.map(session => session.id)) + const resumable = local.filter(session => + !serverIds.has(session.id) && + Boolean(session.resumeToken) && + Boolean(session.threadId || session.conversationId), + ) + const resumed: Session[] = [] + for (const session of resumable) { + if (cancelled) return + try { + const resumeResponse = await apiFetch(`/api/sessions/${session.id}/resume`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ session }), + }) + if (!resumeResponse.ok) continue + const adopted = await resumeResponse.json() as Session + if (cancelled) return + resumed.push(adopted) + persistSession(adopted) + } catch (error) { + cleanLogging.Warn('device-continuity', 'Cached session resume failed', error) + } + } + if (cancelled) return + const reconciledServer = mergeSessions(server, resumed) + const reconciled = mergeSessions(local, reconciledServer) + setSessions(reconciled) + const serverActive = reconciledServer.find(session => session.status === 'active') + setActiveSessionId(previous => serverActive?.id ?? previous ?? reconciled[0]?.id ?? null) + for (const session of server) persistSession(session) + setContinuityState('server-reconciled') + } catch (error) { + cleanLogging.Error('device-continuity', 'Server session reconciliation failed', error) + } + })() + + return () => { cancelled = true } + }, [currentUser, persistSession, store]) + + const loadTranscript = useCallback(async ( + sessionId: string, + onLocal: (messages: TranscriptMessage[]) => void, + ): Promise => { + let local: TranscriptMessage[] = [] + try { + local = mergeMessages(await store.getMessages(sessionId), []) + } catch (error) { + cleanLogging.Warn('device-continuity', 'Local transcript hydration failed', error) + } + onLocal(local) + + try { + const response = await apiFetch(`/api/transcript/${sessionId}`) + if (!response.ok) throw new Error(`HTTP ${response.status}`) + const server = await response.json() as TranscriptMessage[] + for (const message of server) persistMessage(sessionId, message) + setContinuityState('server-reconciled') + return mergeMessages(local, server) + } catch (error) { + cleanLogging.Warn('device-continuity', 'Server transcript unavailable; using local cache', error) + setContinuityState('local-fallback') + return local + } + }, [persistMessage, store]) + + const createSession = useCallback(async (chatId?: string | null): Promise => { + try { + const response = await apiFetch('/api/sessions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ chatId: chatId ?? null }), + }) + if (!response.ok) throw new Error(`HTTP ${response.status}`) + const session = await response.json() as Session + persistSession(session) + setSessions(previous => [session, ...previous.filter(item => item.id !== session.id)]) + setActiveSessionId(session.id) + setContinuityState('server-reconciled') + return session + } catch (error) { + cleanLogging.Error('device-continuity', 'Session creation failed', error) + return null + } + }, [persistSession]) + + const addParticipant = useCallback(async ( + sessionId: string, + userId: string, + displayName: string, + ): Promise => { + try { + const response = await apiFetch(`/api/sessions/${sessionId}/participants`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ userId, displayName }), + }) + if (!response.ok) { + let message = 'Failed to add participant' + try { + const body = await response.json() as { error?: string } + if (typeof body.error === 'string' && body.error.trim().length > 0) message = body.error + } catch { + // Keep the default for non-JSON responses. + } + throw toAddParticipantError( + addParticipantErrorCodeFromStatus(response.status), + message, + response.status, + ) + } + const updated = await response.json() as Session + persistSession(updated) + setSessions(previous => previous.map(session => session.id === updated.id ? updated : session)) + return updated + } catch (error) { + if (isAddParticipantError(error)) throw error + throw toAddParticipantError('NETWORK', 'Network error while adding participant') + } + }, [persistSession]) + + const deleteLocalSession = useCallback(async (sessionId: string): Promise => { + await store.deleteSession(sessionId) + }, [store]) + + const clearLocalData = useCallback(async (): Promise => { + await store.clearAll() + setContinuityState('cleared') + }, [store]) + + return { + sessions, + setSessions, + activeSessionId, + setActiveSessionId, + continuityState, + loadTranscript, + createSession, + addParticipant, + persistMessage, + persistSession, + deleteLocalSession, + clearLocalData, + } +} diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useTeamsUser.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useTeamsUser.ts index 867ff29d..93937b1a 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useTeamsUser.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useTeamsUser.ts @@ -36,8 +36,8 @@ export function useTeamsUser() { }).catch(() => { // Not running inside Teams — use a local fallback setUser({ - userId: 'local-user', - displayName: 'Local User', + userId: 'dev-user', + displayName: 'Dev User', }) }) }, []) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useVoiceLive.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useVoiceLive.ts index f447e8a4..6334f910 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useVoiceLive.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/hooks/useVoiceLive.ts @@ -1,5 +1,11 @@ import { useState, useRef, useCallback, useEffect } from 'react' import type { UseSpeechRecognitionResult } from './useSpeechRecognition.js' +import { cleanLogging } from '../../shared/logging.js' +import { + calculatePlaybackSchedule, + clearPlaybackSources, + decodeBase64Pcm16, +} from '../audio/pcmPlayback.js' /** * Browser-side debug gate. Audio device labels and IDs are PII-adjacent @@ -47,6 +53,8 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition const audioCtxRef = useRef(null) const streamRef = useRef(null) const workletRef = useRef(null) + const nextPlaybackTimeRef = useRef(0) + const playbackSourcesRef = useRef(new Set()) const onFinalResultRef = useRef(options.onFinalResult) onFinalResultRef.current = options.onFinalResult @@ -63,6 +71,12 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition typeof WebSocket !== 'undefined' && typeof AudioWorkletNode !== 'undefined' + const clearPlayback = useCallback(() => { + clearPlaybackSources(playbackSourcesRef.current, () => { + nextPlaybackTimeRef.current = 0 + }) + }, []) + const cleanup = useCallback(() => { const worklet = workletRef.current workletRef.current = null @@ -72,9 +86,11 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition streamRef.current = null stream?.getTracks().forEach(t => t.stop()) + clearPlayback() + const audioCtx = audioCtxRef.current audioCtxRef.current = null - if (audioCtx?.state !== 'closed') { + if (audioCtx && audioCtx.state !== 'closed') { audioCtx.close().catch(() => {}) } @@ -83,7 +99,7 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition if (ws?.readyState === WebSocket.OPEN || ws?.readyState === WebSocket.CONNECTING) { ws.close() } - }, []) + }, [clearPlayback]) // Cleanup on unmount useEffect(() => cleanup, [cleanup]) @@ -118,7 +134,10 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition const devices = await navigator.mediaDevices.enumerateDevices() const audioInputs = devices.filter(d => d.kind === 'audioinput') if (debugEnabled()) { - console.log('[VoiceLive] Audio input devices:', audioInputs.map(d => `${d.label} (${d.deviceId.slice(0, 8)})`)) + cleanLogging.Log('VoiceLive', 'Audio input devices', audioInputs.map(device => ({ + label: device.label, + deviceIdPrefix: device.deviceId.slice(0, 8), + }))) } // Prefer a non-default, non-communications device if one exists @@ -138,7 +157,7 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition if (preferred) { audioConstraints.deviceId = { exact: preferred.deviceId } if (debugEnabled()) { - console.log(`[VoiceLive] Using mic: ${preferred.label}`) + cleanLogging.Log('VoiceLive', 'Using microphone', { label: preferred.label }) } } @@ -151,7 +170,7 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition // exact deviceId to let the browser pick the current default input. if (err instanceof DOMException && err.name === 'OverconstrainedError') { if (debugEnabled()) { - console.warn('[VoiceLive] Preferred mic unavailable, retrying with default input') + cleanLogging.Warn('VoiceLive', 'Preferred microphone unavailable; retrying with default input') } stream = await navigator.mediaDevices.getUserMedia({ audio: { @@ -237,6 +256,73 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition setTranscript('Connecting...') } + function handleVoiceLiveEvent(event: { type?: string; transcript?: string; delta?: string }) { + switch (event.type) { + case 'response.audio.delta': { + if (!event.delta) break + + const audioCtx = audioCtxRef.current + if (!audioCtx) break + + const samples = decodeBase64Pcm16(event.delta) + if (samples.length === 0) break + + const sampleRate = 24000 + const schedule = calculatePlaybackSchedule( + audioCtx.currentTime, + nextPlaybackTimeRef.current, + samples.length, + sampleRate, + ) + const buffer = audioCtx.createBuffer(1, samples.length, sampleRate) + buffer.copyToChannel(samples, 0) + + const source = audioCtx.createBufferSource() + source.buffer = buffer + source.connect(audioCtx.destination) + playbackSourcesRef.current.add(source) + source.addEventListener('ended', () => { + playbackSourcesRef.current.delete(source) + try { + source.disconnect() + } catch { + // The source may already have been disconnected by cleanup. + } + }, { once: true }) + try { + source.start(schedule.startTime) + } catch { + playbackSourcesRef.current.delete(source) + try { + source.disconnect() + } catch { + // Disconnect is best-effort after a failed start. + } + break + } + nextPlaybackTimeRef.current = schedule.endTime + break + } + + case 'input_audio_buffer.speech_started': + clearPlayback() + break + + case 'conversation.item.input_audio_transcription.delta': + if (event.delta) setTranscript(prev => prev + event.delta) + break + + case 'conversation.item.input_audio_transcription.completed': + setTranscript('') + if (event.transcript) onFinalResultRef.current(event.transcript) + break + + case 'error': + setError((event as { error?: { message?: string } }).error?.message ?? 'Voice Live error') + break + } + } + // Only start sending audio after upstream is ready (bridge sends session.ready) ws.onmessage = (ev) => { try { @@ -308,7 +394,7 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition setError(message) cleanup() } - }, [isListening, cleanup]) + }, [isListening, cleanup, clearPlayback]) const stopListening = useCallback(() => { cleanup() @@ -324,22 +410,5 @@ export function useVoiceLive(options: UseVoiceLiveOptions): UseSpeechRecognition } }, [isListening, startListening, stopListening]) - function handleVoiceLiveEvent(event: { type?: string; transcript?: string; delta?: string }) { - switch (event.type) { - case 'conversation.item.input_audio_transcription.delta': - if (event.delta) setTranscript(prev => prev + event.delta) - break - - case 'conversation.item.input_audio_transcription.completed': - setTranscript('') - if (event.transcript) onFinalResultRef.current(event.transcript) - break - - case 'error': - setError((event as { error?: { message?: string } }).error?.message ?? 'Voice Live error') - break - } - } - return { isListening, isSupported, transcript, error, startListening, stopListening, toggleListening } } diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/storage/indexedDbSessionStore.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/storage/indexedDbSessionStore.ts new file mode 100644 index 00000000..80b1cd08 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/storage/indexedDbSessionStore.ts @@ -0,0 +1,235 @@ +import type { Session, TranscriptMessage } from '../../shared/types.js' +import { cleanLogging } from '../../shared/logging.js' + +const DB_NAME = 'chat-with-your-factory-continuity' +const DB_VERSION = 1 +const SESSIONS_STORE = 'sessions' +const MESSAGES_STORE = 'messages' +const META_STORE = 'meta' +const OWNER_KEY = 'owner' +const DEFAULT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000 +const MAX_CLOCK_SKEW_MS = 5 * 60 * 1000 + +interface OwnerRecord { + key: string + userId: string +} + +interface StoredMessage extends TranscriptMessage { + sessionId: string +} + +function requestResult(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result) + request.onerror = () => reject(request.error) + }) +} + +function transactionDone(transaction: IDBTransaction): Promise { + return new Promise((resolve, reject) => { + transaction.oncomplete = () => resolve() + transaction.onerror = () => reject(transaction.error) + transaction.onabort = () => reject(transaction.error) + }) +} + +function timestampToMilliseconds(value: unknown): number { + if (typeof value === 'number') return value + if (typeof value !== 'string' || value.trim().length === 0) return Number.NaN + const numeric = Number(value) + return Number.isNaN(numeric) ? Date.parse(value) : numeric +} + +function backfillParticipants(store: IDBObjectStore): void { + const cursorRequest = store.openCursor() + cursorRequest.onsuccess = () => { + const cursor = cursorRequest.result + if (!cursor) return + const record = cursor.value as Partial + const participants = Array.isArray(record.participants) ? record.participants : [] + if (typeof record.userId === 'string' && !participants.includes(record.userId)) { + const updateRequest = cursor.update({ ...record, participants: [...participants, record.userId] }) + updateRequest.onerror = event => { + cleanLogging.Warn('device-continuity', 'Participant repair failed', updateRequest.error) + event.preventDefault() + } + } else if (!Array.isArray(record.participants)) { + const updateRequest = cursor.update({ ...record, participants }) + updateRequest.onerror = event => { + cleanLogging.Warn('device-continuity', 'Participant repair failed', updateRequest.error) + event.preventDefault() + } + } + cursor.continue() + } +} + +function messageRange(sessionId: string): IDBKeyRange { + return IDBKeyRange.bound([sessionId, ''], [sessionId, '\uffff']) +} + +export class IndexedDbSessionStore { + private databasePromise: Promise | undefined + + private open(): Promise { + if (!this.databasePromise) { + this.databasePromise = new Promise((resolve, reject) => { + const request = indexedDB.open(DB_NAME, DB_VERSION) + request.onupgradeneeded = () => { + const database = request.result + const transaction = request.transaction as IDBTransaction + const sessions = database.objectStoreNames.contains(SESSIONS_STORE) + ? transaction.objectStore(SESSIONS_STORE) + : database.createObjectStore(SESSIONS_STORE, { keyPath: 'id' }) + + if (!sessions.indexNames.contains('participants')) { + sessions.createIndex('participants', 'participants', { multiEntry: true }) + } + if (sessions.indexNames.contains('userId')) { + sessions.deleteIndex('userId') + } + backfillParticipants(sessions) + + const messages = database.objectStoreNames.contains(MESSAGES_STORE) + ? transaction.objectStore(MESSAGES_STORE) + : database.createObjectStore(MESSAGES_STORE, { keyPath: ['sessionId', 'id'] }) + if (!messages.indexNames.contains('sessionId')) { + messages.createIndex('sessionId', 'sessionId') + } + if (!database.objectStoreNames.contains(META_STORE)) { + database.createObjectStore(META_STORE, { keyPath: 'key' }) + } + } + request.onsuccess = () => { + request.result.onversionchange = () => request.result.close() + resolve(request.result) + } + request.onerror = () => reject(request.error) + request.onblocked = () => reject(new Error('Device-local continuity database upgrade is blocked')) + }) + } + return this.databasePromise + } + + async assertOwner(userId: string): Promise { + const database = await this.open() + const readTransaction = database.transaction(META_STORE, 'readonly') + const owner = await requestResult( + readTransaction.objectStore(META_STORE).get(OWNER_KEY), + ) as OwnerRecord | undefined + + if (owner?.userId === userId) return false + const ownerChanged = owner !== undefined + if (ownerChanged) await this.clearAll() + + const writeTransaction = database.transaction(META_STORE, 'readwrite') + const completion = transactionDone(writeTransaction) + writeTransaction.objectStore(META_STORE).put({ key: OWNER_KEY, userId } satisfies OwnerRecord) + await completion + return ownerChanged + } + + async list(userId: string): Promise { + const database = await this.open() + let sessions: Session[] + try { + const transaction = database.transaction(SESSIONS_STORE, 'readonly') + sessions = await requestResult( + transaction.objectStore(SESSIONS_STORE).index('participants').getAll(userId), + ) as Session[] + } catch (error) { + cleanLogging.Warn('device-continuity', 'Participant index unavailable; scanning cache', error) + const transaction = database.transaction(SESSIONS_STORE, 'readonly') + const all = await requestResult(transaction.objectStore(SESSIONS_STORE).getAll()) as Session[] + sessions = all.filter(session => + session.userId === userId || session.participants?.includes(userId), + ) + } + return sessions + .filter(session => session && typeof session.id === 'string') + .sort((left, right) => + timestampToMilliseconds(right.lastActivityAt) - timestampToMilliseconds(left.lastActivityAt), + ) + } + + async saveSession(session: Session): Promise { + const database = await this.open() + const transaction = database.transaction(SESSIONS_STORE, 'readwrite') + const completion = transactionDone(transaction) + transaction.objectStore(SESSIONS_STORE).put(session) + await completion + } + + async deleteSession(sessionId: string): Promise { + const database = await this.open() + const transaction = database.transaction([SESSIONS_STORE, MESSAGES_STORE], 'readwrite') + const completion = transactionDone(transaction) + transaction.objectStore(SESSIONS_STORE).delete(sessionId) + transaction.objectStore(MESSAGES_STORE).delete(messageRange(sessionId)) + await completion + } + + async getMessages(sessionId: string): Promise { + const database = await this.open() + let records: StoredMessage[] + try { + const transaction = database.transaction(MESSAGES_STORE, 'readonly') + records = await requestResult( + transaction.objectStore(MESSAGES_STORE).index('sessionId').getAll(sessionId), + ) as StoredMessage[] + } catch (error) { + cleanLogging.Warn('device-continuity', 'Message index unavailable; scanning cache', error) + const transaction = database.transaction(MESSAGES_STORE, 'readonly') + const all = await requestResult(transaction.objectStore(MESSAGES_STORE).getAll()) as StoredMessage[] + records = all.filter(record => record.sessionId === sessionId) + } + return records.map(record => { + const { sessionId, ...message } = record + void sessionId + return message + }) + } + + async saveMessage(sessionId: string, message: TranscriptMessage): Promise { + const database = await this.open() + const transaction = database.transaction(MESSAGES_STORE, 'readwrite') + const completion = transactionDone(transaction) + transaction.objectStore(MESSAGES_STORE).put({ ...message, sessionId } satisfies StoredMessage) + await completion + } + + async clearAll(): Promise { + const database = await this.open() + const transaction = database.transaction([SESSIONS_STORE, MESSAGES_STORE], 'readwrite') + const completion = transactionDone(transaction) + transaction.objectStore(SESSIONS_STORE).clear() + transaction.objectStore(MESSAGES_STORE).clear() + await completion + } + + async purgeExpired(retentionMs: number = DEFAULT_RETENTION_MS): Promise { + if (!Number.isFinite(retentionMs) || retentionMs < 0) { + throw new RangeError('Retention duration must be finite and non-negative') + } + + const now = Date.now() + const cutoff = now - retentionMs + const futureLimit = now + MAX_CLOCK_SKEW_MS + const database = await this.open() + const transaction = database.transaction([SESSIONS_STORE, MESSAGES_STORE], 'readwrite') + const completion = transactionDone(transaction) + const sessionsStore = transaction.objectStore(SESSIONS_STORE) + const messagesStore = transaction.objectStore(MESSAGES_STORE) + const sessions = await requestResult(sessionsStore.getAll()) as Session[] + + for (const session of sessions) { + const lastActivity = timestampToMilliseconds(session.lastActivityAt) + if (!Number.isFinite(lastActivity) || lastActivity < cutoff || lastActivity > futureLimit) { + sessionsStore.delete(session.id) + messagesStore.delete(messageRange(session.id)) + } + } + await completion + } +} diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/utils/apiFetch.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/utils/apiFetch.ts index 1e7fbe08..f43cdc57 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/utils/apiFetch.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/client/utils/apiFetch.ts @@ -3,7 +3,11 @@ import { decodeJwtPayload } from './jwtPayload.js' let teamsContext: { userId: string; displayName: string } | null = null let contextPromise: Promise | null = null -let ssoToken: string | null = null +let cachedToken: { token: string; expMs: number } | null = null +let inFlightToken: Promise | null = null + +const TOKEN_REFRESH_BUFFER_MS = 120_000 +const MALFORMED_EXPIRY_RETRY_MS = 300_000 function isLocalFallbackHost(): boolean { const hostname = window.location.hostname.toLowerCase() @@ -19,19 +23,10 @@ function resolveTeamsContext(): Promise { if (!contextPromise) { contextPromise = microsoftTeams.app.initialize() .then(() => microsoftTeams.app.getContext()) - .then(async (ctx) => { - let name = ctx.user?.displayName || ctx.user?.userPrincipalName || '' - // Try to get the SSO token once during init - try { - ssoToken = await microsoftTeams.authentication.getAuthToken() - const payload = decodeJwtPayload(ssoToken) - if (typeof payload.name === 'string') name = payload.name - } catch { - // SSO unavailable — fall back to identity headers - } + .then((ctx) => { teamsContext = { userId: ctx.user?.id || '', - displayName: name, + displayName: ctx.user?.displayName || ctx.user?.userPrincipalName || '', } }) .catch(() => { /* Not in Teams */ }) @@ -39,13 +34,64 @@ function resolveTeamsContext(): Promise { return contextPromise } +function tokenExpMs(token: string): number | null { + try { + const exp = decodeJwtPayload(token).exp + if (typeof exp !== 'number' || !Number.isFinite(exp) || exp <= 0) return null + + const expMs = exp * 1000 + return Number.isFinite(expMs) && expMs > 0 ? expMs : null + } catch { + return null + } +} + +async function getFreshToken(): Promise { + if (cachedToken && Date.now() < cachedToken.expMs - TOKEN_REFRESH_BUFFER_MS) { + return cachedToken.token + } + + if (!inFlightToken) { + inFlightToken = (async () => { + try { + const token = await microsoftTeams.authentication.getAuthToken() + if (!token) { + cachedToken = null + return null + } + + const now = Date.now() + const parsedExpMs = tokenExpMs(token) + if (parsedExpMs !== null && parsedExpMs <= now) { + cachedToken = null + return null + } + + cachedToken = { + token, + expMs: parsedExpMs ?? now + MALFORMED_EXPIRY_RETRY_MS, + } + return token + } catch { + cachedToken = null + return null + } finally { + inFlightToken = null + } + })() + } + + return inFlightToken +} + export async function apiFetch(url: string, init?: RequestInit): Promise { const headers = new Headers(init?.headers) await resolveTeamsContext() - if (ssoToken) { - headers.set('Authorization', `Bearer ${ssoToken}`) + const token = teamsContext ? await getFreshToken() : null + if (token) { + headers.set('Authorization', `Bearer ${token}`) } else if (teamsContext && isLocalFallbackHost()) { // SSO unavailable — allow fallback identity headers only for local/devtunnel hosts. headers.set('x-user-id', teamsContext.userId) @@ -54,8 +100,8 @@ export async function apiFetch(url: string, init?: RequestInit): Promise { await resolveTeamsContext() - return ssoToken + return teamsContext ? getFreshToken() : null } diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/chatHandler.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/chatHandler.ts index d63f907f..640f3b05 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/chatHandler.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/chatHandler.ts @@ -1,6 +1,12 @@ import type { Request, Response } from 'express' import { isOutputOfType } from '@azure/ai-agents' -import type { SubmitToolOutputsAction, RequiredFunctionToolCall, ToolOutput } from '@azure/ai-agents' +import type { + RunStatus, + SubmitToolOutputsAction, + RequiredFunctionToolCall, + ToolOutput, +} from '@azure/ai-agents' +import { cleanLogging } from '../shared/logging.js' import * as directLine from './directLineClient.js' import { getAuthorizedSession } from './aclHelper.js' import { handleFactoryTool } from './factoryTool.js' @@ -11,7 +17,17 @@ const AGENT_BACKEND = process.env.AGENT_BACKEND || 'foundry' // Foundry run states that warrant continued polling. Module-scoped and frozen // so it isn't rebuilt per request and can't be mutated. -const POLL_STATUSES: ReadonlySet = new Set(['queued', 'in_progress', 'requires_action']) +const POLL_STATUSES: ReadonlySet = new Set([ + 'queued', + 'in_progress', + 'requires_action', + 'cancelling', +]) +const CANCELLABLE_STATUSES: ReadonlySet = new Set([ + 'queued', + 'in_progress', + 'requires_action', +]) // Hard cap on the run-polling loop. Express 5 has no request timeout, so an // upstream stall (quota pause, hung run) would otherwise hold the HTTP socket // and a worker open indefinitely. @@ -23,6 +39,8 @@ export interface DispatchContext { ssoToken?: string source?: 'voice' | 'text' | 'teams' skipUserMessage?: boolean + clientMessageId?: string + onAssistantCompleted?: (markdown: string) => void /** Optional correlation ID stamped onto the assistant reply broadcast. * Used by voicelive clients to match replies to the dispatch they * initiated, so a co-participant's reply doesn't clear this client's @@ -36,6 +54,27 @@ export interface DispatchResult { title?: string } +function publishAssistantMessage( + sessionId: string, + message: { + id: string + role: 'assistant' + text: string + timestamp: string + source: 'agent' + turnId?: string + }, + onAssistantCompleted?: (markdown: string) => void, +): void { + sessionStore.addMessage(sessionId, message) + sseRegistry.broadcast(sessionId, message) + try { + onAssistantCompleted?.(message.text) + } catch (error) { + cleanLogging.Warn('Chat', 'Assistant completion callback failed', error) + } +} + export async function dispatchChat( sessionId: string, text: string, @@ -51,7 +90,7 @@ export async function dispatchChat( if (!ctx.skipUserMessage) { const userMessage = { - id: crypto.randomUUID(), + id: ctx.clientMessageId ?? crypto.randomUUID(), role: 'user' as const, text, timestamp: new Date().toISOString(), @@ -106,8 +145,7 @@ export async function dispatchChat( source: 'agent' as const, turnId: ctx.turnId, } - sessionStore.addMessage(sessionId, message) - sseRegistry.broadcast(sessionId, message) + publishAssistantMessage(sessionId, message, ctx.onAssistantCompleted) lastAssistantText = activity.text lastAssistantId = message.id } @@ -130,57 +168,98 @@ export async function dispatchChat( let run = await agentsClient.runs.create(threadId, agentId) const pollDeadline = Date.now() + MAX_POLL_MS - while (POLL_STATUSES.has(run.status)) { - if (Date.now() > pollDeadline) { - throw new Error(`Agent run timed out after ${MAX_POLL_MS / 1000} s`) - } - await new Promise(r => setTimeout(r, 1000)) - run = await agentsClient.runs.get(threadId, run.id) + try { + while (POLL_STATUSES.has(run.status)) { + if (Date.now() > pollDeadline) { + throw new Error(`Agent run timed out after ${MAX_POLL_MS / 1000} s`) + } + await new Promise(r => setTimeout(r, 1000)) + run = await agentsClient.runs.get(threadId, run.id) - if ( - run.status === 'requires_action' && - run.requiredAction && - isOutputOfType(run.requiredAction, 'submit_tool_outputs') - ) { - const toolOutputs: ToolOutput[] = [] - for (const toolCall of run.requiredAction.submitToolOutputs.toolCalls) { - if (isOutputOfType(toolCall, 'function')) { - const args = toolCall.function.arguments - ? JSON.parse(toolCall.function.arguments) - : {} - let result: unknown - if (toolCall.function.name === 'query_factory_ontology') { - result = await handleFactoryTool(args) - } else { - result = { error: `unknown tool: ${toolCall.function.name}` } + if ( + run.status === 'requires_action' && + run.requiredAction && + isOutputOfType(run.requiredAction, 'submit_tool_outputs') + ) { + const toolOutputs: ToolOutput[] = [] + for (const toolCall of run.requiredAction.submitToolOutputs.toolCalls) { + if (isOutputOfType(toolCall, 'function')) { + let result: unknown + try { + const args = toolCall.function.arguments + ? JSON.parse(toolCall.function.arguments) + : {} + result = toolCall.function.name === 'query_factory_ontology' + ? await handleFactoryTool(args) + : { error: `unknown tool: ${toolCall.function.name}` } + } catch (error) { + cleanLogging.Warn('Chat', 'Agent tool execution failed', { + toolName: toolCall.function.name, + error, + }) + result = { + error: 'The factory ontology is temporarily unavailable. Confirm that the Fabric capacity is active and retry.', + } + } + toolOutputs.push({ toolCallId: toolCall.id, output: JSON.stringify(result ?? null) }) } - toolOutputs.push({ toolCallId: toolCall.id, output: JSON.stringify(result ?? null) }) + } + if (toolOutputs.length) { + await agentsClient.runs.submitToolOutputs(threadId, run.id, toolOutputs) } } - if (toolOutputs.length) { - await agentsClient.runs.submitToolOutputs(threadId, run.id, toolOutputs) + + } + } catch (error) { + if (CANCELLABLE_STATUSES.has(run.status)) { + try { + await agentsClient.runs.cancel(threadId, run.id) + } catch (cancelError) { + cleanLogging.Warn('Chat', 'Failed to cancel active agent run', cancelError) } } + throw error + } + if (run.status !== 'completed') { if (run.status === 'failed') { - throw new Error(`Agent run failed: ${run.lastError?.code} ${run.lastError?.message}`) + const code = String(run.lastError?.code ?? 'unknown').slice(0, 128) + const message = String(run.lastError?.message ?? 'No error details provided').slice(0, 500) + throw new Error(`Agent run failed: ${code} ${message}`) } + if (run.status === 'cancelled') throw new Error('Agent run was cancelled') + if (run.status === 'expired') throw new Error('Agent run expired') + throw new Error(`Agent run ended without completion: ${String(run.status).slice(0, 64)}`) } - const messages = agentsClient.messages.list(threadId, { order: 'desc', limit: 1 }) + const messages = agentsClient.messages.list(threadId, { + order: 'desc', + limit: 1, + runId: run.id, + }) - let responseText = 'No response from agent.' + let responseText: string | undefined for await (const msg of messages) { - if (msg.role === 'assistant') { + if ( + msg.role === 'assistant' && + msg.runId === run.id && + (msg.status === undefined || msg.status === 'completed') + ) { + const textBlocks: string[] = [] for (const block of msg.content) { if (block.type === 'text') { - responseText = (block as MessageTextContent).text.value + textBlocks.push((block as MessageTextContent).text.value) } } + if (textBlocks.length > 0) responseText = textBlocks.join('\n') } break } + if (responseText === undefined) { + throw new Error(`Agent run ${run.id} completed without a completed assistant message`) + } + sessionStore.updateSession(sessionId, { lastActivityAt: new Date().toISOString() }) const agentMessage = { @@ -191,15 +270,14 @@ export async function dispatchChat( source: 'agent' as const, turnId: ctx.turnId, } - sessionStore.addMessage(sessionId, agentMessage) - sseRegistry.broadcast(sessionId, agentMessage) + publishAssistantMessage(sessionId, agentMessage, ctx.onAssistantCompleted) return { text: responseText, messageId: agentMessage.id, title: generatedTitle } } } export async function chatHandler(req: Request, res: Response): Promise { - const { text, sessionId, source } = req.body + const { text, sessionId, source, messageId } = req.body if (!text || typeof text !== 'string') { res.status(400).json({ error: 'Missing or invalid "text" field' }) @@ -209,6 +287,11 @@ export async function chatHandler(req: Request, res: Response): Promise { res.status(400).json({ error: 'Missing or invalid "sessionId" field' }) return } + const clientMessageId = typeof messageId === 'string' + && messageId.trim().length > 0 + && messageId.length <= 128 + ? messageId.trim() + : undefined const { userId } = req.user const session = getAuthorizedSession(res, sessionId, userId) @@ -220,6 +303,7 @@ export async function chatHandler(req: Request, res: Response): Promise { displayName: req.user.displayName, ssoToken: req.ssoToken, source: source as 'voice' | 'text' | 'teams' | undefined, + clientMessageId, }) if (AGENT_BACKEND === 'directline' || AGENT_BACKEND === 'copilotstudio') { @@ -233,7 +317,7 @@ export async function chatHandler(req: Request, res: Response): Promise { }) } } catch (error) { - console.error('Chat handler error:', error) + cleanLogging.Error('Chat', 'Chat handler error', error) const message = error instanceof Error ? error.message : 'Failed to generate response' if (message.includes('SSO token required')) { res.status(401).json({ error: message }) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/index.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/index.ts index 5cd89567..1f3dac69 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/index.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/index.ts @@ -3,7 +3,7 @@ import rateLimit from 'express-rate-limit' import path from 'path' import { DefaultAzureCredential } from '@azure/identity' import { chatHandler } from './chatHandler.js' -import { listSessions, createSession, updateSession, addParticipant } from './sessionsHandler.js' +import { listSessions, createSession, updateSession, addParticipant, resumeSession } from './sessionsHandler.js' import { getTranscript } from './transcriptHandler.js' import { requireAuth } from './authMiddleware.js' import { sseHandler } from './sseHandler.js' @@ -89,6 +89,7 @@ app.get('/api/sessions', listSessions) app.post('/api/sessions', createSession) app.patch('/api/sessions/:id', updateSession) app.post('/api/sessions/:id/participants', addParticipant) +app.post('/api/sessions/:id/resume', resumeSession) // Transcript retrieval app.get('/api/transcript/:sessionId', getTranscript) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/resumeToken.test.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/resumeToken.test.ts new file mode 100644 index 00000000..f9d4474c --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/resumeToken.test.ts @@ -0,0 +1,86 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +const SECRET = 'test-resume-token-secret' +const USER_ID = 'user-1' +const SESSION_ID = 'session-1' + +async function importResumeToken(secret: string | undefined, nodeEnv = 'test') { + vi.resetModules() + vi.stubEnv('NODE_ENV', nodeEnv) + if (secret === undefined) { + vi.stubEnv('RESUME_TOKEN_SECRET', '') + } else { + vi.stubEnv('RESUME_TOKEN_SECRET', secret) + } + return import('./resumeToken.js') +} + +afterEach(() => { + vi.useRealTimers() + vi.unstubAllEnvs() +}) + +describe('resume tokens', () => { + it('issues and verifies a token bound to its user and session', async () => { + const { issueResumeToken, verifyResumeToken } = await importResumeToken(SECRET) + const pointers = { conversationId: 'conversation-1', threadId: 'thread-1' } + + const token = issueResumeToken(USER_ID, SESSION_ID, pointers) + + expect(token).toBeTypeOf('string') + expect(verifyResumeToken(token, USER_ID, SESSION_ID)).toEqual(pointers) + }) + + it('rejects a tampered signature', async () => { + const { issueResumeToken, verifyResumeToken } = await importResumeToken(SECRET) + const token = issueResumeToken(USER_ID, SESSION_ID, { threadId: 'thread-1' })! + const [payload, signature] = token.split('.') + const replacement = signature.endsWith('A') ? 'B' : 'A' + + expect(verifyResumeToken(`${payload}.${signature.slice(0, -1)}${replacement}`, USER_ID, SESSION_ID)).toBeNull() + }) + + it('rejects an expired token', async () => { + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')) + const { issueResumeToken, verifyResumeToken } = await importResumeToken(SECRET) + const token = issueResumeToken(USER_ID, SESSION_ID, { threadId: 'thread-1' })! + + vi.advanceTimersByTime(30 * 24 * 60 * 60 * 1000 + 1) + + expect(verifyResumeToken(token, USER_ID, SESSION_ID)).toBeNull() + }) + + it('rejects tokens presented for a different user or session', async () => { + const { issueResumeToken, verifyResumeToken } = await importResumeToken(SECRET) + const token = issueResumeToken(USER_ID, SESSION_ID, { threadId: 'thread-1' })! + + expect(verifyResumeToken(token, 'user-2', SESSION_ID)).toBeNull() + expect(verifyResumeToken(token, USER_ID, 'session-2')).toBeNull() + }) + + it.each([ + ['non-string', 42], + ['empty', ''], + ['missing signature', 'payload'], + ['invalid characters', 'payload.signature!'], + ['oversized', 'a'.repeat(2049)], + ])('rejects %s tokens', async (_description: string, token: unknown) => { + const { verifyResumeToken } = await importResumeToken(SECRET) + + expect(verifyResumeToken(token, USER_ID, SESSION_ID)).toBeNull() + }) + + it('disables token issue and verification without a secret in development', async () => { + const { issueResumeToken, verifyResumeToken } = await importResumeToken(undefined, 'development') + + expect(issueResumeToken(USER_ID, SESSION_ID, { threadId: 'thread-1' })).toBeUndefined() + expect(verifyResumeToken('payload.signature', USER_ID, SESSION_ID)).toBeNull() + }) + + it('requires a secret outside development', async () => { + await expect(importResumeToken(undefined, 'production')).rejects.toThrow( + 'RESUME_TOKEN_SECRET is required outside development', + ) + }) +}) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/resumeToken.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/resumeToken.ts new file mode 100644 index 00000000..79c0ae13 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/resumeToken.ts @@ -0,0 +1,122 @@ +import { createHmac, timingSafeEqual } from 'crypto' + +const RESUME_TOKEN_LIFETIME_MS = 30 * 24 * 60 * 60 * 1000 +const MAX_TOKEN_LENGTH = 2048 +const MAX_ENCODED_PAYLOAD_LENGTH = 1536 +const MAX_POINTER_LENGTH = 512 +const BASE64URL_RE = /^[A-Za-z0-9_-]+$/ +const SECRET = process.env.RESUME_TOKEN_SECRET +const NODE_ENV = process.env.NODE_ENV ?? 'development' + +if (!SECRET && NODE_ENV !== 'development') { + throw new Error('RESUME_TOKEN_SECRET is required outside development') +} + +const SIGNING_KEY = SECRET + ? createHmac('sha256', SECRET).update('chat-with-your-factory:resume-token:v1').digest() + : undefined + +export interface ResumePointers { + conversationId?: string + threadId?: string +} + +interface ResumePayload extends ResumePointers { + expiresAt: number + sessionId: string + userId: string +} + +function isBoundedPointer(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 && value.length <= MAX_POINTER_LENGTH +} + +function encodePayload(payload: ResumePayload): string { + return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url') +} + +function sign(encodedPayload: string): Buffer { + return createHmac('sha256', SIGNING_KEY!).update(`${encodedPayload.length}:${encodedPayload}`).digest() +} + +export function issueResumeToken( + userId: string, + sessionId: string, + pointers: ResumePointers, +): string | undefined { + if (!SIGNING_KEY) return undefined + + const payload: ResumePayload = { + userId, + sessionId, + expiresAt: Date.now() + RESUME_TOKEN_LIFETIME_MS, + ...(isBoundedPointer(pointers.conversationId) && { conversationId: pointers.conversationId }), + ...(isBoundedPointer(pointers.threadId) && { threadId: pointers.threadId }), + } + const encodedPayload = encodePayload(payload) + if (encodedPayload.length > MAX_ENCODED_PAYLOAD_LENGTH) return undefined + const signature = sign(encodedPayload).toString('base64url') + return `${encodedPayload}.${signature}` +} + +export function verifyResumeToken( + token: unknown, + expectedUserId: string, + expectedSessionId: string, +): ResumePointers | null { + if (!SIGNING_KEY || typeof token !== 'string' || token.length === 0 || token.length > MAX_TOKEN_LENGTH) { + return null + } + + const parts = token.split('.') + if (parts.length !== 2) return null + const [encodedPayload, encodedSignature] = parts + if ( + encodedPayload.length === 0 || + encodedPayload.length > MAX_ENCODED_PAYLOAD_LENGTH || + !BASE64URL_RE.test(encodedPayload) || + !BASE64URL_RE.test(encodedSignature) + ) { + return null + } + + let providedSignature: Buffer + try { + providedSignature = Buffer.from(encodedSignature, 'base64url') + } catch { + return null + } + if (providedSignature.toString('base64url') !== encodedSignature) return null + const expectedSignature = sign(encodedPayload) + if ( + providedSignature.length !== expectedSignature.length || + !timingSafeEqual(providedSignature, expectedSignature) + ) { + return null + } + + try { + const decoded = Buffer.from(encodedPayload, 'base64url') + if (decoded.toString('base64url') !== encodedPayload) return null + const payload = JSON.parse(decoded.toString('utf8')) as Record + const allowedKeys = new Set(['userId', 'sessionId', 'expiresAt', 'conversationId', 'threadId']) + if (Object.keys(payload).some(key => !allowedKeys.has(key))) return null + if ( + payload.userId !== expectedUserId || + payload.sessionId !== expectedSessionId || + typeof payload.expiresAt !== 'number' || + !Number.isSafeInteger(payload.expiresAt) || + payload.expiresAt <= Date.now() || + (payload.conversationId !== undefined && !isBoundedPointer(payload.conversationId)) || + (payload.threadId !== undefined && !isBoundedPointer(payload.threadId)) + ) { + return null + } + return { + ...(payload.conversationId !== undefined && { conversationId: payload.conversationId }), + ...(payload.threadId !== undefined && { threadId: payload.threadId }), + } + } catch { + return null + } +} diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionStore.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionStore.ts index aa1124eb..78b8ed6c 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionStore.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionStore.ts @@ -3,6 +3,77 @@ import type { Session, TranscriptMessage } from '../shared/types.js' const sessions = new Map() const chatIdToSession = new Map() // chatId → sessionId const sessionMessages = new Map() +const SESSION_ID_RE = /^session-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i +const CONTINUITY_WINDOW_MS = 30 * 24 * 60 * 60 * 1000 +const MAX_CLOCK_SKEW_MS = 5 * 60 * 1000 + +export interface AdoptSessionInput { + id: string + title?: unknown + createdAt?: unknown + lastActivityAt?: unknown + status?: unknown + metadata?: unknown +} + +export interface AdoptSessionIdentity { + userId: string + displayName: string +} + +export interface AdoptSessionPointers { + conversationId?: string + threadId?: string +} + +function sanitizeTitle(value: unknown): string { + if (typeof value !== 'string') return 'Conversation' + const sanitized = Array.from(value) + .filter(character => { + const code = character.charCodeAt(0) + return code >= 0x20 && !(code >= 0x7f && code <= 0x9f) + }) + .join('') + .trim() + return sanitized.slice(0, 200) || 'Conversation' +} + +function sanitizeTimestamp(value: unknown, fallback: string): string { + if (typeof value !== 'string' || value.length > 64) return fallback + const milliseconds = Date.parse(value) + const now = Date.now() + if ( + !Number.isFinite(milliseconds) || + milliseconds < now - CONTINUITY_WINDOW_MS || + milliseconds > now + MAX_CLOCK_SKEW_MS + ) { + return fallback + } + return new Date(milliseconds).toISOString() +} + +function sanitizeMetadata(value: unknown): Session['metadata'] { + if (!value || typeof value !== 'object' || Array.isArray(value)) return {} + const incoming = value as Record + const metadata: Session['metadata'] = {} + const sanitizeValue = (raw: unknown): string | undefined => { + if (typeof raw !== 'string') return undefined + const sanitized = Array.from(raw) + .filter(character => { + const code = character.charCodeAt(0) + return code >= 0x20 && !(code >= 0x7f && code <= 0x9f) + }) + .join('') + .trim() + .slice(0, 256) + return sanitized || undefined + } + const machineId = sanitizeValue(incoming.machineId) + const machineName = sanitizeValue(incoming.machineName) + if (machineId) metadata.machineId = machineId + if (machineName) metadata.machineName = machineName + return metadata +} export const sessionStore = { listByUser(userId: string): Session[] { @@ -38,6 +109,35 @@ export const sessionStore = { return session }, + adoptSession( + incoming: AdoptSessionInput, + identity: AdoptSessionIdentity, + pointers: AdoptSessionPointers, + ): Session | undefined { + if (!SESSION_ID_RE.test(incoming.id)) return undefined + const existing = sessions.get(incoming.id) + if (existing) return existing + + const now = new Date().toISOString() + const createdAt = sanitizeTimestamp(incoming.createdAt, now) + const lastActivityAt = sanitizeTimestamp(incoming.lastActivityAt, createdAt) + const session: Session = { + id: incoming.id, + userId: identity.userId, + ...(pointers.threadId && { threadId: pointers.threadId }), + ...(pointers.conversationId && { conversationId: pointers.conversationId }), + title: sanitizeTitle(incoming.title), + createdAt, + lastActivityAt: lastActivityAt < createdAt ? createdAt : lastActivityAt, + status: incoming.status === 'archived' ? 'archived' : 'active', + participants: [identity.userId], + participantNames: { [identity.userId]: identity.displayName }, + metadata: sanitizeMetadata(incoming.metadata), + } + sessions.set(session.id, session) + return session + }, + updateSession( sessionId: string, updates: Partial>, diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionsHandler.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionsHandler.ts index e6d0e8fe..94b27a88 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionsHandler.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/sessionsHandler.ts @@ -1,10 +1,32 @@ import type { Request, Response } from 'express' +import type { Session } from '../shared/types.js' +import { cleanLogging } from '../shared/logging.js' import * as directLine from './directLineClient.js' import { sseRegistry } from './sseRegistry.js' import { getAuthorizedSession } from './aclHelper.js' import { sessionStore } from './sessionStore.js' +import { issueResumeToken, verifyResumeToken } from './resumeToken.js' const AGENT_BACKEND = process.env.AGENT_BACKEND || 'foundry' +const SESSION_ID_RE = /^session-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i +const MAX_BACKEND_POINTER_LENGTH = 512 + +type SessionResponse = Session & { resumeToken?: string } + +function withResumeToken(session: Session, userId: string): SessionResponse { + const threadId = session.threadId + if ( + AGENT_BACKEND !== 'foundry' || + typeof threadId !== 'string' || + threadId.length === 0 || + threadId.length > MAX_BACKEND_POINTER_LENGTH + ) { + return { ...session } + } + + const resumeToken = issueResumeToken(userId, session.id, { threadId }) + return { ...session, ...(resumeToken && { resumeToken }) } +} /** Strip control characters and limit displayName length for defense-in-depth. */ export function sanitizeDisplayName(raw: string | undefined, fallback = 'Unknown'): string { @@ -23,7 +45,7 @@ export async function listSessions(req: Request, res: Response): Promise { const { userId } = req.user const chatId = req.query.chatId as string | undefined - console.log('[Sessions] listSessions | userId:', userId, '| chatId:', chatId) + cleanLogging.Log('Sessions', 'Listing sessions', { userId, chatId }) let sessions = sessionStore.listByUser(userId) @@ -31,7 +53,7 @@ export async function listSessions(req: Request, res: Response): Promise { // (e.g., created by the bot) and auto-add the user as a participant if (chatId) { const chatSession = sessionStore.findByChatId(chatId) - console.log('[Sessions] findByChatId result:', chatSession?.id ?? 'none') + cleanLogging.Log('Sessions', 'Chat session lookup completed', { sessionId: chatSession?.id ?? 'none' }) if (chatSession) { if (!chatSession.participants.includes(userId)) { sessionStore.addParticipant(chatSession.id, userId, req.user.displayName) @@ -40,7 +62,7 @@ export async function listSessions(req: Request, res: Response): Promise { } } - res.json(sessions) + res.json(sessions.map(session => withResumeToken(session, userId))) } export async function createSession(req: Request, res: Response): Promise { @@ -79,8 +101,9 @@ export async function createSession(req: Request, res: Response): Promise const chatId = req.body?.chatId as string | undefined if (chatId) sessionStore.linkChatId(session.id, chatId) - res.status(201).json(sessionStore.getSession(session.id)) - } + const created = sessionStore.getSession(session.id) + res.status(201).json(withResumeToken(created ?? session, userId)) + } else if (AGENT_BACKEND === 'copilotstudio') { // Copilot Studio Agents SDK — dynamic import to avoid crash when CPS env vars absent @@ -119,7 +142,7 @@ export async function createSession(req: Request, res: Response): Promise ) sessionStore.updateSession(session.id, { conversationId }) } catch (cpsError) { - console.error('CPS startConversation failed:', cpsError) + cleanLogging.Error('Sessions', 'CPS startConversation failed', cpsError) // Mark orphaned session so it's not usable sessionStore.updateSession(session.id, { status: 'archived' }) throw cpsError @@ -128,9 +151,10 @@ export async function createSession(req: Request, res: Response): Promise const chatId = req.body?.chatId as string | undefined if (chatId) sessionStore.linkChatId(session.id, chatId) - res.status(201).json(sessionStore.getSession(session.id)) + const created = sessionStore.getSession(session.id) + res.status(201).json(withResumeToken(created ?? session, userId)) } - else + else { // Foundry flow — dynamic import to avoid crash when Foundry env vars absent const { agentsClient } = await import('./agentsClient.js') @@ -143,10 +167,10 @@ export async function createSession(req: Request, res: Response): Promise ) const chatId = req.body?.chatId as string | undefined if (chatId) sessionStore.linkChatId(session.id, chatId) - res.status(201).json(session) + res.status(201).json(withResumeToken(session, userId)) } } catch (error) { - console.error('Create session error:', error) + cleanLogging.Error('Sessions', 'Create session error', error) res.status(500).json({ error: 'Failed to create session' }) } } @@ -166,7 +190,7 @@ export async function updateSession(req: Request, res: Response): Promise ...(status && { status }), ...(title && { title }), }) - res.json(updated) + res.json(updated ? withResumeToken(updated, userId) : updated) } export async function addParticipant(req: Request, res: Response): Promise { @@ -184,5 +208,69 @@ export async function addParticipant(req: Request, res: Response): Promise const displayName = sanitizeDisplayName(rawDisplayName) const updated = sessionStore.addParticipant(session.id, newUserId, displayName) - res.json(updated) + res.json(updated ? withResumeToken(updated, userId) : updated) +} + +export async function resumeSession(req: Request, res: Response): Promise { + const { userId, displayName } = req.user + const sessionId = req.params.id as string + if (!SESSION_ID_RE.test(sessionId)) { + res.status(400).json({ error: 'Invalid session id format' }) + return + } + + const body = req.body as Record | undefined + const incoming = body?.session + if (!incoming || typeof incoming !== 'object' || Array.isArray(incoming)) { + res.status(400).json({ error: 'Body must include a session object' }) + return + } + const cached = incoming as Record + if (cached.id !== sessionId) { + res.status(400).json({ error: 'Session id must match the route' }) + return + } + + const existing = sessionStore.getSession(sessionId) + if (existing) { + if (!existing.participants.includes(userId)) { + res.status(403).json({ error: 'Not a participant of this session' }) + return + } + res.json(withResumeToken(existing, userId)) + return + } + + if (AGENT_BACKEND !== 'foundry') { + res.status(409).json({ error: 'Restart resume is only available for the Foundry backend' }) + return + } + + const pointers = verifyResumeToken(cached.resumeToken, userId, sessionId) + if (!pointers?.threadId || pointers.conversationId) { + res.status(403).json({ error: 'A valid Foundry resume token with only a thread id is required' }) + return + } + + const adopted = sessionStore.adoptSession( + { + id: sessionId, + title: cached.title, + createdAt: cached.createdAt, + lastActivityAt: cached.lastActivityAt, + status: cached.status, + metadata: cached.metadata, + }, + { userId, displayName: sanitizeDisplayName(displayName) }, + { threadId: pointers.threadId }, + ) + if (!adopted) { + res.status(400).json({ error: 'Session could not be adopted' }) + return + } + if (!adopted.participants.includes(userId)) { + res.status(403).json({ error: 'Not a participant of this session' }) + return + } + res.json(withResumeToken(adopted, userId)) } diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/speechText.test.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/speechText.test.ts new file mode 100644 index 00000000..0a71977d --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/speechText.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' + +import { toSpeechText } from './speechText.js' + +describe('toSpeechText', () => { + it('preserves ordinary text', () => { + expect(toSpeechText('Robot 7 is ready.')).toBe('Robot 7 is ready.') + }) + + it('keeps descriptive link labels while removing URLs', () => { + expect(toSpeechText('Open the [robot manual](https://example.com/manual).')).toBe( + 'Open the robot manual.', + ) + }) + + it('removes numeric citation links and references', () => { + const markdown = 'Robot 7 is ready [1].\n\n[1]: https://example.com/status' + + expect(toSpeechText(markdown)).toBe('Robot 7 is ready.') + }) + + it('removes inline citation markers and raw URLs', () => { + expect(toSpeechText('Robot 7 [2] is documented at https://example.com/robots')).toBe( + 'Robot 7 is documented at', + ) + }) + + it('removes zero-width formatting and normalizes whitespace', () => { + expect(toSpeechText(' Robot\u200B\t7\r\n is ready. ')).toBe('Robot 7 is ready.') + }) + + it('removes empty delimiters without leaving punctuation gaps', () => { + expect(toSpeechText('Ready ( ) [ ] { } , proceed!')).toBe('Ready, proceed!') + }) +}) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/speechText.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/speechText.ts new file mode 100644 index 00000000..d71006d3 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/speechText.ts @@ -0,0 +1,15 @@ +const MARKDOWN_LINK = /\[([^\]\r\n]+)\]\(\s*https?:\/\/(?:[^\s()]|\([^\r\n)]*\))+(?:\s+(?:"[^"\r\n]*"|'[^'\r\n]*'|\([^\r\n)]*\)))?\s*\)/gi +const ZERO_WIDTH_FORMAT = /[\u200B-\u200D\u2060\uFEFF]/g + +export function toSpeechText(markdown: string): string { + return markdown + .replace(ZERO_WIDTH_FORMAT, '') + .replace(/^\s*\[\d+\]:[^\r\n]*(?:\r?\n|$)/gm, '') + .replace(MARKDOWN_LINK, (_link, label: string) => /^\d+$/.test(label.trim()) ? '' : label) + .replace(/\[\d+\]/g, '') + .replace(/https?:\/\/[^\s<>()[\]{}]+/gi, '') + .replace(/\(\s*\)|\[\s*\]|\{\s*\}/g, ' ') + .replace(/\s+([,.;:!?])/g, '$1') + .replace(/\s+/g, ' ') + .trim() +} \ No newline at end of file diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveBridge.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveBridge.ts index 404444c3..0c95553b 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveBridge.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveBridge.ts @@ -2,38 +2,38 @@ import type { Server, IncomingMessage } from 'http' import type { Duplex } from 'stream' import { WebSocketServer, WebSocket } from 'ws' import { DefaultAzureCredential } from '@azure/identity' +import { cleanLogging } from '../shared/logging.js' import { getAuthorizedSessionFor } from './aclHelper.js' import { dispatchChat } from './chatHandler.js' import { sessionStore } from './sessionStore.js' +import { toSpeechText } from './speechText.js' import { sseRegistry } from './sseRegistry.js' +import { buildResponseCreate, buildSessionUpdate } from './voiceLiveProtocol.js' import { consumeTicket } from './wsTicketStore.js' const VOICELIVE_RESOURCE = process.env.AZURE_VOICELIVE_RESOURCE ?? '' const VOICELIVE_MODEL = process.env.AZURE_VOICELIVE_MODEL ?? 'gpt-realtime' -const VOICELIVE_API_VERSION = process.env.AZURE_VOICELIVE_API_VERSION ?? '2025-10-01' +const VOICELIVE_API_VERSION = process.env.AZURE_VOICELIVE_API_VERSION ?? '2026-04-10' const UTTERANCE_SILENCE_MS = 2000 // When true, log raw transcript content to server logs. Off by default so // production logs don't capture sensitive user speech in plaintext. const VOICELIVE_DEBUG = process.env.VOICELIVE_DEBUG === 'true' const credential = new DefaultAzureCredential() +const voiceDispatchTails = new Map>() -/** - * session.update — pure STT/VAD engine, no model responses. - * User transcripts are accumulated and dispatched as a single turn - * after a silence window, enabling natural speech corrections. - */ -function buildSessionUpdate() { - return { - type: 'session.update', - session: { - modalities: ['text'], - input_audio_format: 'pcm16', - input_audio_transcription: { model: 'azure-speech', language: 'en' }, - input_audio_noise_reduction: { type: 'azure_deep_noise_suppression' }, - turn_detection: { type: 'azure_semantic_vad', create_response: false }, - }, +function enqueueVoiceDispatch(sessionId: string, job: () => Promise): Promise { + const previous = voiceDispatchTails.get(sessionId) ?? Promise.resolve() + const current = previous.catch(() => {}).then(job) + voiceDispatchTails.set(sessionId, current) + + const deleteCurrentTail = () => { + if (voiceDispatchTails.get(sessionId) === current) { + voiceDispatchTails.delete(sessionId) + } } + void current.then(deleteCurrentTail, deleteCurrentTail) + return current } /** @@ -98,14 +98,14 @@ export function attachVoiceLiveBridge(httpServer: Server): WebSocketServer { wss.emit('connection', clientWs, req) void handleConnection(clientWs, sessionId, ticket.userId, ticket.displayName, ticket.ssoToken) .catch((err) => { - console.error('[VoiceLive] Unhandled connection error:', err) + cleanLogging.Error('VoiceLive', 'Unhandled connection error', err) if (clientWs.readyState === WebSocket.OPEN) { clientWs.close(1011, 'Connection setup failed') } }) }) } catch (err) { - console.error('[VoiceLive] Upgrade handler error:', err) + cleanLogging.Error('VoiceLive', 'Upgrade handler error', err) try { socket.write('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n') } catch { @@ -133,10 +133,53 @@ async function handleConnection( const pendingUtterances: string[] = [] let dispatchTimer: ReturnType | null = null + function sendCompletedResponse(markdown: string): void { + if (upstreamWs?.readyState !== WebSocket.OPEN) return + upstreamWs.send(JSON.stringify(buildResponseCreate(toSpeechText(markdown)))) + } + + function queueDispatch(fullText: string, turnId: string, errorMessage: string): Promise { + const dispatchContext = { + userId, + displayName, + ssoToken, + source: 'voice' as const, + skipUserMessage: true, + turnId, + onAssistantCompleted: sendCompletedResponse, + } + const notifyFailure = (err: unknown) => { + if (clientWs.readyState !== WebSocket.OPEN) return + const message = err instanceof Error ? err.message : 'Dispatch failed' + try { + clientWs.send(JSON.stringify({ type: 'dispatch.failed', error: message, turnId })) + } catch (sendErr) { + cleanLogging.Warn('VoiceLive', 'Failed to send dispatch.failed event', sendErr) + } + } + + if (clientWs.readyState === WebSocket.OPEN) { + try { + clientWs.send(JSON.stringify({ type: 'dispatching', turnId })) + } catch (sendErr) { + cleanLogging.Warn('VoiceLive', 'Failed to send dispatching event', sendErr) + } + } + + return enqueueVoiceDispatch(sessionId, async () => { + try { + await dispatchChat(sessionId, fullText, dispatchContext) + } catch (err) { + cleanLogging.Error('VoiceLive', errorMessage, err) + notifyFailure(err) + } + }) + } + function scheduleDispatch() { if (dispatchTimer) clearTimeout(dispatchTimer) dispatchTimer = setTimeout(() => { - void (async () => { + void Promise.resolve().then(() => { dispatchTimer = null if (pendingUtterances.length === 0) return const fullText = pendingUtterances.join(' ') @@ -148,44 +191,18 @@ async function handleConnection( // from prematurely clearing the local pending counter. const turnId = crypto.randomUUID() if (VOICELIVE_DEBUG) { - console.log(`[VoiceLive] Dispatching session=${sessionId} turn=${turnId}: ${fullText}`) + cleanLogging.Log('VoiceLive', 'Dispatching transcript', { sessionId, turnId, transcript: fullText }) } else { - console.log(`[VoiceLive] Dispatching session=${sessionId} turn=${turnId} utterances=${utteranceCount} chars=${fullText.length}`) - } - // Notify client that a dispatch is in flight - if (clientWs.readyState === WebSocket.OPEN) { - try { - clientWs.send(JSON.stringify({ type: 'dispatching', turnId })) - } catch (sendErr) { - console.warn('[VoiceLive] Failed to send dispatching event:', sendErr) - } - } - try { - await dispatchChat(sessionId, fullText, { - userId, - displayName, - ssoToken, - source: 'voice', - skipUserMessage: true, + cleanLogging.Log('VoiceLive', 'Dispatching transcript', { + sessionId, turnId, + utteranceCount, + characterCount: fullText.length, }) - } catch (err) { - console.error('[VoiceLive] dispatchChat error:', err) - // Tell the client the dispatch failed so it can clear its pending - // counter / spinner. Without this, the UI stays stuck in a loading - // state because it normally clears the counter on the assistant SSE - // event that will never arrive. - if (clientWs.readyState === WebSocket.OPEN) { - const message = err instanceof Error ? err.message : 'Dispatch failed' - try { - clientWs.send(JSON.stringify({ type: 'dispatch.failed', error: message, turnId })) - } catch (sendErr) { - console.warn('[VoiceLive] Failed to send dispatch.failed event:', sendErr) - } - } } - })().catch((err) => { - console.error('[VoiceLive] Unhandled dispatch timer error:', err) + return queueDispatch(fullText, turnId, 'dispatchChat error') + }).catch((err) => { + cleanLogging.Error('VoiceLive', 'Unhandled dispatch timer error', err) }) }, UTTERANCE_SILENCE_MS) } @@ -197,7 +214,7 @@ async function handleConnection( // can produce a token (e.g. missing managed identity, expired CLI login). // Fail fast with a clear close reason rather than letting a downstream // throw surface as a generic 1011. - console.error('[VoiceLive] Failed to acquire AAD token for Cognitive Services') + cleanLogging.Error('VoiceLive', 'Failed to acquire AAD token for Cognitive Services') if (clientWs.readyState === WebSocket.OPEN) { clientWs.close(1011, 'Upstream auth unavailable') } @@ -213,7 +230,7 @@ async function handleConnection( }) upstreamWs.on('open', () => { - console.log(`[VoiceLive] Upstream connected for session ${sessionId}`) + cleanLogging.Log('VoiceLive', 'Upstream connected', { sessionId }) upstreamWs!.send(JSON.stringify(buildSessionUpdate())) }) @@ -244,7 +261,7 @@ async function handleConnection( // every one floods production logs, so gate behind VOICELIVE_DEBUG // (the same flag that controls raw transcript content logging). if (VOICELIVE_DEBUG && !event.type?.includes('.delta')) { - console.log(`[VoiceLive] Upstream event: ${event.type}`) + cleanLogging.Log('VoiceLive', 'Upstream event', { type: event.type }) } // Signal client to start sending audio after session is configured @@ -264,9 +281,12 @@ async function handleConnection( const transcript = (event as { transcript?: string }).transcript ?? '' if (transcript.trim()) { if (VOICELIVE_DEBUG) { - console.log(`[VoiceLive] User said (session=${sessionId}): ${transcript}`) + cleanLogging.Log('VoiceLive', 'User transcript completed', { sessionId, transcript }) } else { - console.log(`[VoiceLive] Transcript completed session=${sessionId} chars=${transcript.trim().length}`) + cleanLogging.Log('VoiceLive', 'User transcript completed', { + sessionId, + characterCount: transcript.trim().length, + }) } const userMsg = { id: crypto.randomUUID(), @@ -284,12 +304,12 @@ async function handleConnection( } } } catch (err) { - console.error('[VoiceLive] Error handling upstream message:', err) + cleanLogging.Error('VoiceLive', 'Error handling upstream message', err) } }) upstreamWs.on('close', (code, reason) => { - console.log(`[VoiceLive] Upstream closed: ${code} ${reason}`) + cleanLogging.Log('VoiceLive', 'Upstream closed', { code, reason: reason.toString() }) if (dispatchTimer) clearTimeout(dispatchTimer) const clientCode = code === 1000 ? 1000 : 1011 const clientReason = code === 1000 ? 'Upstream closed' : `Upstream failure (${code})` @@ -297,7 +317,7 @@ async function handleConnection( }) upstreamWs.on('error', (err) => { - console.error('[VoiceLive] Upstream error:', err) + cleanLogging.Error('VoiceLive', 'Upstream error', err) if (clientWs.readyState === WebSocket.OPEN) clientWs.close(1011, 'Upstream error') }) @@ -328,7 +348,7 @@ async function handleConnection( }) clientWs.on('close', () => { - console.log(`[VoiceLive] Client disconnected for session ${sessionId}`) + cleanLogging.Log('VoiceLive', 'Client disconnected', { sessionId }) if (dispatchTimer) clearTimeout(dispatchTimer) // Flush pending utterances on disconnect. // Each individual transcript was already persisted + broadcast as a @@ -345,35 +365,18 @@ async function handleConnection( // ignore the turnId since it isn't in their pending set, which is // the correct behavior. const turnId = crypto.randomUUID() - // Best-effort `dispatching` event so the originating client (if - // its socket flushes one more frame before close) can register - // the turnId in its pending set; without this, a later - // `dispatch.failed` carrying the same turnId can't correlate. - if (clientWs.readyState === WebSocket.OPEN) { - clientWs.send(JSON.stringify({ type: 'dispatching', turnId })) - } - dispatchChat(sessionId, fullText, { userId, displayName, ssoToken, source: 'voice', skipUserMessage: true, turnId }) - .catch(err => { - console.error('[VoiceLive] Final dispatch error:', err) - // Best-effort: client is already disconnecting, but if the socket - // somehow flushes this in time the UI will clear its spinner. - // Include turnId so the client can correlate to its pending set - // (the timer-dispatch path does the same — see above). - if (clientWs.readyState === WebSocket.OPEN) { - const message = err instanceof Error ? err.message : 'Dispatch failed' - clientWs.send(JSON.stringify({ type: 'dispatch.failed', error: message, turnId })) - } - }) + void queueDispatch(fullText, turnId, 'Final dispatch error') + .catch(err => cleanLogging.Error('VoiceLive', 'Unhandled final dispatch error', err)) } if (upstreamWs?.readyState === WebSocket.OPEN) upstreamWs.close() }) clientWs.on('error', (err) => { - console.error('[VoiceLive] Client error:', err) + cleanLogging.Error('VoiceLive', 'Client error', err) if (upstreamWs?.readyState === WebSocket.OPEN) upstreamWs.close() }) } catch (err) { - console.error('[VoiceLive] Connection setup failed:', err) + cleanLogging.Error('VoiceLive', 'Connection setup failed', err) if (clientWs.readyState === WebSocket.OPEN) clientWs.close(1011, 'Setup failed') if (upstreamWs?.readyState === WebSocket.OPEN) upstreamWs.close() } diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveProtocol.test.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveProtocol.test.ts new file mode 100644 index 00000000..45b31e90 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveProtocol.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest' + +import { buildResponseCreate, buildSessionUpdate } from './voiceLiveProtocol.js' + +describe('Voice Live protocol payloads', () => { + it('builds the configured PCM16 session update', () => { + expect(buildSessionUpdate()).toEqual({ + type: 'session.update', + session: { + modalities: ['text', 'audio'], + input_audio_format: 'pcm16', + input_audio_transcription: { model: 'azure-speech', language: 'en' }, + input_audio_noise_reduction: { type: 'azure_deep_noise_suppression' }, + input_audio_echo_cancellation: { type: 'server_echo_cancellation' }, + output_audio_format: 'pcm16', + voice: { + type: 'azure-standard', + name: 'en-US-TonyNeural', + }, + turn_detection: { + type: 'azure_semantic_vad', + create_response: false, + interrupt_response: true, + auto_truncate: true, + }, + }, + }) + }) + + it('wraps authoritative text as a pre-generated assistant message', () => { + expect(buildResponseCreate('Robot 7 is ready.')).toEqual({ + type: 'response.create', + response: { + pre_generated_assistant_message: { + type: 'message', + role: 'assistant', + content: [{ type: 'text', text: 'Robot 7 is ready.' }], + }, + }, + }) + }) + + it('does not normalize response text', () => { + expect( + buildResponseCreate(' exact text ').response.pre_generated_assistant_message.content[0].text, + ).toBe(' exact text ') + }) +}) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveProtocol.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveProtocol.ts new file mode 100644 index 00000000..55b6c519 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/server/voiceLiveProtocol.ts @@ -0,0 +1,41 @@ +export function buildSessionUpdate() { + return { + type: 'session.update', + session: { + modalities: ['text', 'audio'], + input_audio_format: 'pcm16', + input_audio_transcription: { model: 'azure-speech', language: 'en' }, + input_audio_noise_reduction: { type: 'azure_deep_noise_suppression' }, + input_audio_echo_cancellation: { type: 'server_echo_cancellation' }, + output_audio_format: 'pcm16', + voice: { + type: 'azure-standard', + name: 'en-US-TonyNeural', + }, + turn_detection: { + type: 'azure_semantic_vad', + create_response: false, + interrupt_response: true, + auto_truncate: true, + }, + }, + } +} + +export function buildResponseCreate(text: string) { + return { + type: 'response.create', + response: { + pre_generated_assistant_message: { + type: 'message', + role: 'assistant', + content: [ + { + type: 'text', + text, + }, + ], + }, + }, + } +} diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/logging.test.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/logging.test.ts new file mode 100644 index 00000000..ba926eea --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/logging.test.ts @@ -0,0 +1,71 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { cleanLogging } from './logging.js' + +afterEach(() => { + vi.restoreAllMocks() +}) + +describe('cleanLogging', () => { + it('removes control characters and repeated whitespace from labels', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + + cleanLogging.Log('voice\nbridge', 'queued\t response') + + expect(log).toHaveBeenCalledWith('[voice bridge] queued response') + }) + + it('sanitizes nested values and circular references', () => { + const log = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const value: Record = { status: 'ready\r\n now' } + value.self = value + + cleanLogging.Warn('session', 'state', value) + + expect(log).toHaveBeenCalledWith('[session] state', { + status: 'ready now', + self: '[Circular]', + }) + }) + + it('truncates long strings to the configured bound', () => { + const log = vi.spyOn(console, 'error').mockImplementation(() => undefined) + + cleanLogging.Error('session', 'failure', 'x'.repeat(600)) + + const loggedValue = log.mock.calls[0][1] as string + expect(loggedValue).toHaveLength(500) + expect(loggedValue.endsWith('... [truncated]')).toBe(true) + }) + + it('bounds arrays and object properties', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + const array = Array.from({ length: 21 }, (_, index) => index) + const object = Object.fromEntries(Array.from({ length: 21 }, (_, index) => [`key${index}`, index])) + + cleanLogging.Log('limits', 'values', array, object) + + expect(log.mock.calls[0][1]).toEqual([...array.slice(0, 20), '[Truncated array items]']) + expect(log.mock.calls[0][2]).toMatchObject({ '[Truncated object properties]': 1 }) + expect(Object.keys(log.mock.calls[0][2] as object)).toHaveLength(21) + }) + + it('bounds nested object depth', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + + cleanLogging.Log('limits', 'depth', { one: { two: { three: { four: true } } } }) + + expect(log.mock.calls[0][1]).toEqual({ + one: { two: { three: '[Object depth truncated]' } }, + }) + }) + + it('limits the number of logged arguments', () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined) + + cleanLogging.Log('limits', 'arguments', ...Array.from({ length: 21 }, (_, index) => index)) + + expect(log.mock.calls[0]).toHaveLength(22) + expect(log.mock.calls[0].at(-1)).toBe('[Truncated log arguments]') + }) +}) diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/logging.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/logging.ts new file mode 100644 index 00000000..b8dc0551 --- /dev/null +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/logging.ts @@ -0,0 +1,121 @@ +type ConsoleLevel = 'log' | 'warn' | 'error' + +const REPEATED_WHITESPACE = /\s{2,}/g +const MAX_STRING_LENGTH = 500 +const MAX_KEY_LENGTH = 100 +const MAX_DEPTH = 3 +const MAX_LOG_ARGUMENTS = 20 +const MAX_ARRAY_ITEMS = 20 +const MAX_OBJECT_PROPERTIES = 20 +const STRING_TRUNCATION_SENTINEL = '... [truncated]' +const ARRAY_TRUNCATION_SENTINEL = '[Truncated array items]' +const OBJECT_TRUNCATION_KEY = '[Truncated object properties]' + +function cleanString(value: string, maxLength = MAX_STRING_LENGTH): string { + const withoutControlCharacters = Array.from(value, character => { + const codePoint = character.codePointAt(0) ?? 0 + return codePoint <= 0x1f || (codePoint >= 0x7f && codePoint <= 0x9f) ? ' ' : character + }).join('') + const normalized = withoutControlCharacters + .replace(REPEATED_WHITESPACE, ' ') + .trim() + + if (normalized.length <= maxLength) return normalized + + const contentLength = Math.max(0, maxLength - STRING_TRUNCATION_SENTINEL.length) + return `${normalized.slice(0, contentLength)}${STRING_TRUNCATION_SENTINEL}` +} + +function sanitizeForLog(value: unknown): unknown { + const seen = new WeakSet() + + function sanitize(currentValue: unknown, depth: number): unknown { + try { + if (currentValue == null) return currentValue + + switch (typeof currentValue) { + case 'string': + return cleanString(currentValue) + case 'number': + case 'boolean': + case 'bigint': + return currentValue + case 'symbol': + return cleanString(String(currentValue)) + case 'function': + return '[Function]' + case 'object': + break + default: + return cleanString(String(currentValue)) + } + + if (currentValue instanceof Error) { + return { + name: cleanString(currentValue.name), + message: cleanString(currentValue.message), + stack: currentValue.stack ? cleanString(currentValue.stack) : undefined, + } + } + + if (currentValue instanceof Date) { + return Number.isNaN(currentValue.getTime()) ? '[Invalid Date]' : currentValue.toISOString() + } + + if (currentValue instanceof URL) return cleanString(currentValue.toString()) + + if (seen.has(currentValue)) return '[Circular]' + if (depth >= MAX_DEPTH) return Array.isArray(currentValue) ? '[Array depth truncated]' : '[Object depth truncated]' + seen.add(currentValue) + + if (Array.isArray(currentValue)) { + const result = currentValue + .slice(0, MAX_ARRAY_ITEMS) + .map(item => sanitize(item, depth + 1)) + if (currentValue.length > MAX_ARRAY_ITEMS) result.push(ARRAY_TRUNCATION_SENTINEL) + return result + } + + const prototype = Object.getPrototypeOf(currentValue) + if (prototype !== Object.prototype && prototype !== null) return '[Object]' + + const keys = Object.keys(currentValue) + const result: Record = {} + for (const key of keys.slice(0, MAX_OBJECT_PROPERTIES)) { + const sanitizedKey = cleanString(key, MAX_KEY_LENGTH) || '[Empty key]' + try { + result[sanitizedKey] = sanitize((currentValue as Record)[key], depth + 1) + } catch { + result[sanitizedKey] = '[Unserializable property]' + } + } + if (keys.length > MAX_OBJECT_PROPERTIES) { + result[OBJECT_TRUNCATION_KEY] = keys.length - MAX_OBJECT_PROPERTIES + } + return result + } catch { + return '[Unserializable value]' + } + } + + return sanitize(value, 0) +} + +function writeCleanLog(level: ConsoleLevel, scope: string, message: string, ...args: unknown[]): void { + const output: unknown[] = [`[${cleanString(scope)}] ${cleanString(message)}`] + output.push(...args.slice(0, MAX_LOG_ARGUMENTS).map(sanitizeForLog)) + if (args.length > MAX_LOG_ARGUMENTS) output.push('[Truncated log arguments]') + console[level](...output) +} + +export const cleanLogging = { + Log(scope: string, message: string, ...args: unknown[]): void { + writeCleanLog('log', scope, message, ...args) + }, + Warn(scope: string, message: string, ...args: unknown[]): void { + writeCleanLog('warn', scope, message, ...args) + }, + Error(scope: string, message: string, ...args: unknown[]): void { + writeCleanLog('error', scope, message, ...args) + }, +} \ No newline at end of file diff --git a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/types.ts b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/types.ts index ad052749..1643e4ce 100644 --- a/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/types.ts +++ b/src/500-application/516-chat-with-your-factory/services/chat-with-your-factory/src/shared/types.ts @@ -3,6 +3,7 @@ export interface Session { userId: string threadId?: string conversationId?: string + resumeToken?: string title: string createdAt: string lastActivityAt: string @@ -47,6 +48,12 @@ export interface UserContext { chatId?: string | null } +export type DeviceContinuityState = + | 'loading' + | 'local-fallback' + | 'server-reconciled' + | 'cleared' + export type AddParticipantErrorCode = | 'NO_ACTIVE_SESSION' | 'INVALID_INPUT'