feat(voice): port AvatarSession base class and transcript sync asymmetric detach warning

.changeset/avatar-session-base-port.md

@@ -0,0 +1,9 @@
+---
+'@livekit/agents': patch
+'@livekit/agents-plugin-anam': patch
+'@livekit/agents-plugin-bey': patch
+'@livekit/agents-plugin-lemonslice': patch
+'@livekit/agents-plugin-trugen': patch
+---
+
+Add `voice.AvatarSession` base class and port the asymmetric-detach warning from the Python `TranscriptSynchronizer`. The new base class registers `aclose` as a job shutdown callback and warns when an avatar session is started after `AgentSession.start()` has already wired an audio output. The transcript synchronizer now tracks `_audioAttached` / `_textAttached` via `onAttached` / `onDetached` and logs a one-shot warning when audio or text is detached asymmetrically (covering external avatars and manual `session.output.audio` / `.transcription` replacement). Existing avatar plugins (anam, bey, lemonslice, trugen) now inherit from `voice.AvatarSession` and call `super.start(agentSession, room)` first.

agents/src/voice/agent_session.ts

@@ -267,6 +267,12 @@ export class AgentSession<
   /** @internal - Timestamp when the session started (milliseconds) */
   _startedAt?: number;
 
+  // Ref: python livekit-agents/livekit/agents/voice/avatar/_types.py - 62-71 lines
+  /** @internal - Whether `start()` has been called and completed. */
+  get _started(): boolean {
+    return this.started;
+  }
+
   /** @internal - Current run state for testing */
   _globalRunState?: RunResult;
 

agents/src/voice/avatar/avatar_session.ts

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: 2026 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import type { Room } from '@livekit/rtc-node';
+import { getJobContext } from '../../job.js';
+import { log } from '../../log.js';
+import type { AgentSession } from '../agent_session.js';
+
+// Ref: python livekit-agents/livekit/agents/voice/avatar/_types.py - 53-78 lines
+/**
+ * Base class for avatar plugin sessions.
+ *
+ * Plugin implementations should extend this class and call `super.start(agentSession, room)`
+ * first in their own `start()` method. The base:
+ * - Registers {@link AvatarSession.aclose} as a job shutdown callback, so avatar resources
+ *   are released when the job shuts down.
+ * - Warns when the avatar session is started after {@link AgentSession.start} — in that
+ *   case the existing audio output will be replaced by the avatar's.
+ */
+export class AvatarSession {
+  #logger = log();
+
+  /**
+   * Start the avatar session.
+   *
+   * Subclasses should override this method and call `super.start(agentSession, room)` at the
+   * top of their implementation. Subclasses may widen the return type (e.g. returning a
+   * session id), matching the `# type: ignore[override]` escape hatch used in Python.
+   */
+  async start(agentSession: AgentSession, _room: Room): Promise<unknown> {
+    const jobCtx = getJobContext(false);
+    if (jobCtx !== undefined) {
+      jobCtx.addShutdownCallback(() => this.aclose());
+    } else {
+      this.#logger.debug(
+        'AvatarSession started outside a job context; call aclose() manually to ' +
+          'release resources when the job shuts down',
+      );
+    }
+
+    const audioOutput = agentSession.output.audio;
+    if (agentSession._started && audioOutput !== null) {
+      this.#logger.warn(
+        { audioOutput: audioOutput.constructor.name },
+        'AvatarSession.start() was called after AgentSession.start(); ' +
+          'the existing audio output may be replaced by the avatar. ' +
+          'Please start the avatar session before AgentSession.start() to avoid this.',
+      );
+    }
+    return undefined;
+  }
+
+  /**
+   * Release any resources owned by this avatar session. Default implementation is a no-op;
+   * subclasses can override to perform cleanup.
+   */
+  async aclose(): Promise<void> {}
+}

agents/src/voice/avatar/index.ts

@@ -1,4 +1,5 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
+export * from './avatar_session.js';
 export * from './datastream_io.js';

agents/src/voice/transcription/synchronizer.ts

@@ -450,6 +450,16 @@ export class TranscriptionSynchronizer {
   /** @internal */
   _impl: SegmentSynchronizerImpl;
 
+  // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 416, 434-436 lines
+  /** @internal */
+  _audioAttached: boolean = true;
+  /** @internal */
+  _textAttached: boolean = true;
+  // warn once per enabled cycle when only one of audio/text is detached; reset when
+  // the synchronizer transitions back to enabled
+  /** @internal */
+  _warnedAsymmetricDetach: boolean = false;
+
   private logger = log();
 
   constructor(
@@ -483,9 +493,25 @@ export class TranscriptionSynchronizer {
     }
 
     this._enabled = enabled;
+    // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 464-465 lines
+    if (enabled) {
+      this._warnedAsymmetricDetach = false;
+    }
     this.rotateSegment();
   }
 
+  // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 471-483 lines
+  /** @internal */
+  _onAttachmentChanged(args: { audioAttached?: boolean; textAttached?: boolean }): void {
+    if (args.audioAttached !== undefined) {
+      this._audioAttached = args.audioAttached;
+    }
+    if (args.textAttached !== undefined) {
+      this._textAttached = args.textAttached;
+    }
+    this.enabled = this._audioAttached && this._textAttached;
+  }
+
   rotateSegment() {
     if (this.closed) {
       return;
@@ -548,6 +574,19 @@ class SyncedAudioOutput extends AudioOutput {
     this.pushedDuration += frame.samplesPerChannel / frame.sampleRate;
 
     if (!this.synchronizer.enabled) {
+      // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 547-557 lines
+      if (
+        this.synchronizer._audioAttached &&
+        !this.synchronizer._textAttached &&
+        !this.synchronizer._warnedAsymmetricDetach
+      ) {
+        this.synchronizer._warnedAsymmetricDetach = true;
+        this.logger.warn(
+          'TranscriptSynchronizer text output was detached while audio output is ' +
+            'still active; transcription sync is disabled. This usually means ' +
+            'session.output.transcription was replaced after AgentSession.start().',
+        );
+      }
       return;
     }
 
@@ -607,6 +646,17 @@ class SyncedAudioOutput extends AudioOutput {
     this.synchronizer.rotateSegment();
     this.pushedDuration = 0.0;
   }
+
+  // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 618-624 lines
+  onAttached(): void {
+    super.onAttached();
+    this.synchronizer._onAttachmentChanged({ audioAttached: true });
+  }
+
+  onDetached(): void {
+    super.onDetached();
+    this.synchronizer._onAttachmentChanged({ audioAttached: false });
+  }
 }
 
 class SyncedTextOutput extends TextOutput {
@@ -626,6 +676,19 @@ class SyncedTextOutput extends TextOutput {
     const textStr = isTimedString(text) ? text.text : text;
 
     if (!this.synchronizer.enabled) {
+      // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 651-664 lines
+      if (
+        this.synchronizer._textAttached &&
+        !this.synchronizer._audioAttached &&
+        !this.synchronizer._warnedAsymmetricDetach
+      ) {
+        this.synchronizer._warnedAsymmetricDetach = true;
+        this.logger.warn(
+          'TranscriptSynchronizer audio output was detached while text output is ' +
+            'still active; transcription sync is disabled. This usually means ' +
+            'session.output.audio was replaced after AgentSession.start().',
+        );
+      }
       // pass through to the next in chain (extract string from TimedString if needed)
       await this.nextInChain.captureText(textStr);
       return;
@@ -659,4 +722,15 @@ class SyncedTextOutput extends TextOutput {
     this.capturing = false;
     this.synchronizer._impl.endTextInput();
   }
+
+  // Ref: python livekit-agents/livekit/agents/voice/transcription/synchronizer.py - 692-698 lines
+  onAttached(): void {
+    super.onAttached();
+    this.synchronizer._onAttachmentChanged({ textAttached: true });
+  }
+
+  onDetached(): void {
+    super.onDetached();
+    this.synchronizer._onAttachmentChanged({ textAttached: false });
+  }
 }

plugins/anam/src/avatar.ts

@@ -37,7 +37,7 @@ export async function mintAvatarJoinToken({
 const AVATAR_IDENTITY = 'anam-avatar-agent';
 const _AVATAR_NAME = 'anam-avatar-agent';
 
-export class AvatarSession {
+export class AvatarSession extends voice.AvatarSession {
   private sessionId?: string;
 
   constructor(
@@ -49,7 +49,9 @@ export class AvatarSession {
       avatarParticipantName?: string;
       connOptions?: APIConnectOptions;
     },
-  ) {}
+  ) {
+    super();
+  }
 
   async start(
     agentSession: voice.AgentSession,
@@ -60,6 +62,9 @@ export class AvatarSession {
       livekitApiSecret?: string;
     },
   ) {
+    // Ref: python livekit-plugins/livekit-plugins-anam/livekit/plugins/anam/avatar.py - 76 lines
+    await super.start(agentSession, room);
+
     const logger = log().child({ module: 'AnamAvatar' });
     const apiKey = this.opts.apiKey ?? process.env.ANAM_API_KEY;
     if (!apiKey) throw new AnamException('ANAM_API_KEY is required');

plugins/bey/src/avatar.ts

@@ -96,7 +96,7 @@ export interface StartOptions {
  * await avatar.start(agentSession, room);
  * ```
  */
-export class AvatarSession {
+export class AvatarSession extends voice.AvatarSession {
   private avatarId: string;
   private apiUrl: string;
   private apiKey: string;
@@ -113,6 +113,7 @@ export class AvatarSession {
    * @throws BeyException if BEY_API_KEY is not set
    */
   constructor(options: AvatarSessionOptions = {}) {
+    super();
     this.avatarId = options.avatarId || STOCK_AVATAR_ID;
     this.apiUrl = options.apiUrl || process.env.BEY_API_URL || DEFAULT_API_URL;
     this.apiKey = options.apiKey || process.env.BEY_API_KEY || '';
@@ -147,6 +148,9 @@ export class AvatarSession {
     room: Room,
     options: StartOptions = {},
   ): Promise<void> {
+    // Ref: python livekit-plugins/livekit-plugins-bey/livekit/plugins/bey/avatar.py - 78 lines
+    await super.start(agentSession, room);
+
     const livekitUrl = options.livekitUrl || process.env.LIVEKIT_URL;
     const livekitApiKey = options.livekitApiKey || process.env.LIVEKIT_API_KEY;
     const livekitApiSecret = options.livekitApiSecret || process.env.LIVEKIT_API_SECRET;

plugins/lemonslice/src/avatar.ts

@@ -122,7 +122,7 @@ export interface StartOptions {
  * await avatar.start(agentSession, room);
  * ```
  */
-export class AvatarSession {
+export class AvatarSession extends voice.AvatarSession {
   private agentId: string | null;
   private agentImageUrl: string | null;
   private agentPrompt: string | null;
@@ -143,6 +143,7 @@ export class AvatarSession {
    * @throws LemonSliceException if invalid agentId or agentImageUrl is provided, or if LemonSlice API key is not set
    */
   constructor(options: AvatarSessionOptions = {}) {
+    super();
     this.agentId = options.agentId ?? null;
     this.agentImageUrl = options.agentImageUrl ?? null;
 
@@ -191,6 +192,9 @@ export class AvatarSession {
     room: Room,
     options: StartOptions = {},
   ): Promise<string> {
+    // Ref: python livekit-plugins/livekit-plugins-lemonslice/livekit/plugins/lemonslice/avatar.py - 68 lines
+    await super.start(agentSession, room);
+
     const livekitUrl = options.livekitUrl || process.env.LIVEKIT_URL;
     const livekitApiKey = options.livekitApiKey || process.env.LIVEKIT_API_KEY;
     const livekitApiSecret = options.livekitApiSecret || process.env.LIVEKIT_API_SECRET;

plugins/trugen/src/avatar.ts

@@ -64,7 +64,7 @@ export interface StartOptions {
  * Routing agent audio output to the avatar for visual representation.
  * @public
  */
-export class AvatarSession {
+export class AvatarSession extends voice.AvatarSession {
   private avatarId: string;
   private apiUrl: string;
   private apiKey: string;
@@ -81,6 +81,7 @@ export class AvatarSession {
    * @throws TrugenException if TRUGEN_API_KEY is not set
    */
   constructor(options: AvatarSessionOptions = {}) {
+    super();
     this.avatarId = options.avatarId || DEFAULT_AVATAR_ID;
     this.apiUrl = options.apiUrl || process.env.TRUGEN_API_URL || DEFAULT_API_URL;
     this.apiKey = options.apiKey || process.env.TRUGEN_API_KEY || '';
@@ -112,6 +113,9 @@ export class AvatarSession {
     room: Room,
     options: StartOptions = {},
   ): Promise<void> {
+    // Ref: python livekit-plugins/livekit-plugins-trugen/livekit/plugins/trugen/avatar.py - 83 lines
+    await super.start(agentSession, room);
+
     const livekitUrl = options.livekitUrl || process.env.LIVEKIT_URL;
     const livekitApiKey = options.livekitApiKey || process.env.LIVEKIT_API_KEY;
     const livekitApiSecret = options.livekitApiSecret || process.env.LIVEKIT_API_SECRET;