diff --git a/.gitignore b/.gitignore
index 2cf45d00..e4d5a948 100644
--- a/.gitignore
+++ b/.gitignore
@@ -199,3 +199,8 @@ ollama.pid
# Local design reference library (cloned, not vendored)
.reference/
+
+# Swift Package Manager build artifacts for the diarize-sidecar helper
+# (fetched FluidAudio checkout + compiled binaries) — scripts/build-diarize-sidecar.sh
+# regenerates this; only diarize-sidecar/{Package.swift,Package.resolved,Sources/} are tracked.
+diarize-sidecar/.build/
diff --git a/CLAUDE.md b/CLAUDE.md
index 8af92f9e..4962c477 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -13,7 +13,7 @@ The app is a thin Electron shell over a PyInstaller-bundled Python CLI. There is
- **Python CLI (`simple_recorder.py`, ~2.9k lines, ~60 click commands)** is the single entry point bundled by `stenoai.spec`. Sub-modules in `src/`: `audio_recorder` (sounddevice), `transcriber` (pywhispercpp), `summarizer` (Ollama HTTP client), `ollama_manager` (lifecycle of the bundled `ollama serve`), `config` (JSON-backed user settings + model registry), `folders`, `models`, `whisper_models`.
- **State across CLI invocations** is persisted to `recorder_state.json` and similar small JSON files — there is no daemon. Long-running recordings are a `record` subprocess kept alive by the Electron main process.
- **User data lives in `~/Library/Application Support/stenoai/`** (`recordings/`, `transcripts/`, `output/`), resolved via `src.config.get_data_dirs()`. Repo-root `recordings/`/`transcripts/`/`output/` dirs are dev-only scratch.
-- **Bundled binaries (`bin/`)**: Ollama + ffmpeg, downloaded by `scripts/download-ollama.sh`. PyInstaller copies them into `dist/stenoai/ollama/` and `dist/stenoai/ffmpeg`. Electron then re-bundles `dist/stenoai/` as an `extraResource`.
+- **Bundled binaries (`bin/`)**: Ollama + ffmpeg, downloaded by `scripts/download-ollama.sh`. PyInstaller copies them into `dist/stenoai/ollama/` and `dist/stenoai/ffmpeg`. Electron then re-bundles `dist/stenoai/` as an `extraResource`. `bin/steno-diarize` (macOS only) is a separate Swift/CoreML sidecar built by `scripts/build-diarize-sidecar.sh` — see "Speaker diarization" below.
- **Deep links**: app registers the `stenoai://` URL scheme. Handler logic is in `app/main.js` near `SHORTCUT_PROTOCOL`. Used by macOS Shortcuts: `stenoai://record/start?name=...` and `stenoai://record/stop`.
## Development Commands
@@ -39,6 +39,28 @@ The Electron build pulls the bundled backend from `../dist/stenoai` via `extraRe
For setup from a clean checkout, see `CONTRIBUTING.md` and `README.md`.
+### Speaker diarization (macOS only)
+Per-channel acoustic speaker diarization (splitting multiple speakers sharing
+one side of a call — e.g. two people around one mic, or multiple remote
+participants on system audio) runs through `bin/steno-diarize`, a Swift/
+CoreML sidecar (`diarize-sidecar/`) wrapping FluidAudio's Sortformer
+diarizer, invoked from Python (`src.transcriber._run_steno_diarize`) — never
+from Electron, since the batch pipeline is entirely Python-orchestrated.
+Build it *before* `pyinstaller stenoai.spec`, same as `download-ollama.sh`:
+
+```
+scripts/build-diarize-sidecar.sh # outputs bin/steno-diarize
+scripts/download-ollama.sh
+pyinstaller stenoai.spec --noconfirm
+```
+
+`stenoai.spec` bundles `bin/steno-diarize` only when it exists and only on
+macOS (`_IS_DARWIN`), so skipping the build step is safe — the app falls
+back to the legacy channel-only "You"/"Others" labeling whenever the binary
+or a diarization run is unavailable (missing binary, timeout, bad output);
+this never fails a meeting. Windows/Linux never get acoustic diarization —
+`_resolve_steno_diarize()` returns `None` immediately off-darwin.
+
### End-to-end tests (Playwright)
The e2e suite drives the **real Electron app** (real window, real clicks) to catch
full-app regressions like the org-provider reset before they reach users. It lives
@@ -97,7 +119,14 @@ overrides an agent's own test-level defaults.
`setup-check.t2` (the setup-wizard allGood + checks contract) (all model-free,
run in `t2-macos` /
`t2-windows`); `transcription-pipeline.t2` and `honest-failure.t2` (tagged
- `@pipeline`, run in `t2-pipeline-macos` / `t2-pipeline-windows`). Engine selection
+ `@pipeline`, run in `t2-pipeline-macos` / `t2-pipeline-windows`), and
+ `speaker-diarization.t2` (also `@pipeline`, macOS-only — skips loudly off-darwin
+ and when macOS's `say` TTS is unavailable — synthesizes real speech via `say` so
+ Parakeet/whisper.cpp produce real ASR segments, points
+ `STENOAI_DIARIZE_SIDECAR_PATH` at a fixture script returning fixed 2-speaker JSON,
+ and asserts the saved transcript's per-channel "You"/"Speaker 2"/"Others" labeling
+ and cross-channel numbering; the real `steno-diarize`/Sortformer binary itself was
+ validated directly against real recordings rather than in CI). Engine selection
for `@pipeline` specs is shared via `e2e/fixtures/engine.ts`; model-free T2 setup
helpers (deterministic recording config + seeded meeting summaries) live in
`e2e/fixtures/user-config.ts`. The core-loop specs drive the preload IPC bridge and
diff --git a/app/main.js b/app/main.js
index 82d35e97..7b732eab 100644
--- a/app/main.js
+++ b/app/main.js
@@ -7098,6 +7098,13 @@ function logPipelineStdoutLine(line, source) {
processingLog.logLine(source, l);
return;
}
+ if (l.startsWith('PROGRESS:diarize:')) {
+ // Only :start/:done markers exist on this pipeline (rare, at most twice
+ // per channel) -- no per-chunk flood risk, so no throttle needed unlike
+ // HEARTBEAT above.
+ processingLog.logLine(source, l);
+ return;
+ }
if (
l.startsWith('TRANSCRIPTION_COMPLETE') ||
l.startsWith('TRANSCRIPTION_FAILED') ||
diff --git a/app/renderer/src/components/TranscriptPanel.tsx b/app/renderer/src/components/TranscriptPanel.tsx
index 1bfe659f..78fe9362 100644
--- a/app/renderer/src/components/TranscriptPanel.tsx
+++ b/app/renderer/src/components/TranscriptPanel.tsx
@@ -133,8 +133,10 @@ function TranscriptRow({ segment, highlight }: { segment: Segment; highlight: st
// showing their content as un-bubbled plain text (or worse, on the
// "Others" side) reads as wrong. Granola takes the same charitable
// default — when in doubt, attribute to the mic owner. Explicit
- // `Others` markers still render as grey/left.
- const isYou = segment.speaker !== 'Others';
+ // `Others` markers and acoustically-diarized `Speaker N` markers
+ // (transcriber.py's _resolve_speaker_placeholders) both render as
+ // grey/left — only an exact "You" (or no marker at all) is "self".
+ const isYou = segment.speaker === 'You' || segment.speaker == null;
return (
{segment.timestamp && (
diff --git a/app/renderer/src/lib/transcriptSegments.test.ts b/app/renderer/src/lib/transcriptSegments.test.ts
index 23887ae0..60f9afee 100644
--- a/app/renderer/src/lib/transcriptSegments.test.ts
+++ b/app/renderer/src/lib/transcriptSegments.test.ts
@@ -40,6 +40,18 @@ describe('parseTranscript — diarised', () => {
expect(segs[0].text).toContain('line two');
expect(segs[1].timestamp).toBe('00:05');
});
+
+ test('parses acoustically-diarized "Speaker N" labels alongside You/Others', () => {
+ const segs = parseTranscript(
+ '[00:00] [You] Hello\n\n[00:05] [Speaker 2] Hi there\n\n[00:10] [Others] Welcome',
+ true,
+ );
+ expect(segs).toEqual([
+ { speaker: 'You', text: 'Hello', timestamp: '00:00' },
+ { speaker: 'Speaker 2', text: 'Hi there', timestamp: '00:05' },
+ { speaker: 'Others', text: 'Welcome', timestamp: '00:10' },
+ ]);
+ });
});
describe('parseTranscript — non-diarised', () => {
diff --git a/app/renderer/src/lib/transcriptSegments.ts b/app/renderer/src/lib/transcriptSegments.ts
index 6dfb182f..aaf1a54a 100644
--- a/app/renderer/src/lib/transcriptSegments.ts
+++ b/app/renderer/src/lib/transcriptSegments.ts
@@ -3,7 +3,10 @@
// unit-tested in isolation, and so the live dock could share it later.
export interface Segment {
- speaker: 'You' | 'Others' | null;
+ /** 'You' / 'Others' for the channel-only fallback, or 'Speaker N' for an
+ * acoustically-diarized secondary speaker on a channel (see
+ * transcriber.py's _resolve_speaker_placeholders). */
+ speaker: string | null;
text: string;
/** `MM:SS` / `H:MM:SS` offset parsed from a diarised line's leading
* `[MM:SS]` marker (transcriber.py writes it). Absent on older transcripts
@@ -11,12 +14,13 @@ export interface Segment {
timestamp?: string;
}
-// A diarised line: an optional `[MM:SS]` / `[H:MM:SS]` timestamp, then the
-// `[You]`/`[Others]` speaker marker, then the text up to the next marker (or
-// end). Matched globally rather than split so the optional timestamp stays
-// attached to its own segment instead of trailing the previous one.
+// A diarised line: an optional `[MM:SS]` / `[H:MM:SS]` timestamp, then a
+// `[You]` / `[Others]` / `[Speaker N]` speaker marker, then the text up to
+// the next marker (or end). Matched globally rather than split so the
+// optional timestamp stays attached to its own segment instead of trailing
+// the previous one.
const DIARISED_SEGMENT_RE =
- /(?:\[(\d{1,3}:\d{2}(?::\d{2})?)\]\s*)?\[(You|Others)\]\s*([\s\S]*?)(?=(?:\[\d{1,3}:\d{2}(?::\d{2})?\]\s*)?\[(?:You|Others)\]|$)/g;
+ /(?:\[(\d{1,3}:\d{2}(?::\d{2})?)\]\s*)?\[([^\]]+)\]\s*([\s\S]*?)(?=(?:\[\d{1,3}:\d{2}(?::\d{2})?\]\s*)?\[[^\]]+\]|$)/g;
export function parseTranscript(text: string, isDiarised: boolean): Segment[] {
if (isDiarised) {
@@ -24,7 +28,7 @@ export function parseTranscript(text: string, isDiarised: boolean): Segment[] {
for (const m of text.matchAll(DIARISED_SEGMENT_RE)) {
const body = m[3].trim();
if (!body) continue;
- segments.push({ speaker: m[2] as 'You' | 'Others', text: body, timestamp: m[1] });
+ segments.push({ speaker: m[2], text: body, timestamp: m[1] });
}
return segments;
}
diff --git a/app/renderer/src/routes/Processing.tsx b/app/renderer/src/routes/Processing.tsx
index 91c82d6d..668ad635 100644
--- a/app/renderer/src/routes/Processing.tsx
+++ b/app/renderer/src/routes/Processing.tsx
@@ -16,7 +16,7 @@ import { getLiveDraft, useLiveDraftStore } from '@/hooks/liveDraftStore';
import { ipc } from '@/lib/ipc';
import { stripReasoning } from '@/lib/markdown';
-type ProcessingStage = 'transcribing' | 'summarizing' | 'finalizing' | 'error';
+type ProcessingStage = 'transcribing' | 'diarizing' | 'summarizing' | 'finalizing' | 'error';
// Shared flag so the sibling ProcessingDock (the bottom "Processing" chip,
// rendered by App while on this route) can hide once the watchdog concludes
@@ -33,6 +33,7 @@ const useProcessingWatchdogStore = create<{
const STAGE_LABEL: Record
= {
transcribing: 'Analyzing transcript',
+ diarizing: 'Identifying speakers',
summarizing: 'Generating notes',
finalizing: 'Almost done…',
error: 'Couldn’t process this recording.',
@@ -59,6 +60,13 @@ const STAGE_LABEL: Record = {
const WATCHDOG_TICK_MS = 1500;
const WATCHDOG_IDLE_TICKS = 8;
+function formatElapsedSeconds(totalSeconds: number): string {
+ if (totalSeconds < 60) return `${totalSeconds}s`;
+ const m = Math.floor(totalSeconds / 60);
+ const s = totalSeconds % 60;
+ return `${m}m ${s}s`;
+}
+
export function Processing() {
const navigate = useNavigate();
const recording = useRecording();
@@ -143,6 +151,21 @@ export function Processing() {
setRetryError(null);
}
+ // Diarization's segmentation pass has no per-chunk checkpoint to report
+ // progress from -- unlike embedding extraction (not part of this branch's
+ // sidecar), so there is nothing to turn into a percentage here. A
+ // client-side elapsed-time ticker is the only way to show visible motion
+ // during that stretch without a backend change.
+ const diarizeTimerRef = React.useRef | null>(null);
+ const diarizeStartedAtRef = React.useRef(null);
+ const clearDiarizeTimer = React.useCallback(() => {
+ if (diarizeTimerRef.current) {
+ clearInterval(diarizeTimerRef.current);
+ diarizeTimerRef.current = null;
+ }
+ }, []);
+ React.useEffect(() => () => clearDiarizeTimer(), [clearDiarizeTimer]);
+
// Buffer streamed chunks and flush at most every 50ms (~20fps). At a
// typical token rate of 30-60 tokens/sec, this batches ~3 tokens per
// commit which keeps the UI smooth without re-parsing the entire markdown
@@ -172,8 +195,9 @@ export function Processing() {
// "saw activity" flag (which would hide a real no-job dead-end).
if (e.summaryFile) return;
if (activeSession && e.sessionName !== activeSession) return;
+ clearDiarizeTimer();
pendingChunkRef.current += e.chunk;
- setStage((s) => (s === 'transcribing' ? 'summarizing' : s));
+ setStage((s) => (s === 'transcribing' || s === 'diarizing' ? 'summarizing' : s));
if (!flushTimerRef.current) {
flushTimerRef.current = setTimeout(flushPending, 50);
}
@@ -188,12 +212,16 @@ export function Processing() {
// fresh-recording job.
if (e.summaryFile) return;
if (activeSession && e.sessionName !== activeSession) return;
+ clearDiarizeTimer();
+ setChunkProgress(null);
setStage((s) => (s === 'error' ? s : 'finalizing'));
}),
ipc().on.processingComplete((e) => {
if (activeSession && e.sessionName !== activeSession) return;
if (!e.success) {
setRetryAudioFile(e.audioFile ?? null);
+ clearDiarizeTimer();
+ setChunkProgress(null);
setStage('error');
return;
}
@@ -214,20 +242,41 @@ export function Processing() {
// DIFFERENT meeting emits summaryFile-scoped progress — ignore those so
// another meeting's "Summarizing part N" can't hijack this screen's stage.
if (e.summaryFile) return;
- const raw = e.line.replace(/^PROGRESS:summarize:/, '');
- if (raw === 'reducing') {
- setChunkProgress('Merging summaries…');
- } else {
- const [step, total] = raw.split('/').map(Number);
- if (!Number.isNaN(step) && !Number.isNaN(total)) {
- setChunkProgress(`Summarizing part ${step} of ${total}…`);
+ if (e.line.startsWith('PROGRESS:summarize:')) {
+ const raw = e.line.slice('PROGRESS:summarize:'.length);
+ if (raw === 'reducing') {
+ setChunkProgress('Merging summaries…');
+ } else {
+ const [step, total] = raw.split('/').map(Number);
+ if (!Number.isNaN(step) && !Number.isNaN(total)) {
+ setChunkProgress(`Summarizing part ${step} of ${total}…`);
+ }
+ }
+ setStage((s) => (s === 'transcribing' || s === 'diarizing' ? 'summarizing' : s));
+ } else if (e.line.startsWith('PROGRESS:diarize:')) {
+ const rest = e.line.slice('PROGRESS:diarize:'.length);
+ const [label, kind] = rest.split(':');
+ if (kind === 'start') {
+ setStage((s) => (s === 'transcribing' ? 'diarizing' : s));
+ clearDiarizeTimer();
+ diarizeStartedAtRef.current = Date.now();
+ setChunkProgress(`Diarizing ${label} channel…`);
+ diarizeTimerRef.current = setInterval(() => {
+ if (diarizeStartedAtRef.current === null) return;
+ const elapsedS = Math.floor((Date.now() - diarizeStartedAtRef.current) / 1000);
+ setChunkProgress(`Diarizing ${label} channel… (${formatElapsedSeconds(elapsedS)})`);
+ }, 1000);
+ } else if (kind === 'done') {
+ // Stop the elapsed ticker; no other UI action needed -- the
+ // next channel's :start, or the first summarize: chunk,
+ // supersedes this sub-label shortly after.
+ clearDiarizeTimer();
}
}
- setStage((s) => (s === 'transcribing' ? 'summarizing' : s));
}),
];
return () => offs.forEach((fn) => fn());
- }, [activeSession, updateMeeting]);
+ }, [activeSession, updateMeeting, clearDiarizeTimer]);
// Watchdog. Any real processing activity — a streamed chunk, chunk progress,
// a stage move past 'transcribing' (summaryComplete/processingComplete) —
@@ -523,10 +572,12 @@ function StageCard({
style={{ color: 'var(--fg-2)' }}
/>
- {chunkProgress && stage === 'summarizing' ? chunkProgress : STAGE_LABEL[stage]}
+ {chunkProgress && stage !== 'finalizing' && stage !== 'error' ? chunkProgress : STAGE_LABEL[stage]}
diff --git a/diarize-sidecar/Package.resolved b/diarize-sidecar/Package.resolved
new file mode 100644
index 00000000..cefa925f
--- /dev/null
+++ b/diarize-sidecar/Package.resolved
@@ -0,0 +1,14 @@
+{
+ "pins" : [
+ {
+ "identity" : "fluidaudio",
+ "kind" : "remoteSourceControl",
+ "location" : "https://github.com/FluidInference/FluidAudio.git",
+ "state" : {
+ "revision" : "7f963cdc43ba89c5993654f1e138047d517a818d",
+ "version" : "0.15.2"
+ }
+ }
+ ],
+ "version" : 2
+}
diff --git a/diarize-sidecar/Package.swift b/diarize-sidecar/Package.swift
new file mode 100644
index 00000000..4fc79c2f
--- /dev/null
+++ b/diarize-sidecar/Package.swift
@@ -0,0 +1,19 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+ name: "diarize-sidecar",
+ platforms: [.macOS(.v14)],
+ dependencies: [
+ .package(url: "https://github.com/FluidInference/FluidAudio.git", from: "0.15.2"),
+ ],
+ targets: [
+ .executableTarget(
+ name: "diarize-sidecar",
+ dependencies: [
+ .product(name: "FluidAudio", package: "FluidAudio"),
+ ],
+ path: "Sources"
+ ),
+ ]
+)
\ No newline at end of file
diff --git a/diarize-sidecar/Sources/main.swift b/diarize-sidecar/Sources/main.swift
new file mode 100644
index 00000000..ef24431d
--- /dev/null
+++ b/diarize-sidecar/Sources/main.swift
@@ -0,0 +1,232 @@
+// diarize-sidecar — offline speaker diarization via FluidAudio's Sortformer.
+//
+// Usage:
+// steno-diarize
+//
+// Sortformer has a fixed 4-speaker-slot architecture (SortformerConfig.numSpeakers
+// is hardcoded to 4) — there is no speaker-count hint to pass.
+//
+// Output (stdout): one JSON line on success
+// [{"speakerId":"SPEAKER_0","start":0.0,"end":3.2}, ...]
+//
+// Exit 0 on success, 1 on failure (error written to stderr).
+// stdout is unbuffered so the parent receives the line immediately.
+//
+// Audio loading avoids all CoreAudio file APIs (which fail with error
+// 1954115647 when spawned from a PyInstaller bundle) by delegating to
+// ffmpeg, which uses its own decoders entirely outside CoreAudio. This is
+// also why we call SortformerDiarizer.processComplete(_:sourceSampleRate:)
+// with raw samples rather than the processComplete(audioFileURL:) overload —
+// that overload internally uses AudioConverter.resampleAudioFile, which
+// calls AVAudioFile(forReading:) and would reintroduce the same crash.
+
+import CoreML
+import Foundation
+import FluidAudio
+
+setbuf(stdout, nil)
+
+// Segments shorter than this are spurious artifacts (an ~80ms noise-blip
+// pattern observed empirically against real meeting audio), dropped before
+// emitting rather than surfaced as a phantom speaker turn.
+let minSegmentDurationSeconds: Float = 0.25
+
+// SortformerConfig.highContextV2's chunk loader (SortformerFeatureLoader)
+// requires a full chunkLen+rightContext window -- 340+40 frames at 0.08s
+// each, 30.4s -- before it emits ANYTHING; audio shorter than that gets
+// ZERO segments, not degraded ones (confirmed empirically: a 12.25s test
+// clip returned [] with highContextV2). This threshold gives real margin
+// above that hard minimum before switching away from the always-safe
+// .default config.
+let sortformerHighContextMinDuration: Double = 90.0
+
+func fail(_ message: String) -> Never {
+ fputs("steno-diarize error: \(message)\n", stderr)
+ exit(1)
+}
+
+// Compute-unit override, primarily for one-off bulk backfill runs where
+// throughput matters more than the power/thermal cost of spinning up the
+// GPU (unlike live recording, where the default below is deliberately
+// power-efficient). Unset -> unchanged default (.cpuAndNeuralEngine, see
+// the call sites below for why that's forced rather than left at .all).
+// STENOAI_DIARIZE_COMPUTE_UNITS: "all" | "cpuAndGPU" | "cpuOnly" |
+// "cpuAndNeuralEngine" (default).
+func resolveComputeUnits() -> MLComputeUnits {
+ switch ProcessInfo.processInfo.environment["STENOAI_DIARIZE_COMPUTE_UNITS"] {
+ case "all": return .all
+ case "cpuAndGPU": return .cpuAndGPU
+ case "cpuOnly": return .cpuOnly
+ default: return .cpuAndNeuralEngine
+ }
+}
+
+guard CommandLine.arguments.count == 2 else {
+ fail("usage: steno-diarize ")
+}
+
+let inputPath = CommandLine.arguments[1]
+
+guard FileManager.default.fileExists(atPath: inputPath) else {
+ fail("file not found: \(inputPath)")
+}
+
+// Locate ffmpeg. The binary lives next to steno-diarize in the bundle;
+// fall back to common Homebrew / system paths for terminal use.
+func findFfmpeg() -> String? {
+ let execDir = URL(fileURLWithPath: CommandLine.arguments[0])
+ .resolvingSymlinksInPath()
+ .deletingLastPathComponent()
+ .path
+ let candidates = [
+ "\(execDir)/ffmpeg",
+ "/opt/homebrew/bin/ffmpeg",
+ "/usr/local/bin/ffmpeg",
+ "/usr/bin/ffmpeg",
+ ]
+ return candidates.first { FileManager.default.isExecutableFile(atPath: $0) }
+}
+
+// Decode any audio format to 16 kHz mono Float32 using ffmpeg, writing
+// output to a temp file.
+//
+// Uses terminationHandler + CheckedContinuation so the Task suspends
+// instead of blocking a thread. ffmpeg stdin is redirected to /dev/null
+// to prevent it from blocking on an inherited terminal.
+func loadSamplesViaFfmpeg(path: String) async throws -> [Float] {
+ guard let ffmpegPath = findFfmpeg() else {
+ throw NSError(domain: "steno-diarize", code: 1,
+ userInfo: [NSLocalizedDescriptionKey: "ffmpeg not found"])
+ }
+ let tmpURL = URL(fileURLWithPath: NSTemporaryDirectory())
+ .appendingPathComponent("steno-diarize-\(UUID().uuidString).f32le")
+ defer { try? FileManager.default.removeItem(at: tmpURL) }
+
+ let process = Process()
+ process.executableURL = URL(fileURLWithPath: ffmpegPath)
+ process.arguments = [
+ "-loglevel", "error",
+ "-i", path,
+ "-ar", "16000",
+ "-ac", "1",
+ "-f", "f32le",
+ "-y",
+ tmpURL.path,
+ ]
+ // Redirect stdin to /dev/null so ffmpeg never blocks waiting for
+ // interactive input on an inherited terminal.
+ process.standardInput = FileHandle.nullDevice
+ // Redirect ffmpeg stderr to /dev/null. When steno-diarize is spawned by
+ // Python with capture_output=True, our own stderr is a pipe that Python
+ // only drains after we exit. ffmpeg inherits that pipe and fills the
+ // buffer with progress/profiling lines, causing it to block — which means
+ // terminationHandler never fires and steno-diarize hangs. /dev/null
+ // prevents the buffer fill; ffmpeg errors are still visible via exit status.
+ process.standardError = FileHandle(forWritingAtPath: "/dev/null")
+
+ try await withCheckedThrowingContinuation { (cont: CheckedContinuation) in
+ process.terminationHandler = { p in
+ if p.terminationStatus == 0 {
+ cont.resume()
+ } else {
+ cont.resume(throwing: NSError(
+ domain: "steno-diarize", code: Int(p.terminationStatus),
+ userInfo: [NSLocalizedDescriptionKey: "ffmpeg exited \(p.terminationStatus)"]))
+ }
+ }
+ do {
+ try process.run()
+ } catch {
+ cont.resume(throwing: error)
+ }
+ }
+
+ let rawData = try Data(contentsOf: tmpURL, options: .mappedIfSafe)
+
+ guard !rawData.isEmpty else {
+ throw NSError(domain: "steno-diarize", code: 3,
+ userInfo: [NSLocalizedDescriptionKey: "ffmpeg produced no output"])
+ }
+
+ // f32le on Apple Silicon (little-endian) — reinterpret bytes as Float directly.
+ return rawData.withUnsafeBytes { ptr in
+ Array(ptr.bindMemory(to: Float.self))
+ }
+}
+
+// Keep the main run loop alive so dispatch sources (including the
+// process-exit source that drives terminationHandler) can fire normally.
+// sema.wait() blocks the main thread, which prevents dispatch delivery
+// and causes terminationHandler to never fire.
+Task {
+ do {
+ let samples = try await loadSamplesViaFfmpeg(path: inputPath)
+
+ // .cpuAndNeuralEngine forces genuine ANE execution — the default
+ // .all silently routes Sortformer to GPU instead (confirmed via
+ // Activity Monitor during evaluation). resolveComputeUnits() keeps
+ // that as the default but allows STENOAI_DIARIZE_COMPUTE_UNITS=all
+ // (or =cpuAndGPU) to opt into GPU for one-off bulk backfill runs
+ // where throughput matters more than the live-recording-path's
+ // power/thermal efficiency.
+ //
+ // Sortformer config: this app only ever diarizes a fully-recorded,
+ // already-finished channel (no live/streaming diarization exists
+ // yet), so .default's low-latency 0.48s-per-invocation chunking
+ // (tuned for real-time responsiveness this app has no use for)
+ // costs ~56x more CoreML invocations than .highContextV2's
+ // 27.2s-per-invocation chunking on the same audio (measured:
+ // ~22,500 vs ~400 invocations for a 3-hour recording) — but
+ // highContextV2 needs a full ~30.4s window before it emits
+ // anything at all, so it's only used once the recording is
+ // comfortably longer than that (see sortformerHighContextMinDuration).
+ // V2, not V2.1: FluidAudio's own docs note V2.1 "may degrade when
+ // many speakers are talking simultaneously" — a real risk given
+ // this app's crosstalk/echo findings from earlier this session.
+ // Both the model-loading config below AND SortformerDiarizer's own
+ // config must match — its internal chunk/fifo/spkcache buffers are
+ // sized from whatever config it's constructed with, independent of
+ // which model weights get loaded.
+ let durationSeconds = Double(samples.count) / 16000.0
+ let sortformerConfig: SortformerConfig =
+ durationSeconds >= sortformerHighContextMinDuration ? .highContextV2 : .default
+ let models = try await SortformerModels.loadFromHuggingFace(
+ config: sortformerConfig,
+ computeUnits: resolveComputeUnits()
+ )
+ let diarizer = SortformerDiarizer(config: sortformerConfig)
+ diarizer.initialize(models: models)
+
+ let timeline = try diarizer.processComplete(samples, sourceSampleRate: nil)
+
+ struct Segment: Encodable {
+ let speakerId: String
+ let start: Double
+ let end: Double
+ }
+
+ let segments = timeline.speakers.values
+ .flatMap { $0.finalizedSegments }
+ .filter { $0.duration >= minSegmentDurationSeconds }
+ .map { seg in
+ Segment(
+ speakerId: "SPEAKER_\(seg.speakerIndex)",
+ start: Double(seg.startTime),
+ end: Double(seg.endTime)
+ )
+ }
+ .sorted { $0.start < $1.start }
+
+ let encoded = try JSONEncoder().encode(segments)
+ guard let line = String(data: encoded, encoding: .utf8) else {
+ exit(1)
+ }
+ print(line)
+ exit(0)
+ } catch {
+ fputs("steno-diarize error: \(error)\n", stderr)
+ exit(1)
+ }
+}
+
+RunLoop.main.run()
diff --git a/docs/faq.mdx b/docs/faq.mdx
index ebc58707..a7c215ee 100644
--- a/docs/faq.mdx
+++ b/docs/faq.mdx
@@ -90,7 +90,7 @@ Steno can capture system audio -- the audio playing through your Mac's speakers
-Yes. Steno records from your Mac's microphone, which will pick up voices in the room. For best results, place your laptop centrally. In-person recordings are mic-only, so they have no `[You]` / `[Others]` speaker labels.
+Yes. Steno records from your Mac's microphone, which will pick up voices in the room. For best results, place your laptop centrally. Steno separates up to four distinct voices per recording channel, so an in-person meeting of four or fewer people gets per-speaker labels; beyond that, additional speakers are merged into the four it detects.
diff --git a/docs/features/recording.mdx b/docs/features/recording.mdx
index 391f3c1a..fd36730f 100644
--- a/docs/features/recording.mdx
+++ b/docs/features/recording.mdx
@@ -28,7 +28,7 @@ After this one-time setup, the toggle stays on for future recordings until you t
## Speaker labels
-When Steno captures system audio, it labels lines in the transcript as `[You]` (microphone audio) or `[Others]` (system audio). This makes it easier to follow who said what in a virtual meeting.
+Steno labels transcript lines by who's speaking. When system audio is on, lines are split into `[You]` (microphone audio) and `[Others]` (system audio) by channel. Within either channel, Steno also separates distinct voices acoustically, so an in-person meeting with several people sharing your Mac's microphone, or several remote participants on the system-audio side, get their own `[Speaker 2]`, `[Speaker 3]`, and so on. Each channel can distinguish up to four voices this way; additional speakers beyond that are merged into the four it detects.
Speaker labels appear automatically -- no configuration is needed.
diff --git a/e2e/fixtures/say-stereo-wav.ts b/e2e/fixtures/say-stereo-wav.ts
new file mode 100644
index 00000000..3045c858
--- /dev/null
+++ b/e2e/fixtures/say-stereo-wav.ts
@@ -0,0 +1,127 @@
+// Synthesizes real (non-silent, transcribable) speech into a stereo WAV
+// using macOS's built-in `say` TTS, for e2e specs that need genuine ASR
+// output rather than the sine-tone fixture in make-wav.js. A sine tone is
+// non-speech and Parakeet/whisper.cpp both correctly return empty text for
+// it (verified — that's what the plumbing-only @pipeline spec relies on),
+// so it can never exercise anything downstream of real transcript segments,
+// like the per-channel speaker-diarization labeling path.
+//
+// macOS-only: `say` doesn't exist on Windows/Linux. Callers must gate on
+// isSayAvailable() (and process.platform) and skip loudly otherwise.
+import { execFileSync } from 'child_process';
+import { mkdtempSync, readFileSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import path from 'path';
+
+// `say -o out.wav --data-format=LEI16@16000` writes a real (non-canonical)
+// WAV header — macOS prepends a JUNK chunk before `fmt `/`data`, so the
+// data chunk is not at the fixed 44-byte offset make-wav.js assumes.
+// Locate it by tag instead of guessing an offset.
+function readPcm16Mono(filePath: string): Buffer {
+ const buf = readFileSync(filePath);
+ const dataTag = buf.indexOf('data', 12);
+ if (dataTag < 0) throw new Error(`no data chunk found in ${filePath}`);
+ const size = buf.readUInt32LE(dataTag + 4);
+ return buf.subarray(dataTag + 8, dataTag + 8 + size);
+}
+
+function synthesize(text: string, destPath: string): Buffer {
+ execFileSync('say', ['-o', destPath, '--data-format=LEI16@16000', text]);
+ return readPcm16Mono(destPath);
+}
+
+// A real sentence should take at least this long to speak. Guards against a
+// `say` that "works" (exits 0, produces a well-formed WAV) but synthesizes
+// essentially nothing -- observed on GitHub-hosted macOS runners: `say -v ?`
+// (listing voices) exits 0 even when actual synthesis produces ~170 bytes of
+// near-silent PCM (~5ms), presumably because the runner image ships the
+// `say` binary without voice assets. Probing `-v ?` alone is a false
+// positive; this measures what `say` actually wrote.
+const MIN_SAY_DURATION_SECONDS = 1.0;
+
+let sayAvailable: boolean | undefined;
+
+export function isSayAvailable(): boolean {
+ if (sayAvailable === undefined) {
+ try {
+ const dir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-say-probe-'));
+ const pcm = synthesize('Testing one two three.', path.join(dir, 'probe.wav'));
+ const seconds = pcm.length / 2 / 16000;
+ sayAvailable = seconds >= MIN_SAY_DURATION_SECONDS;
+ } catch {
+ sayAvailable = false;
+ }
+ }
+ return sayAvailable;
+}
+
+function silencePcm(seconds: number): Buffer {
+ return Buffer.alloc(Math.round(seconds * 16000) * 2);
+}
+
+function writeStereoWav(destPath: string, leftPcm: Buffer, rightPcm: Buffer): void {
+ const frames = Math.max(leftPcm.length, rightPcm.length) / 2;
+ const bytesPerSample = 2;
+ const channels = 2;
+ const dataBytes = frames * channels * bytesPerSample;
+ const out = Buffer.alloc(44 + dataBytes);
+
+ out.write('RIFF', 0);
+ out.writeUInt32LE(36 + dataBytes, 4);
+ out.write('WAVE', 8);
+ out.write('fmt ', 12);
+ out.writeUInt32LE(16, 16);
+ out.writeUInt16LE(1, 20);
+ out.writeUInt16LE(channels, 22);
+ out.writeUInt32LE(16000, 24);
+ out.writeUInt32LE(16000 * channels * bytesPerSample, 28);
+ out.writeUInt16LE(channels * bytesPerSample, 32);
+ out.writeUInt16LE(16, 34);
+ out.write('data', 36);
+ out.writeUInt32LE(dataBytes, 40);
+
+ for (let i = 0; i < frames; i++) {
+ const l = i * 2 < leftPcm.length ? leftPcm.readInt16LE(i * 2) : 0;
+ const r = i * 2 < rightPcm.length ? rightPcm.readInt16LE(i * 2) : 0;
+ out.writeInt16LE(l, 44 + i * 4);
+ out.writeInt16LE(r, 44 + i * 4 + 2);
+ }
+ writeFileSync(destPath, out);
+}
+
+export interface StereoSpeechResult {
+ /** Seconds into the mic channel's own timeline marking the midpoint of
+ * the silence gap between micUtteranceA and micUtteranceB — the point
+ * a fixture diarizer's two speaker-cluster boundary should sit at so
+ * each utterance's ASR sentence lands in a different cluster. */
+ micBoundarySeconds: number;
+}
+
+/**
+ * Builds a stereo WAV (left = mic, right = system — the layout
+ * transcribe_diarised splits) where the mic channel contains TWO short
+ * utterances separated by exactly 1s of real digital silence (so real ASR
+ * segments them into two distinct sentences with a known gap), and the
+ * system channel contains one short utterance. Real speaking-rate timing
+ * varies by voice/macOS version, so the returned boundary is computed from
+ * the ACTUALLY measured synthesized audio, not guessed.
+ */
+export function makeStereoSpeechWav(
+ destPath: string,
+ opts: { micUtteranceA: string; micUtteranceB: string; systemUtterance: string },
+): StereoSpeechResult {
+ const dir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-say-'));
+
+ const uttA = synthesize(opts.micUtteranceA, path.join(dir, 'mic_a.wav'));
+ const gapSeconds = 1.0;
+ const uttB = synthesize(opts.micUtteranceB, path.join(dir, 'mic_b.wav'));
+ const micPcm = Buffer.concat([uttA, silencePcm(gapSeconds), uttB]);
+
+ const sysUtt = synthesize(opts.systemUtterance, path.join(dir, 'sys.wav'));
+ const sysPcm = Buffer.concat([sysUtt, silencePcm(Math.max(0, micPcm.length / 2 / 16000 - sysUtt.length / 2 / 16000))]);
+
+ writeStereoWav(destPath, micPcm, sysPcm);
+
+ const durA = uttA.length / 2 / 16000;
+ return { micBoundarySeconds: durA + gapSeconds / 2 };
+}
diff --git a/e2e/specs/processing-stages.t1.spec.ts b/e2e/specs/processing-stages.t1.spec.ts
new file mode 100644
index 00000000..52827ed0
--- /dev/null
+++ b/e2e/specs/processing-stages.t1.spec.ts
@@ -0,0 +1,128 @@
+import { test, expect } from '../fixtures/electron';
+import type { ElectronApplication, Page } from '@playwright/test';
+
+/**
+ * T1 -- renderer-only, mock IPC, no backend. Processing.tsx had zero test
+ * coverage of any kind before this spec (confirmed via a repo-wide search).
+ * The multi-stage transition logic it drives -- transcribing -> diarizing
+ * (per-channel) -> summarizing -> finalizing/error, all sharing one
+ * `chunkProgress` sub-label state across three different stages -- is
+ * exactly the kind of thing that's easy to get a stale-label leak wrong, so
+ * it earns T1 coverage per CLAUDE.md's "the interaction itself is the risk"
+ * carve-out.
+ *
+ * Drives the real PROGRESS:, summary-chunk/complete, and processing-complete
+ * IPC events directly via ElectronApplication.evaluate (main-process webContents.send),
+ * rather than adding any test-only production code to main.js/preload.js --
+ * this is a one-way push protocol, so there's nothing for a renderer-side
+ * mock invoke to intercept.
+ */
+
+async function openProcessing(page: Page) {
+ // Reached in real usage only via an active recording (never a bare URL
+ // hash) -- start one first so activeSession/recording.sessionName is
+ // truthy, matching what canRetry (retryAudioFile && activeSession) needs
+ // for the retry-button test below.
+ await page.evaluate(() => window.stenoai.recording.start('test-session'));
+ await page.evaluate(() => {
+ window.location.hash = '/meetings/processing';
+ });
+ await expect(page.getByTestId('processing-stage-label')).toBeVisible();
+ // The queue poll that first reports this recording bumps Processing's
+ // `generation`, whose render-phase reset clears stage + retryAudioFile.
+ // Wait until that has landed (the header switches from 'Note' to the
+ // session name) so a later emit can't race the reset.
+ await expect(page.getByRole('heading', { name: 'test-session' })).toBeVisible();
+}
+
+function emit(app: ElectronApplication, channel: string, payload: unknown) {
+ return app.evaluate(
+ ({ BrowserWindow }, arg) => {
+ const win = BrowserWindow.getAllWindows()[0];
+ win.webContents.send(arg.channel, arg.payload);
+ },
+ { channel, payload },
+ );
+}
+
+test('walks transcribing -> diarizing -> summarizing -> finalizing without leaking a stale sub-label', async ({ launchApp }) => {
+ const { app, page } = await launchApp({ mockIpc: true, fakeAudio: true });
+ await openProcessing(page);
+
+ const label = page.getByTestId('processing-stage-label');
+ await expect(label).toHaveText('Analyzing transcript');
+ await expect(label).toHaveAttribute('data-stage', 'transcribing');
+
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' });
+ await expect(label).toHaveText('Diarizing You channel…');
+ await expect(label).toHaveAttribute('data-stage', 'diarizing');
+
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:done' });
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:start' });
+ await expect(label).toHaveText('Diarizing Others channel…');
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:Others:done' });
+
+ await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:1/3' });
+ await expect(label).toHaveText('Summarizing part 1 of 3…');
+ await expect(label).toHaveAttribute('data-stage', 'summarizing');
+
+ await emit(app, 'processing-progress', { line: 'PROGRESS:summarize:reducing' });
+ await expect(label).toHaveText('Merging summaries…');
+
+ // The bug this spec exists to catch: chunkProgress used to persist
+ // unconditionally across stage transitions, so a stale sub-label (here,
+ // "Merging summaries…") could leak into 'finalizing'.
+ await emit(app, 'summary-complete', { success: true, sessionName: 'test-session' });
+ await expect(label).toHaveText('Almost done…');
+ await expect(label).toHaveAttribute('data-stage', 'finalizing');
+});
+
+test('shows a ticking elapsed-time counter throughout diarization, since this branch has no per-chunk percentage', async ({ launchApp }) => {
+ // Real bug found via a live app test: on a real ~21-minute recording,
+ // segmentation (the diarizer's own pass) took 100+ seconds with the label
+ // frozen on "Diarizing You channel…" the whole time -- indistinguishable
+ // from actually being stuck. The user quit the app twice. This ticker is
+ // the fix; this branch's sidecar has no per-chunk checkpoint at all, so
+ // the ticker runs for the whole diarization call rather than yielding to
+ // a percentage partway through.
+ const { app, page } = await launchApp({ mockIpc: true, fakeAudio: true });
+ await openProcessing(page);
+
+ const label = page.getByTestId('processing-stage-label');
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' });
+ await expect(label).toHaveText('Diarizing You channel…');
+
+ await expect(label).toHaveText(/Diarizing You channel… \(\d+s\)/, { timeout: 3000 });
+
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:done' });
+});
+
+test('a processing failure swaps to the error panel, and retrying does not leak the stale sub-label back', async ({ launchApp }) => {
+ const { app, page } = await launchApp({ mockIpc: true, fakeAudio: true });
+ await openProcessing(page);
+
+ const label = page.getByTestId('processing-stage-label');
+ await emit(app, 'processing-progress', { line: 'PROGRESS:diarize:You:start' });
+ await expect(label).toHaveText('Diarizing You channel…');
+
+ // On failure, the stage card is replaced entirely by a distinct error
+ // panel (retry button, no processing-stage-label) rather than the label
+ // just changing text in place.
+ await emit(app, 'processing-complete', {
+ success: false,
+ sessionName: 'test-session',
+ audioFile: '/fake/audio.wav',
+ message: 'boom',
+ });
+ await expect(label).toHaveCount(0);
+ await expect(page.getByText('Couldn’t process this recording.')).toBeVisible();
+
+ // The bug this half of the spec exists to catch: without clearing
+ // chunkProgress on the failure transition, clicking "Try again" (which
+ // resets the stage back to 'transcribing') would remount the stage card
+ // still showing the stale "Diarizing You channel…" sub-label instead of
+ // the correct fresh-start label.
+ await page.getByRole('button', { name: 'Try again' }).click();
+ await expect(label).toHaveText('Analyzing transcript');
+ await expect(label).toHaveAttribute('data-stage', 'transcribing');
+});
diff --git a/e2e/specs/speaker-diarization.t2.spec.ts b/e2e/specs/speaker-diarization.t2.spec.ts
new file mode 100644
index 00000000..60f3cb2d
--- /dev/null
+++ b/e2e/specs/speaker-diarization.t2.spec.ts
@@ -0,0 +1,129 @@
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import path from 'path';
+import { test, expect } from '../fixtures/electron';
+import { startMockOllama } from '../fixtures/mock-ollama';
+import { realUserDataDir, fileSig } from '../fixtures/real-user-data';
+import { selectEngine, isEngineModelReady, E2E_ENGINE } from '../fixtures/engine';
+import { killOllama } from '../fixtures/kill-ollama';
+import { isSayAvailable, makeStereoSpeechWav } from '../fixtures/say-stereo-wav';
+
+/**
+ * T2 — real-backend per-channel speaker diarization (@pipeline). Drives a
+ * REAL synthesized-speech stereo recording through the app's streaming
+ * pipeline with STENOAI_DIARIZE_SIDECAR_PATH pointed at a fixture script
+ * (fixed 2-speaker JSON, independent of the real steno-diarize/Sortformer
+ * binary — that binary was validated directly against real recordings
+ * during development; this spec proves the PYTHON-SIDE integration: env
+ * override resolution, subprocess invocation + JSON parsing, per-channel
+ * cluster labeling, and cross-channel "Speaker N" numbering, end to end
+ * through the real Electron + Python app).
+ *
+ * Needs real (non-sine) speech so Parakeet/whisper.cpp produce real ASR
+ * sentence segments to diarize — synthesized via macOS's `say` (see
+ * fixtures/say-stereo-wav.ts), since there's no ASR mock seam in this
+ * codebase. macOS-only (both `say` and diarize-sidecar are macOS-only);
+ * skips loudly elsewhere or when the active engine's model isn't installed.
+ */
+
+type StenoWindow = Window & {
+ stenoai: {
+ recording: { processSystemAudio: (p: string, name: string) => Promise<{ success?: boolean }> };
+ };
+};
+
+test('@pipeline synthesized two-speaker mic channel becomes You + Speaker 2', async ({
+ launchApp,
+ userDataDir,
+}) => {
+ test.setTimeout(180_000);
+
+ test.skip(process.platform !== 'darwin', 'diarize-sidecar and the say TTS fixture are macOS-only');
+ test.skip(!isSayAvailable(), 'macOS `say` TTS unavailable on this runner');
+
+ killOllama();
+ const ollama = await startMockOllama();
+ const realDirBefore = fileSig(realUserDataDir());
+
+ try {
+ // Build the real speech fixture BEFORE launching the app: mic channel =
+ // two utterances separated by a real acoustic gap, system channel = one
+ // short utterance. Pure Node/TTS — no app instance needed yet.
+ const recordingsDir = path.join(userDataDir, 'recordings');
+ mkdirSync(recordingsDir, { recursive: true });
+ const wavPath = path.join(recordingsDir, 'diarize.wav');
+ const { micBoundarySeconds } = makeStereoSpeechWav(wavPath, {
+ micUtteranceA: 'Hello there, I hope you are having a wonderful and productive day today.',
+ micUtteranceB: 'Thanks a lot.',
+ systemUtterance: 'Thanks.',
+ });
+
+ // Fixture diarizer: always reports the SAME two speaker clusters,
+ // independent of which channel WAV it's actually pointed at. The tail
+ // cluster is deliberately shorter than the boundary so SPEAKER_0 always
+ // stays the dominant (most total speaking time) cluster on BOTH
+ // channels — sanity-checked below rather than assumed. Expressed as a
+ // fraction of micBoundarySeconds with no floor above it, so the
+ // invariant below holds for any micBoundarySeconds > 0 -- a previous
+ // version floored this at 1.5s, which made the assertion mathematically
+ // impossible whenever isSayAvailable()'s probe let through a runner
+ // where `say` produced near-silent audio (micBoundarySeconds well under
+ // 1.5s). isSayAvailable() now measures real synthesized duration, so
+ // that shouldn't recur, but this formula no longer depends on it either.
+ const tailSeconds = Math.min(micBoundarySeconds * 0.6, 3.0);
+ expect(tailSeconds).toBeLessThan(micBoundarySeconds);
+
+ const fixtureDir = mkdtempSync(path.join(tmpdir(), 'stenoai-e2e-diarize-'));
+ const scriptPath = path.join(fixtureDir, 'mock-steno-diarize.sh');
+ writeFileSync(
+ scriptPath,
+ [
+ '#!/usr/bin/env bash',
+ 'set -euo pipefail',
+ `echo '[{"speakerId":"SPEAKER_0","start":0,"end":${micBoundarySeconds}},` +
+ `{"speakerId":"SPEAKER_1","start":${micBoundarySeconds},"end":${micBoundarySeconds + tailSeconds}}]'`,
+ '',
+ ].join('\n'),
+ );
+ chmodSync(scriptPath, 0o755);
+
+ const { page } = await launchApp({
+ env: { STENOAI_DIARIZE_SIDECAR_PATH: scriptPath },
+ });
+ await selectEngine(page);
+
+ const modelReady = await isEngineModelReady(page);
+ if (!modelReady) {
+ // eslint-disable-next-line no-console
+ console.warn(`[t2:@pipeline] SKIPPED: ${E2E_ENGINE} model not installed on this runner.`);
+ test.info().annotations.push({ type: 'skip-reason', description: `${E2E_ENGINE} model not installed` });
+ }
+ test.skip(!modelReady, `${E2E_ENGINE} model not installed`);
+
+ const queued = await page.evaluate(
+ (p) => (window as StenoWindow).stenoai.recording.processSystemAudio(p, 'E2E Diarize'),
+ wavPath,
+ );
+ expect(queued?.success).toBe(true);
+
+ const transcriptPath = path.join(userDataDir, 'transcripts', 'diarize_transcript.txt');
+ await expect
+ .poll(() => existsSync(transcriptPath), { timeout: 120_000, intervals: [1000] })
+ .toBe(true);
+
+ const transcript = readFileSync(transcriptPath, 'utf8');
+ // Mic's dominant cluster (utterance A) keeps the legacy "You" label;
+ // its minority cluster (utterance B) becomes "Speaker 2". The system
+ // channel's single utterance lands in the shared dominant cluster too,
+ // so it keeps "Others" rather than becoming a second placeholder.
+ expect(transcript).toMatch(/\[You\]/);
+ expect(transcript).toMatch(/\[Speaker 2\]/);
+ expect(transcript).toMatch(/\[Others\]/);
+ expect(transcript).not.toMatch(/\[Speaker 3\]/);
+
+ expect(fileSig(realUserDataDir())).toBe(realDirBefore);
+ } finally {
+ await ollama.close();
+ killOllama();
+ }
+});
diff --git a/scripts/build-diarize-sidecar.sh b/scripts/build-diarize-sidecar.sh
new file mode 100755
index 00000000..f8961df5
--- /dev/null
+++ b/scripts/build-diarize-sidecar.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Build the Swift diarize-sidecar helper.
+#
+# Usage: scripts/build-diarize-sidecar.sh [arch]
+# arch defaults to host arch (arm64 / x86_64).
+#
+# Requires Xcode Command Line Tools and an internet connection on first run
+# (SPM fetches the FluidAudio dependency).
+set -euo pipefail
+
+ARCH="${1:-$(uname -m)}"
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+PKG="$ROOT/diarize-sidecar"
+OUT="$ROOT/bin/steno-diarize"
+
+mkdir -p "$ROOT/bin"
+
+cd "$PKG"
+
+swift build \
+ -c release \
+ --arch "$ARCH"
+
+BUILD_BIN="$PKG/.build/${ARCH}-apple-macosx/release/diarize-sidecar"
+cp "$BUILD_BIN" "$OUT"
+
+# Ad-hoc signature so the binary runs locally; CI re-signs with the Developer
+# ID when packaging the .app bundle.
+codesign --sign - "$OUT" 2>/dev/null || true
+
+file "$OUT"
\ No newline at end of file
diff --git a/src/_parakeet_mlx.py b/src/_parakeet_mlx.py
index 5333ddda..61e0f5de 100644
--- a/src/_parakeet_mlx.py
+++ b/src/_parakeet_mlx.py
@@ -294,6 +294,22 @@ def _result_to_dict(result, language: Optional[str]) -> dict:
"text": (getattr(s, "text", "") or "").strip(),
"start": float(getattr(s, "start", 0.0) or 0.0),
"end": float(getattr(s, "end", 0.0) or 0.0),
+ # Word-level timing, kept alongside the sentence text so the
+ # diarised path can split an abnormally long run-on sentence
+ # (Parakeet sometimes fails to break long unpunctuated speech
+ # into multiple sentences) across several diarizer turns
+ # instead of forcing the whole thing onto one speaker. Token
+ # text already carries its own leading-space markers —
+ # "".join(t["text"] for t in tokens) reconstructs the sentence
+ # exactly, no separator needed.
+ "tokens": [
+ {
+ "text": getattr(t, "text", "") or "",
+ "start": float(getattr(t, "start", 0.0) or 0.0),
+ "end": float(getattr(t, "end", 0.0) or 0.0),
+ }
+ for t in (getattr(s, "tokens", None) or [])
+ ],
}
for s in sentences
if (getattr(s, "text", "") or "").strip()
diff --git a/src/transcriber.py b/src/transcriber.py
index 76f8b1a7..959fa1b4 100644
--- a/src/transcriber.py
+++ b/src/transcriber.py
@@ -27,7 +27,9 @@
anything; the model is the source of truth.
"""
+import contextlib
import inspect
+import json
import logging
import math
import os
@@ -96,6 +98,14 @@
# pre-process floor; the helper scales it up with duration for long meetings.
DIARISED_SPLIT_TIMEOUT_S = 600
+# Timeout for the channel-COUNT probe in _split_stereo_to_channels (`-t 0`,
+# header-only -- runs BEFORE we know duration, so unlike the timeouts above
+# this can't scale with it). Should be near-instant, but a live-recorded
+# WebM (no seek index) measured >15s on a real ~3.5h recording -- same
+# silent-mono-fallback failure this whole file already guards against
+# elsewhere, just one step earlier in the pipeline.
+CHANNEL_DETECT_TIMEOUT_S = 60
+
# RMS energy gate for "channel has speech". Intentionally low (-70 dB) so
# headphones-mode mic recordings — captured at much lower amplitude than
# speakers-mode — still pass. The model handles low-amplitude speech fine;
@@ -106,6 +116,28 @@
# recording doesn't pull all 30 min of int16 samples into Python lists.
RMS_MAX_WINDOWS = 60
+# Acoustic per-channel speaker diarization (steno-diarize sidecar, macOS
+# only). Merge gap for consecutive same-speaker diarizer segments — reduces
+# diarization flicker and shrinks the gaps that cause boundary sentence
+# misattribution in _assign_asr_segments_to_diar_segments. Matches the value
+# validated against real meeting audio in the research playground.
+STENO_DIARIZE_MERGE_GAP_S = 0.3
+
+# Floor for the steno-diarize subprocess timeout. Measured runtime against
+# real meeting-length audio is single-digit-to-tens-of-seconds; this leaves
+# generous headroom under Electron's 8-minute inactivity watchdog while
+# still bounding a runaway on pathological input. Scaled up by duration for
+# long recordings, same pattern as _diarised_split_timeout.
+STENO_DIARIZE_TIMEOUT_FLOOR_S = 120
+
+# If one diarizer cluster holds this share (or more) of a channel's total
+# speaking time, the channel is treated as single-speaker — any other
+# cluster is almost certainly a brief misdiarization blip (observed
+# empirically: short/overlapping noise segments from Sortformer on
+# single-mic audio), not a real second speaker. Gates the "Speaker N"
+# placeholder path (_cluster_channel_labels).
+CHANNEL_DOMINANCE_THRESHOLD = 0.92
+
# Sentinel text substituted when transcription produces no usable output
# (genuine silence or all-hallucination). Callers compare against this to
# distinguish "really nothing was said" from a real (possibly short)
@@ -164,6 +196,51 @@ def _resolve_ffmpeg() -> Optional[str]:
return None
+# Resolve the bundled steno-diarize binary (macOS-only Swift/CoreML speaker
+# diarization sidecar, built by scripts/build-diarize-sidecar.sh). Mirrors
+# _resolve_ffmpeg()'s bundle-then-fallback lookup, but checks executability
+# instead of running a cheap probe invocation — there's no equivalent to
+# `ffmpeg -version` for this binary, so a bad binary still fails safely at
+# actual call time via _run_steno_diarize's blanket failure handling.
+_STENO_DIARIZE_PATH_CACHE: Optional[str] = None
+_STENO_DIARIZE_PATH_LOCK = threading.Lock()
+
+
+def _resolve_steno_diarize() -> Optional[str]:
+ global _STENO_DIARIZE_PATH_CACHE
+ if sys.platform != "darwin":
+ return None
+ if _STENO_DIARIZE_PATH_CACHE is not None:
+ return _STENO_DIARIZE_PATH_CACHE
+ with _STENO_DIARIZE_PATH_LOCK:
+ if _STENO_DIARIZE_PATH_CACHE is not None:
+ return _STENO_DIARIZE_PATH_CACHE
+ candidates: list[str] = []
+ # Same spirit as the STENO_DIARIZE_* env knobs on the Swift side —
+ # lets a T2 e2e spec point at a fixture binary without touching the
+ # real bundle resolution.
+ override = os.environ.get("STENOAI_DIARIZE_SIDECAR_PATH")
+ if override:
+ candidates.append(override)
+ if getattr(sys, 'frozen', False):
+ exe_dir = Path(sys.executable).parent
+ candidates.extend([
+ str(exe_dir / 'steno-diarize'),
+ str(exe_dir / '_internal' / 'steno-diarize'),
+ ])
+ else:
+ # Dev: repo-root bin/, built by scripts/build-diarize-sidecar.sh
+ repo_root = Path(__file__).resolve().parent.parent
+ candidates.append(str(repo_root / 'bin' / 'steno-diarize'))
+ for cand in candidates:
+ if os.access(cand, os.X_OK):
+ _STENO_DIARIZE_PATH_CACHE = cand
+ logger.info(f"steno-diarize resolved at: {cand}")
+ return cand
+ logger.info("steno-diarize not found; per-channel speaker labeling falls back to legacy You/Others")
+ return None
+
+
def _audio_filter_chain() -> str:
"""The ffmpeg ``-af`` chain applied to mono audio before transcription."""
return f"highpass=f={AUDIO_HIGHPASS_HZ},loudnorm={AUDIO_LOUDNORM}"
@@ -411,6 +488,400 @@ def _drop_per_segment_bleed(
return kept_mic, kept_sys
+# ---------------------------------------------------------------------------
+# Per-channel acoustic speaker diarization (steno-diarize sidecar, macOS
+# only). Ported from the research playground (scripts/diarize_playground.py)
+# as dict-based helpers — that script's own docstring states it's a research
+# tool that doesn't touch src/, and its functions use attribute access on
+# AlignedSentence objects while our real ASR segments are dicts.
+# ---------------------------------------------------------------------------
+
+def _merge_close_diar_segments(segments: list[dict], max_gap: float) -> list[dict]:
+ """Merge consecutive same-speaker diarizer segments separated by a gap
+ smaller than max_gap. Segments must already be sorted by start time.
+ Reduces diarization flicker and shrinks the gaps that cause boundary
+ sentence misattribution in _assign_asr_segments_to_diar_segments."""
+ if not segments:
+ return []
+ merged = [dict(segments[0])]
+ for segment in segments[1:]:
+ last = merged[-1]
+ if segment["speaker"] == last["speaker"] and segment["start"] - last["end"] <= max_gap:
+ last["end"] = segment["end"]
+ else:
+ merged.append(dict(segment))
+ return merged
+
+
+# A single mic capturing two people is a much harder acoustic problem than a
+# clean mic-vs-system-audio split — the diarizer's own turn boundaries can be
+# genuinely noisy/overlapping there (observed empirically: alternating and
+# even overlapping SPEAKER_0/SPEAKER_1 segments across a real back-and-forth).
+# Parakeet sometimes fails to break a long run of natural speech (no strong
+# terminal punctuation) into separate sentences, producing a single sentence
+# that spans many real diarizer turns. Assigning that whole sentence to
+# whichever one diarizer segment its midpoint happens to land in then forces
+# an entire multi-turn exchange onto one speaker. A sentence at or above this
+# duration gets word-level splitting instead (see _find_nearest_diar_segment)
+# whenever it actually overlaps more than one distinct diarizer speaker.
+LONG_SENTENCE_SPLIT_THRESHOLD_S = 5.0
+
+
+def _find_nearest_diar_segment(start: float, end: float, diar_segments: list[dict]) -> Optional[int]:
+ """Index of the diar segment containing [start, end]'s midpoint, or the
+ nearest one by boundary distance if the midpoint falls in an uncovered
+ gap. Returns None only when diar_segments is empty."""
+ midpoint = (start + end) / 2
+ best_i, best_dist = None, float("inf")
+ for i, segment in enumerate(diar_segments):
+ if segment["start"] <= midpoint <= segment["end"]:
+ return i
+ dist = (
+ segment["start"] - midpoint
+ if midpoint < segment["start"]
+ else midpoint - segment["end"]
+ )
+ if dist < best_dist:
+ best_i, best_dist = i, dist
+ return best_i
+
+
+def _assign_asr_segments_to_diar_segments(asr_segments: list[dict], diar_segments: list[dict]) -> None:
+ """Assign each ASR (Parakeet) sentence to the diarizer segment(s) it
+ belongs to.
+
+ Normal case (sentence fits inside one real diarizer turn): assign the
+ whole sentence as one block to the nearest/containing diar segment.
+ Sentence granularity (not word/token) is deliberate here — Parakeet
+ already does its own sentence segmentation, so this never tears a word
+ or clause in half for the common case.
+
+ Long-sentence case: if a sentence runs at or above
+ LONG_SENTENCE_SPLIT_THRESHOLD_S AND genuinely overlaps more than one
+ distinct diarizer speaker, assigning it as one block would force an
+ entire multi-turn exchange onto whichever speaker the midpoint happened
+ to land on. Fall back to word-level assignment instead: each word goes
+ to its own nearest diar segment (using its own timing, from
+ segment["tokens"] — see src/_parakeet_mlx.py), and runs of consecutive
+ words landing in the same diar segment are joined back together. This
+ needs word timing to exist at all (`tokens`); if it doesn't (e.g. the
+ whisper.cpp backend, or an older cached result), the sentence falls back
+ to whole-block assignment like the normal case.
+
+ Mutates diar_segments in place, attaching joined text as segment["text"]."""
+ for segment in diar_segments:
+ segment["text"] = ""
+ if not diar_segments:
+ return
+
+ texts_by_segment: dict[int, list[str]] = {i: [] for i in range(len(diar_segments))}
+ for asr_segment in asr_segments:
+ text = (asr_segment.get("text") or "").strip()
+ if not text:
+ continue
+ start = float(asr_segment.get("start") or 0.0)
+ end = float(asr_segment.get("end") or start)
+ tokens = asr_segment.get("tokens") or []
+ duration = end - start
+
+ multi_speaker_span = False
+ if duration >= LONG_SENTENCE_SPLIT_THRESHOLD_S and tokens:
+ overlapping_speakers = {
+ seg["speaker"] for seg in diar_segments
+ if seg["start"] < end and seg["end"] > start
+ }
+ multi_speaker_span = len(overlapping_speakers) > 1
+
+ if multi_speaker_span:
+ run_index: Optional[int] = None
+ run_words: list[str] = []
+ for token in tokens:
+ token_text = token.get("text") or ""
+ if not token_text.strip():
+ continue
+ t_start = float(token.get("start") or 0.0)
+ t_end = float(token.get("end") or t_start)
+ idx = _find_nearest_diar_segment(t_start, t_end, diar_segments)
+ if idx is None:
+ continue
+ if run_index is not None and idx != run_index:
+ texts_by_segment[run_index].append("".join(run_words))
+ run_words = []
+ run_index = idx
+ run_words.append(token_text)
+ if run_index is not None and run_words:
+ texts_by_segment[run_index].append("".join(run_words))
+ else:
+ idx = _find_nearest_diar_segment(start, end, diar_segments)
+ if idx is not None:
+ texts_by_segment[idx].append(text)
+
+ for i, segment in enumerate(diar_segments):
+ segment["text"] = " ".join(t.strip() for t in texts_by_segment[i] if t.strip()).strip()
+
+
+# How often to print a HEARTBEAT: line while blocked waiting on
+# steno-diarize. Comfortably under Electron's TRANSCRIBE_INACTIVITY_MS
+# (8 minutes, app/main.js) -- see _heartbeat_while_waiting's docstring for
+# why this can't just reuse the existing chunk-progress heartbeat registry.
+STENO_DIARIZE_HEARTBEAT_INTERVAL_S = 60.0
+
+
+@contextlib.contextmanager
+def _heartbeat_while_waiting(label: str, interval_s: float = STENO_DIARIZE_HEARTBEAT_INTERVAL_S):
+ """Print a HEARTBEAT: line every ``interval_s`` seconds on a background
+ thread for the duration of the ``with`` block.
+
+ src._heartbeat's chunk-progress registry only works for backends that
+ call back into Python from INSIDE their own per-chunk loop (Parakeet,
+ Whisper.cpp) -- steno-diarize is an opaque external binary invoked via a
+ single blocking call, with no such checkpoint to hang a callback off of.
+ Without this, a diarization run on an hours-long channel prints nothing
+ for its entire duration, which Electron's inactivity watchdog
+ (app/main.js) can't tell apart from a hung process -- and kills,
+ discarding a real, working meeting.
+
+ Never affects the wrapped call's own return value or exceptions --
+ the background thread only ever writes heartbeat lines.
+ """
+ stop = threading.Event()
+
+ def _beat():
+ while not stop.wait(interval_s):
+ try:
+ sys.stdout.write(f"HEARTBEAT:{label}\n")
+ sys.stdout.flush()
+ except Exception:
+ pass
+
+ t = threading.Thread(target=_beat, daemon=True)
+ t.start()
+ try:
+ yield
+ finally:
+ stop.set()
+ t.join(timeout=1.0)
+
+
+def _run_steno_diarize(channel_path: Path, timeout: int) -> Optional[list[dict]]:
+ """Run the steno-diarize sidecar on a single mono channel WAV.
+
+ Returns merged diarizer segments (each ``{"start", "end", "speaker"}``)
+ on success. Returns None on ANY failure — missing binary, timeout,
+ non-zero exit, or unparseable output — so callers always have a safe
+ fallback to legacy channel-only labeling and this can never fail a
+ meeting.
+
+ Uses Popen with two concurrent reader threads rather than
+ subprocess.run(capture_output=True) so stdout and stderr are both
+ drained WHILE the process is still running -- matching what
+ subprocess.run's own communicate() does internally to avoid the
+ classic pipe-deadlock (real stdout payloads have measured up to
+ ~211KB in production, well past an OS pipe buffer, so neither stream
+ can safely be read to completion only after the process exits).
+ """
+ binary = _resolve_steno_diarize()
+ if not binary:
+ return None
+ try:
+ proc = subprocess.Popen(
+ [binary, str(channel_path)],
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+ )
+ stdout_chunks: list[bytes] = []
+ stderr_chunks: list[bytes] = []
+
+ def _read_stdout():
+ for chunk in iter(lambda: proc.stdout.read(65536), b""):
+ stdout_chunks.append(chunk)
+
+ def _read_stderr():
+ for chunk in iter(lambda: proc.stderr.read(4096), b""):
+ stderr_chunks.append(chunk)
+
+ t_out = threading.Thread(target=_read_stdout, daemon=True)
+ t_err = threading.Thread(target=_read_stderr, daemon=True)
+ t_out.start()
+ t_err.start()
+ try:
+ proc.wait(timeout=timeout)
+ except subprocess.TimeoutExpired:
+ proc.kill()
+ proc.wait()
+ t_out.join(timeout=2.0)
+ t_err.join(timeout=2.0)
+ logger.warning("steno-diarize timed out after %ss", timeout)
+ return None
+ t_out.join(timeout=5.0)
+ t_err.join(timeout=5.0)
+
+ if proc.returncode != 0:
+ stderr_text = b"".join(stderr_chunks).decode(errors="replace")
+ logger.warning(
+ "steno-diarize exited %s: %s",
+ proc.returncode, stderr_text[:300],
+ )
+ return None
+ stdout = b"".join(stdout_chunks).decode(errors="replace")
+ # Known FluidAudio/CoreML warnings ("E5RT encountered an STL
+ # exception... key not found") can print directly to stdout
+ # BEFORE, BETWEEN, or AFTER the real JSON payload -- skipping to
+ # the first '[' assumes stdout is pure JSON, which breaks the
+ # moment any warning text (or an interstitial, incomplete blob)
+ # appears ahead of the real array. Scan every '[' in stdout with a
+ # real JSON decoder and keep the LAST array of segment-shaped
+ # dicts -- better than latching onto the first bracket found.
+ raw_segments = None
+ decoder = json.JSONDecoder()
+ search_from = 0
+ while True:
+ next_bracket = stdout.find("[", search_from)
+ if next_bracket < 0:
+ break
+ try:
+ candidate, end = decoder.raw_decode(stdout, next_bracket)
+ except json.JSONDecodeError:
+ search_from = next_bracket + 1
+ continue
+ if (
+ isinstance(candidate, list) and candidate
+ and all(isinstance(s, dict) and "speakerId" in s for s in candidate)
+ ):
+ raw_segments = candidate
+ search_from = max(end, next_bracket + 1)
+ if raw_segments is None:
+ logger.warning(
+ "steno-diarize produced no usable JSON output "
+ "(stdout length %d, last 500 chars: %r)",
+ len(stdout), stdout[-500:],
+ )
+ return None
+ except (subprocess.TimeoutExpired, OSError, ValueError) as e:
+ logger.warning("steno-diarize failed: %s", e)
+ return None
+
+ segments = sorted(
+ (
+ {
+ "start": float(s["start"]),
+ "end": float(s["end"]),
+ "speaker": str(s["speakerId"]),
+ }
+ for s in raw_segments
+ ),
+ key=lambda s: s["start"],
+ )
+ return _merge_close_diar_segments(segments, STENO_DIARIZE_MERGE_GAP_S)
+
+
+def _cluster_channel_labels(diar_segments: list[dict], legacy_label: str) -> Optional[dict[str, str]]:
+ """Map each diarizer speaker id in diar_segments to either the channel's
+ legacy label (the cluster with the most total speaking time) or a
+ placeholder key for every other cluster, later resolved to "Speaker N"
+ by _resolve_speaker_placeholders.
+
+ Returns None when diar_segments contains a single (or zero) distinct
+ speaker, OR when one cluster's share of total speaking time is at or
+ above CHANNEL_DOMINANCE_THRESHOLD — the byte-identical-to-legacy fast
+ path, since a barely-there second cluster is almost certainly
+ misdiarization noise rather than a real second speaker.
+ """
+ speaker_ids = {s["speaker"] for s in diar_segments}
+ if len(speaker_ids) <= 1:
+ return None
+ totals: dict[str, float] = {sid: 0.0 for sid in speaker_ids}
+ for s in diar_segments:
+ totals[s["speaker"]] += s["end"] - s["start"]
+ dominant = max(totals, key=totals.get)
+ total_time = sum(totals.values())
+ if total_time > 0 and totals[dominant] / total_time >= CHANNEL_DOMINANCE_THRESHOLD:
+ return None
+ return {
+ sid: (legacy_label if sid == dominant else f"__diar__{legacy_label}__{sid}")
+ for sid in speaker_ids
+ }
+
+
+def _tag_channel_segments(
+ asr_segments: list[dict],
+ channel_path: Optional[Path],
+ duration_seconds: Optional[float],
+ legacy_label: str,
+) -> list[tuple[float, str, str]]:
+ """Build (start, label, text) tuples for one channel's ASR segments.
+
+ Tries acoustic diarization first: if the steno-diarize sidecar succeeds
+ and finds more than one real speaker cluster, turns are built from the
+ diarizer's own segment boundaries (with ASR sentences reassigned into
+ them via _assign_asr_segments_to_diar_segments). The cluster with the
+ most total speaking time keeps the channel's legacy label
+ ("You"/"Others"); every other cluster gets a placeholder resolved to
+ "Speaker N" later. ANY failure — missing binary, timeout, bad JSON, or
+ a single-cluster result — falls back to the byte-identical legacy
+ behaviour of labeling every ASR segment with legacy_label.
+ """
+ if not asr_segments:
+ return []
+
+ if channel_path is not None:
+ timeout = max(STENO_DIARIZE_TIMEOUT_FLOOR_S, int(duration_seconds or 0))
+ logger.info(f"Diarizing {legacy_label} channel acoustically (up to {timeout}s)...")
+ print(f"PROGRESS:diarize:{legacy_label}:start", flush=True)
+ try:
+ with _heartbeat_while_waiting(f"diarize:{legacy_label}"):
+ diar_segments = _run_steno_diarize(channel_path, timeout)
+ if diar_segments:
+ cluster_labels = _cluster_channel_labels(diar_segments, legacy_label)
+ if cluster_labels:
+ _assign_asr_segments_to_diar_segments(asr_segments, diar_segments)
+ diar_tagged = []
+ for segment in diar_segments:
+ text = (segment.get("text") or "").strip()
+ if text:
+ diar_tagged.append((segment["start"], cluster_labels[segment["speaker"]], text))
+ if diar_tagged:
+ logger.info(
+ f"Diarizing {legacy_label} channel found "
+ f"{len(set(cluster_labels.values()))} speaker cluster(s)"
+ )
+ return diar_tagged
+ logger.info(f"Diarizing {legacy_label} channel: falling back to legacy single-speaker labeling")
+ finally:
+ print(f"PROGRESS:diarize:{legacy_label}:done", flush=True)
+
+ legacy_tagged: list[tuple[float, str, str]] = []
+ for s in asr_segments:
+ text = (s.get("text") or "").strip()
+ if text:
+ legacy_tagged.append((float(s.get("start") or 0.0), legacy_label, text))
+ return legacy_tagged
+
+
+def _resolve_speaker_placeholders(
+ tagged: list[tuple[float, str, str]],
+) -> list[tuple[float, str, str]]:
+ """Replace placeholder cluster labels (see _cluster_channel_labels) with
+ "Speaker N", numbered by first chronological appearance across BOTH
+ channels merged and time-sorted — so a reader sees new speakers
+ introduced as 2, 3, 4... regardless of which channel they came from.
+
+ No cross-channel identity matching: a mic placeholder and a system
+ placeholder are always treated as different people — telling them
+ apart would need voiceprint embeddings, out of scope here.
+ """
+ numbering: dict[str, str] = {}
+ next_n = 2
+ resolved: list[tuple[float, str, str]] = []
+ for start, label, text in tagged:
+ if label.startswith("__diar__"):
+ if label not in numbering:
+ numbering[label] = f"Speaker {next_n}"
+ next_n += 1
+ label = numbering[label]
+ resolved.append((start, label, text))
+ return resolved
+
+
# Try Parakeet first (preferred — same engine as live, arm64 Macs only).
try:
from src.parakeet import transcribe_file as _parakeet_transcribe_file
@@ -633,6 +1104,11 @@ def _preprocess_audio(self, audio_filepath: Path) -> Tuple[Path, bool]:
return audio_filepath, False
temp_path = Path(temp_name)
try:
+ # loudnorm's two-pass loudness analysis can take real wall-clock
+ # time on a long recording, with zero other output in between --
+ # without this, the terminal goes silent for that whole stretch
+ # right after "Saved: ...", which reads as a hang.
+ logger.info(f"Pre-processing audio (highpass + loudnorm): {audio_filepath.name}...")
result = subprocess.run(
[ffmpeg, '-y', '-i', str(audio_filepath),
'-af', _audio_filter_chain(),
@@ -982,12 +1458,21 @@ def _split_stereo_to_channels(self, audio_filepath: Path) -> Tuple[Optional[Path
# Detect channel count via ffmpeg. `-t 0` makes ffmpeg parse the
# input header (where the channel layout lives) and exit immediately
# without decoding any audio frames — without it, a 1-hour recording
- # would actually decode in full just to read metadata.
+ # would actually decode in full just to read metadata. In practice
+ # this ISN'T always instant: a WebM written live by MediaRecorder
+ # (our sysaudio capture path) has no seek index, so on an unusually
+ # large file ffmpeg's demuxer can still need real time to find the
+ # first decodable packet. A real ~3.5h recording measured this at
+ # >15s. Same failure shape _diarised_split_timeout's docstring
+ # documents for the later full-channel-split step: a too-tight fixed
+ # timeout here silently drops the whole recording to mono (no
+ # [You]/[Others]) instead of failing loudly — so this budget needs
+ # real headroom, not just enough for the common case.
try:
probe = subprocess.run(
[ffmpeg, '-hide_banner', '-t', '0', '-i', str(audio_filepath),
'-f', 'null', '-'],
- capture_output=True, timeout=15, text=True
+ capture_output=True, timeout=CHANNEL_DETECT_TIMEOUT_S, text=True
)
stderr = probe.stderr or ''
channels = _parse_channels_from_ffmpeg_stderr(stderr)
@@ -1087,18 +1572,16 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt
"""Transcribe with stereo channel diarisation.
If the audio is stereo (left=mic, right=system), each channel is
- transcribed separately and labelled as [You] and [Others]. Falls
- back to normal transcription for mono audio.
+ transcribed separately and labelled as [You] and [Others]. Mono
+ audio (e.g. many imported recordings) has no channel split to lean
+ on, but can still contain multiple speakers — acoustic diarization
+ runs directly against the whole file instead; see
+ _transcribe_diarised_mono.
"""
mic_path, system_path, duration = self._split_stereo_to_channels(audio_filepath)
if mic_path is None:
- # Mono audio — use standard transcription
- result = self.transcribe_audio(audio_filepath, language)
- if result:
- result['is_diarised'] = False
- result['diarised_text'] = None
- return result
+ return self._transcribe_diarised_mono(audio_filepath, language)
try:
mic_has_audio = self._check_rms_energy(mic_path)
@@ -1247,17 +1730,16 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt
# Chronologically interleave segments from both channels and
# collapse runs of consecutive same-speaker segments into a
- # single labelled turn.
+ # single labelled turn. Each channel is first run through
+ # acoustic diarization (steno-diarize, macOS only) to split
+ # multiple speakers sharing one side of the call; any failure
+ # or a single-cluster result falls back to the legacy
+ # "You"/"Others" channel-only labeling.
tagged: list[tuple[float, str, str]] = []
- for s in mic_segments:
- text = (s.get("text") or "").strip()
- if text:
- tagged.append((float(s.get("start") or 0.0), "You", text))
- for s in system_segments:
- text = (s.get("text") or "").strip()
- if text:
- tagged.append((float(s.get("start") or 0.0), "Others", text))
+ tagged.extend(_tag_channel_segments(mic_segments, mic_path, duration, "You"))
+ tagged.extend(_tag_channel_segments(system_segments, system_path, duration, "Others"))
tagged.sort(key=lambda t: t[0])
+ tagged = _resolve_speaker_placeholders(tagged)
# Each turn carries the start offset of its FIRST segment so the
# diarised transcript can be timestamped. Only diarised_text is
@@ -1277,7 +1759,19 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt
plain_parts = [' '.join(parts) for _start, _speaker, parts in turns]
plain_text = "\n\n".join(plain_parts) if plain_parts else SILENCE_SENTINEL
- is_diarised = bool(mic_segments) and bool(system_segments)
+ # Diarised means "more than one voice is distinguishable in this
+ # transcript" — NOT "both channels had content". The old
+ # bool(mic_segments) and bool(system_segments) check predates
+ # per-channel acoustic diarization and silently discarded the
+ # whole labelled transcript whenever one channel was empty (e.g.
+ # an in-person conversation with no system audio playing at
+ # all), even when _tag_channel_segments had already split the
+ # OTHER channel into "You" + "Speaker 2". Counting distinct
+ # labels covers both the classic two-channel case (You + Others)
+ # and the new single-channel multi-speaker case correctly, and
+ # still suppresses labelling a genuine one-voice monologue.
+ distinct_labels = {speaker for _start, speaker, _parts in turns}
+ is_diarised = len(distinct_labels) > 1
if is_diarised:
labelled_parts = [
f"[{_format_timestamp(start)}] [{speaker}] {' '.join(parts)}"
@@ -1305,6 +1799,52 @@ def transcribe_diarised(self, audio_filepath: Path, language: str = "en") -> Opt
except Exception:
pass
+ def _transcribe_diarised_mono(self, audio_filepath: Path, language: str) -> Optional[dict]:
+ """Diarise a mono recording (no channel split to lean on).
+
+ Runs standard transcription, then acoustic diarization directly
+ against the whole file via the same _tag_channel_segments helper the
+ stereo path uses per-channel — steno-diarize decodes any input
+ format itself (via ffmpeg), so no split/preprocessing is needed
+ first. The single track is treated as the "You" channel, matching
+ the pre-diarization convention of attributing an unlabelled mono
+ recording to the user; a second acoustic cluster becomes "Speaker 2"
+ exactly as a second cluster on the mic channel would in the stereo
+ path. Any diarization failure or a single-cluster result leaves
+ is_diarised False, matching the historical mono behaviour exactly.
+ """
+ result = self.transcribe_audio(audio_filepath, language)
+ if not result:
+ return result
+ result['is_diarised'] = False
+ result['diarised_text'] = None
+ if result.get("transcription_failed") or result.get("transcription_empty"):
+ return result
+
+ asr_segments = result.get("segments") or []
+ duration = result.get("duration_seconds")
+ tagged = _tag_channel_segments(asr_segments, audio_filepath, duration, "You")
+ tagged.sort(key=lambda t: t[0])
+ tagged = _resolve_speaker_placeholders(tagged)
+
+ turns: list[tuple[float, str, list[str]]] = []
+ for start, speaker, text in tagged:
+ if turns and turns[-1][1] == speaker:
+ turns[-1][2].append(text)
+ else:
+ turns.append((start, speaker, [text]))
+
+ distinct_labels = {speaker for _start, speaker, _parts in turns}
+ if len(distinct_labels) > 1:
+ labelled_parts = [
+ f"[{_format_timestamp(start)}] [{speaker}] {' '.join(parts)}"
+ for start, speaker, parts in turns
+ ]
+ result['diarised_text'] = "\n\n".join(labelled_parts)
+ result['is_diarised'] = True
+
+ return result
+
def transcribe_with_timestamps(self, audio_filepath: Path) -> Optional[dict]:
"""Batch transcribe and return segment-level timing.
diff --git a/stenoai.spec b/stenoai.spec
index f8c5b6a0..7233eec3 100644
--- a/stenoai.spec
+++ b/stenoai.spec
@@ -257,6 +257,15 @@ if os.path.exists(ollama_bin_dir):
if base in ('ffmpeg', 'ffmpeg.exe'):
# Put ffmpeg at the root of the bundle for easy PATH access
binaries.append((filepath, '.'))
+ elif base == 'steno-diarize' and _IS_DARWIN:
+ # macOS-only Swift/CoreML diarization sidecar (built by
+ # scripts/build-diarize-sidecar.sh). Root-level like ffmpeg
+ # since src/transcriber.py resolves it the same way. Gated
+ # on _IS_DARWIN + os.path.exists above so a checkout that
+ # hasn't run the Swift build (or a non-macOS platform) just
+ # skips it — src/transcriber.py falls back to legacy
+ # channel-only labeling when the binary is missing.
+ binaries.append((filepath, '.'))
elif _IS_DARWIN:
# COLLECT DATA TOC 3-tuple: (dest_path_including_filename,
# abs_src_path, 'DATA'). Everything lives under ollama/,
diff --git a/tests/test_transcriber_diarisation.py b/tests/test_transcriber_diarisation.py
index ba86bd39..532718d8 100644
--- a/tests/test_transcriber_diarisation.py
+++ b/tests/test_transcriber_diarisation.py
@@ -9,23 +9,34 @@
so a recording where speech starts mid-stream isn't classified as silent.
"""
+import io
+import json
import math
import struct
+import subprocess
import tempfile
import unittest
import wave
from pathlib import Path
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
from src.transcriber import (
BLEED_JACCARD_THRESHOLD,
+ CHANNEL_DOMINANCE_THRESHOLD,
DIARISED_SPLIT_TIMEOUT_S,
MIN_RMS_THRESHOLD,
+ STENO_DIARIZE_MERGE_GAP_S,
WhisperTranscriber,
+ _assign_asr_segments_to_diar_segments,
+ _cluster_channel_labels,
_diarised_split_timeout,
_format_timestamp,
+ _merge_close_diar_segments,
_parse_channels_from_ffmpeg_stderr,
_parse_duration_from_ffmpeg_stderr,
+ _resolve_speaker_placeholders,
+ _run_steno_diarize,
+ _tag_channel_segments,
_token_jaccard,
)
@@ -80,8 +91,16 @@ def setUp(self):
return_value=(self.mic_path, self.system_path, 3.0)
)
self.transcriber._check_rms_energy = Mock(return_value=True)
+ # These are pinned-contract tests for the legacy You/Others-only
+ # behaviour, so the sidecar must be explicitly forced off — without
+ # this they'd pass by accident on a clean checkout (no binary) and
+ # break the moment a contributor has one built locally (see
+ # TranscribeDiarisedMultiSpeakerTests for the sidecar-present cases).
+ self._diar_patcher = patch("src.transcriber._run_steno_diarize", return_value=None)
+ self._diar_patcher.start()
def tearDown(self):
+ self._diar_patcher.stop()
self._tmp.cleanup()
def test_interleaves_diarised_segments_with_timestamps(self):
@@ -112,6 +131,253 @@ def test_single_source_is_not_timestamped_or_diarised(self):
self.assertIsNone(result["diarised_text"])
+class TranscribeDiarisedMultiSpeakerTests(unittest.TestCase):
+ """Acoustic per-channel diarization (steno-diarize sidecar) layered on
+ top of the legacy You/Others channel split. Mocks _run_steno_diarize
+ directly (module-level, not an instance method) since transcribe_diarised
+ calls it as a free function via _tag_channel_segments."""
+
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ d = Path(self._tmp.name)
+ self.audio_path = d / "source.wav"
+ self.mic_path = d / "mic.wav"
+ self.system_path = d / "system.wav"
+ for p in (self.audio_path, self.mic_path, self.system_path):
+ p.write_bytes(b"stub")
+ self.transcriber = WhisperTranscriber.__new__(WhisperTranscriber)
+ self.transcriber.backend = "parakeet"
+ self.transcriber._split_stereo_to_channels = Mock(
+ return_value=(self.mic_path, self.system_path, 10.0)
+ )
+ self.transcriber._check_rms_energy = Mock(return_value=True)
+
+ def tearDown(self):
+ self._tmp.cleanup()
+
+ def test_two_speakers_on_mic_channel_become_you_and_speaker_two(self):
+ # Mic channel has two acoustic clusters (SPEAKER_0 dominant at 5s
+ # total, SPEAKER_1 minor at 2s total); system channel has a single
+ # trivial cluster so is_diarised (which requires both channels to
+ # contribute) stays True.
+ mic_diar = [
+ {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"},
+ {"start": 2.5, "end": 4.5, "speaker": "SPEAKER_1"},
+ {"start": 5.0, "end": 8.0, "speaker": "SPEAKER_0"},
+ ]
+ system_diar = [{"start": 9.0, "end": 9.5, "speaker": "SPEAKER_0"}]
+ with patch("src.transcriber._run_steno_diarize", side_effect=[mic_diar, system_diar]):
+ self.transcriber.transcribe_audio = Mock(side_effect=[
+ {"text": "Hi there. Not bad. Great.", "segments": [
+ {"text": "Hi there.", "start": 0.5, "end": 1.5},
+ {"text": "Not bad.", "start": 3.0, "end": 3.8},
+ {"text": "Great.", "start": 6.0, "end": 6.8},
+ ]},
+ {"text": "Ok.", "segments": [{"text": "Ok.", "start": 9.2, "end": 9.4}]},
+ ])
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertTrue(result["is_diarised"])
+ self.assertIn("[You] Hi there.", result["diarised_text"])
+ self.assertIn("[Speaker 2] Not bad.", result["diarised_text"])
+ self.assertIn("[You] Great.", result["diarised_text"])
+ self.assertIn("[Others] Ok.", result["diarised_text"])
+
+ def test_mic_only_multi_speaker_is_diarised_even_with_system_silent(self):
+ # Regression: an in-person conversation with no computer audio
+ # playing at all (system channel genuinely silent, not bled/dropped)
+ # must still produce a labelled transcript from the mic channel's
+ # own acoustic diarization. The old is_diarised computation
+ # (bool(mic_segments) and bool(system_segments)) discarded the
+ # whole labelled transcript whenever system was empty, even though
+ # _tag_channel_segments had already split mic into You + Speaker 2.
+ mic_diar = [
+ {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"},
+ {"start": 2.5, "end": 4.5, "speaker": "SPEAKER_1"},
+ {"start": 5.0, "end": 8.0, "speaker": "SPEAKER_0"},
+ ]
+ with patch("src.transcriber._run_steno_diarize", return_value=mic_diar):
+ self.transcriber._check_rms_energy = Mock(side_effect=[True, False])
+ self.transcriber.transcribe_audio = Mock(return_value={
+ "text": "Hi there. Not bad. Great.", "segments": [
+ {"text": "Hi there.", "start": 0.5, "end": 1.5},
+ {"text": "Not bad.", "start": 3.0, "end": 3.8},
+ {"text": "Great.", "start": 6.0, "end": 6.8},
+ ],
+ })
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertTrue(result["is_diarised"])
+ self.assertIsNotNone(result["diarised_text"])
+ self.assertIn("[You] Hi there.", result["diarised_text"])
+ self.assertIn("[Speaker 2] Not bad.", result["diarised_text"])
+ self.assertIn("[You] Great.", result["diarised_text"])
+
+ def test_speaker_numbering_is_chronological_across_both_channels(self):
+ # System's placeholder speaker turns up chronologically before
+ # mic's placeholder speaker, so it must be numbered "Speaker 2"
+ # even though the mic channel's diarization result is processed
+ # first in transcribe_diarised. Dominant/minor durations are
+ # clearly unequal (4s vs 1s) so cluster dominance is unambiguous.
+ system_diar = [
+ {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_1"}, # minor, first chronologically
+ {"start": 10.0, "end": 14.0, "speaker": "SPEAKER_0"}, # dominant -> "Others"
+ ]
+ mic_diar = [
+ {"start": 15.0, "end": 16.0, "speaker": "SPEAKER_1"}, # minor
+ {"start": 20.0, "end": 24.0, "speaker": "SPEAKER_0"}, # dominant -> "You"
+ ]
+ with patch("src.transcriber._run_steno_diarize", side_effect=[mic_diar, system_diar]):
+ self.transcriber.transcribe_audio = Mock(side_effect=[
+ {"text": "Mic minor. Mic dominant.", "segments": [
+ {"text": "Mic minor.", "start": 15.2, "end": 15.8},
+ {"text": "Mic dominant.", "start": 21.0, "end": 21.5},
+ ]},
+ {"text": "Sys minor. Sys dominant.", "segments": [
+ {"text": "Sys minor.", "start": 0.2, "end": 0.8},
+ {"text": "Sys dominant.", "start": 11.0, "end": 11.5},
+ ]},
+ ])
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ # System's minority cluster appears first chronologically (t=0.0)
+ # so it gets "Speaker 2"; mic's minority cluster (t=15.0) gets
+ # "Speaker 3" even though mic is diarized first in the pipeline.
+ # Each turn is timestamped by the diarizer's own segment boundary
+ # (not the ASR sentence start) — see _tag_channel_segments.
+ self.assertEqual(
+ result["diarised_text"],
+ "[00:00] [Speaker 2] Sys minor."
+ "\n\n[00:10] [Others] Sys dominant."
+ "\n\n[00:15] [Speaker 3] Mic minor."
+ "\n\n[00:20] [You] Mic dominant.",
+ )
+
+ def test_single_cluster_per_channel_is_byte_identical_to_legacy(self):
+ # Sidecar runs successfully but finds only one speaker per channel —
+ # must fall back to plain You/Others, not "Speaker 1" everywhere.
+ mic_diar = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}]
+ system_diar = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}]
+ with patch("src.transcriber._run_steno_diarize", side_effect=[mic_diar, system_diar]):
+ self.transcriber.transcribe_audio = Mock(side_effect=[
+ {"text": "Hello.", "segments": [{"text": "Hello.", "start": 1.0, "end": 1.5}]},
+ {"text": "Reply.", "segments": [{"text": "Reply.", "start": 2.0, "end": 2.5}]},
+ ])
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertEqual(
+ result["diarised_text"],
+ "[00:01] [You] Hello.\n\n[00:02] [Others] Reply.",
+ )
+
+ def test_sidecar_failure_falls_back_without_failing_meeting(self):
+ # Missing binary / timeout / bad JSON all surface as None from
+ # _run_steno_diarize — transcribe_diarised must never fail the
+ # meeting because of it.
+ with patch("src.transcriber._run_steno_diarize", return_value=None):
+ self.transcriber.transcribe_audio = Mock(side_effect=[
+ {"text": "Hello.", "segments": [{"text": "Hello.", "start": 1.0, "end": 1.5}]},
+ {"text": "Reply.", "segments": [{"text": "Reply.", "start": 2.0, "end": 2.5}]},
+ ])
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertNotIn("transcription_failed", result)
+ self.assertEqual(
+ result["diarised_text"],
+ "[00:01] [You] Hello.\n\n[00:02] [Others] Reply.",
+ )
+
+
+class TranscribeDiarisedMonoTests(unittest.TestCase):
+ """Mono audio has no mic/system channel split to lean on, but a single
+ track can still contain multiple speakers (e.g. an imported in-person
+ recording). transcribe_diarised must run acoustic diarization directly
+ against the whole file in that case, instead of unconditionally skipping
+ diarization the way the pre-diarization mono fallback did."""
+
+ def setUp(self):
+ self._tmp = tempfile.TemporaryDirectory()
+ d = Path(self._tmp.name)
+ self.audio_path = d / "mono.wav"
+ self.audio_path.write_bytes(b"stub")
+ self.transcriber = WhisperTranscriber.__new__(WhisperTranscriber)
+ self.transcriber.backend = "parakeet"
+ self.transcriber._split_stereo_to_channels = Mock(return_value=(None, None, None))
+
+ def tearDown(self):
+ self._tmp.cleanup()
+
+ def test_two_speakers_in_mono_file_become_you_and_speaker_two(self):
+ diar_segments = [
+ {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"},
+ {"start": 2.5, "end": 4.5, "speaker": "SPEAKER_1"},
+ {"start": 5.0, "end": 8.0, "speaker": "SPEAKER_0"},
+ ]
+ with patch("src.transcriber._run_steno_diarize", return_value=diar_segments):
+ self.transcriber.transcribe_audio = Mock(return_value={
+ "text": "Hi there. Not bad. Great.",
+ "segments": [
+ {"text": "Hi there.", "start": 0.5, "end": 1.5},
+ {"text": "Not bad.", "start": 3.0, "end": 3.8},
+ {"text": "Great.", "start": 6.0, "end": 6.8},
+ ],
+ "duration_seconds": 8.0,
+ })
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertTrue(result["is_diarised"])
+ self.assertIn("[You] Hi there.", result["diarised_text"])
+ self.assertIn("[Speaker 2] Not bad.", result["diarised_text"])
+ self.assertIn("[You] Great.", result["diarised_text"])
+ # The plain text field is untouched by diarisation.
+ self.assertEqual(result["text"], "Hi there. Not bad. Great.")
+
+ def test_single_speaker_mono_is_not_diarised(self):
+ # Byte-identical-to-legacy fast path: one real cluster means nothing
+ # to disambiguate, matching the pre-diarization mono behaviour.
+ diar_segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}]
+ with patch("src.transcriber._run_steno_diarize", return_value=diar_segments):
+ self.transcriber.transcribe_audio = Mock(return_value={
+ "text": "Just me talking.",
+ "segments": [{"text": "Just me talking.", "start": 0.5, "end": 1.5}],
+ "duration_seconds": 5.0,
+ })
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertFalse(result["is_diarised"])
+ self.assertIsNone(result["diarised_text"])
+
+ def test_sidecar_failure_falls_back_without_diarisation(self):
+ with patch("src.transcriber._run_steno_diarize", return_value=None):
+ self.transcriber.transcribe_audio = Mock(return_value={
+ "text": "Hello world.",
+ "segments": [{"text": "Hello world.", "start": 0.5, "end": 1.5}],
+ "duration_seconds": 2.0,
+ })
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ self.assertFalse(result["is_diarised"])
+ self.assertIsNone(result["diarised_text"])
+ self.assertEqual(result["text"], "Hello world.")
+
+ def test_transcription_failure_propagates_without_diarizing(self):
+ with patch("src.transcriber._run_steno_diarize") as mock_diarize:
+ self.transcriber.transcribe_audio = Mock(return_value={
+ "text": None,
+ "segments": [],
+ "transcription_failed": True,
+ "error": "boom",
+ })
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ mock_diarize.assert_not_called()
+ self.assertTrue(result["transcription_failed"])
+ self.assertFalse(result["is_diarised"])
+ self.assertIsNone(result["diarised_text"])
+
+ def test_empty_transcription_does_not_attempt_diarization(self):
+ with patch("src.transcriber._run_steno_diarize") as mock_diarize:
+ self.transcriber.transcribe_audio = Mock(return_value={
+ "text": "No speech detected in audio",
+ "segments": [],
+ "transcription_empty": True,
+ })
+ result = self.transcriber.transcribe_diarised(self.audio_path)
+ mock_diarize.assert_not_called()
+ self.assertFalse(result["is_diarised"])
+
+
class TokenJaccardTests(unittest.TestCase):
def test_identical_strings_score_one(self):
self.assertEqual(_token_jaccard("hello world", "hello world"), 1.0)
@@ -343,5 +609,344 @@ def test_returns_int(self):
self.assertIsInstance(_diarised_split_timeout(1234.5), int)
+class MergeCloseDiarSegmentsTests(unittest.TestCase):
+ def test_empty_input_returns_empty(self):
+ self.assertEqual(_merge_close_diar_segments([], 0.3), [])
+
+ def test_merges_same_speaker_within_gap(self):
+ segments = [
+ {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"},
+ {"start": 1.2, "end": 2.0, "speaker": "SPEAKER_0"},
+ ]
+ merged = _merge_close_diar_segments(segments, 0.3)
+ self.assertEqual(merged, [{"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"}])
+
+ def test_does_not_merge_across_gap_larger_than_threshold(self):
+ segments = [
+ {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"},
+ {"start": 2.0, "end": 3.0, "speaker": "SPEAKER_0"},
+ ]
+ merged = _merge_close_diar_segments(segments, 0.3)
+ self.assertEqual(len(merged), 2)
+
+ def test_does_not_merge_different_speakers(self):
+ segments = [
+ {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"},
+ {"start": 1.1, "end": 2.0, "speaker": "SPEAKER_1"},
+ ]
+ merged = _merge_close_diar_segments(segments, 0.3)
+ self.assertEqual(len(merged), 2)
+
+ def test_does_not_mutate_input(self):
+ segments = [{"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}]
+ _merge_close_diar_segments(segments, STENO_DIARIZE_MERGE_GAP_S)
+ self.assertEqual(segments, [{"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}])
+
+
+class AssignAsrSegmentsToDiarSegmentsTests(unittest.TestCase):
+ def test_empty_diar_segments_is_a_no_op(self):
+ diar_segments = []
+ _assign_asr_segments_to_diar_segments(
+ [{"text": "Hello", "start": 0.0, "end": 1.0}], diar_segments
+ )
+ self.assertEqual(diar_segments, [])
+
+ def test_assigns_sentence_within_segment_bounds(self):
+ diar_segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}]
+ _assign_asr_segments_to_diar_segments(
+ [{"text": "Hello there", "start": 1.0, "end": 2.0}], diar_segments
+ )
+ self.assertEqual(diar_segments[0]["text"], "Hello there")
+
+ def test_assigns_multiple_sentences_to_nearest_segment(self):
+ diar_segments = [
+ {"start": 0.0, "end": 2.0, "speaker": "SPEAKER_0"},
+ {"start": 3.0, "end": 5.0, "speaker": "SPEAKER_1"},
+ ]
+ _assign_asr_segments_to_diar_segments(
+ [
+ {"text": "First.", "start": 0.5, "end": 1.0},
+ {"text": "Second.", "start": 3.5, "end": 4.0},
+ # Falls in the gap between segments (2.0-3.0) but its
+ # midpoint (2.6) is closer to the second segment's start.
+ {"text": "Gap.", "start": 2.4, "end": 2.8},
+ ],
+ diar_segments,
+ )
+ self.assertEqual(diar_segments[0]["text"], "First.")
+ self.assertEqual(diar_segments[1]["text"], "Second. Gap.")
+
+ def test_blank_sentences_are_skipped(self):
+ diar_segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}]
+ _assign_asr_segments_to_diar_segments(
+ [{"text": " ", "start": 1.0, "end": 2.0}], diar_segments
+ )
+ self.assertEqual(diar_segments[0]["text"], "")
+
+ def test_long_sentence_spanning_multiple_speakers_splits_by_word(self):
+ # Regression for a real observed failure: a single mic capturing two
+ # people can produce one long Parakeet "sentence" (no punctuation
+ # break) that actually spans a genuine back-and-forth. Whole-block
+ # midpoint assignment forced the entire run onto one speaker;
+ # word-level splitting should recover the real turn boundaries.
+ diar_segments = [
+ {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_0"},
+ {"start": 3.0, "end": 6.0, "speaker": "SPEAKER_1"},
+ ]
+ tokens = [
+ {"text": " one", "start": 0.5, "end": 1.0},
+ {"text": " two", "start": 1.0, "end": 1.5},
+ {"text": " three", "start": 4.5, "end": 5.0},
+ {"text": " four", "start": 5.0, "end": 5.5},
+ ]
+ _assign_asr_segments_to_diar_segments(
+ [{"text": "one two three four", "start": 0.5, "end": 5.5, "tokens": tokens}],
+ diar_segments,
+ )
+ self.assertEqual(diar_segments[0]["text"], "one two")
+ self.assertEqual(diar_segments[1]["text"], "three four")
+
+ def test_long_sentence_within_single_speaker_is_not_split(self):
+ # Long duration alone isn't enough to trigger splitting — the
+ # diarizer segments it overlaps must belong to more than one
+ # distinct speaker. Fragmented same-speaker segments (diarizer
+ # flicker) should still be treated as one block.
+ diar_segments = [
+ {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_0"},
+ {"start": 3.0, "end": 6.0, "speaker": "SPEAKER_0"},
+ ]
+ tokens = [
+ {"text": " one", "start": 0.5, "end": 1.0},
+ {"text": " two", "start": 5.0, "end": 5.5},
+ ]
+ _assign_asr_segments_to_diar_segments(
+ [{"text": "one two", "start": 0.5, "end": 5.5, "tokens": tokens}],
+ diar_segments,
+ )
+ self.assertEqual(diar_segments[0]["text"], "one two")
+ self.assertEqual(diar_segments[1]["text"], "")
+
+ def test_short_sentence_not_split_even_across_speakers(self):
+ # Below LONG_SENTENCE_SPLIT_THRESHOLD_S — must stay whole-block
+ # (matching the historical short-sentence behaviour) even though it
+ # technically overlaps two different speakers, to avoid tearing
+ # short utterances apart on noisy diarizer boundaries.
+ diar_segments = [
+ {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"},
+ {"start": 1.0, "end": 2.0, "speaker": "SPEAKER_1"},
+ ]
+ tokens = [
+ {"text": " hi", "start": 0.8, "end": 1.0},
+ {"text": " there", "start": 1.0, "end": 1.2},
+ ]
+ _assign_asr_segments_to_diar_segments(
+ [{"text": "hi there", "start": 0.8, "end": 1.2, "tokens": tokens}],
+ diar_segments,
+ )
+ texts = [d["text"] for d in diar_segments]
+ self.assertEqual(texts.count("hi there"), 1)
+
+ def test_long_sentence_without_tokens_falls_back_to_whole_block(self):
+ # No word-level timing (e.g. the whisper.cpp backend never
+ # populates "tokens") must never crash — falls back to the same
+ # whole-block nearest assignment as a normal sentence.
+ diar_segments = [
+ {"start": 0.0, "end": 3.0, "speaker": "SPEAKER_0"},
+ {"start": 3.0, "end": 6.0, "speaker": "SPEAKER_1"},
+ ]
+ _assign_asr_segments_to_diar_segments(
+ [{"text": "one two three four", "start": 0.5, "end": 5.5}],
+ diar_segments,
+ )
+ texts = [d["text"] for d in diar_segments]
+ self.assertEqual(texts.count("one two three four"), 1)
+
+
+class ClusterChannelLabelsTests(unittest.TestCase):
+ def test_single_speaker_returns_none(self):
+ segments = [{"start": 0.0, "end": 5.0, "speaker": "SPEAKER_0"}]
+ self.assertIsNone(_cluster_channel_labels(segments, "You"))
+
+ def test_empty_segments_returns_none(self):
+ self.assertIsNone(_cluster_channel_labels([], "You"))
+
+ def test_dominant_speaker_by_total_duration_keeps_legacy_label(self):
+ segments = [
+ {"start": 0.0, "end": 1.0, "speaker": "SPEAKER_0"}, # 1s
+ {"start": 1.0, "end": 6.0, "speaker": "SPEAKER_1"}, # 5s, dominant
+ ]
+ labels = _cluster_channel_labels(segments, "You")
+ self.assertEqual(labels["SPEAKER_1"], "You")
+ self.assertEqual(labels["SPEAKER_0"], "__diar__You__SPEAKER_0")
+
+ def test_overwhelmingly_dominant_speaker_returns_none(self):
+ # A tiny misdiarization blip (e.g. a ~0.3s noise artifact) must not
+ # spawn a phantom second speaker — regression for the real-world
+ # oversplitting issue observed on single-mic multi-person audio.
+ segments = [
+ {"start": 0.0, "end": 59.7, "speaker": "SPEAKER_0"},
+ {"start": 59.7, "end": 60.0, "speaker": "SPEAKER_1"}, # 0.5% of total
+ ]
+ self.assertIsNone(_cluster_channel_labels(segments, "You"))
+
+ def test_dominance_ratio_just_under_threshold_still_clusters(self):
+ total = 100.0
+ minor = total * (1 - CHANNEL_DOMINANCE_THRESHOLD) + 0.5 # comfortably above the gate
+ segments = [
+ {"start": 0.0, "end": total - minor, "speaker": "SPEAKER_0"},
+ {"start": total - minor, "end": total, "speaker": "SPEAKER_1"},
+ ]
+ self.assertIsNotNone(_cluster_channel_labels(segments, "You"))
+
+
+class ResolveSpeakerPlaceholdersTests(unittest.TestCase):
+ def test_legacy_labels_are_untouched(self):
+ tagged = [(0.0, "You", "hi"), (1.0, "Others", "hey")]
+ self.assertEqual(_resolve_speaker_placeholders(tagged), tagged)
+
+ def test_placeholders_numbered_from_two_by_first_appearance(self):
+ tagged = [
+ (0.0, "You", "a"),
+ (1.0, "__diar__You__SPEAKER_1", "b"),
+ (2.0, "__diar__Others__SPEAKER_1", "c"),
+ (3.0, "__diar__You__SPEAKER_1", "d"),
+ ]
+ resolved = _resolve_speaker_placeholders(tagged)
+ self.assertEqual(resolved[0], (0.0, "You", "a"))
+ self.assertEqual(resolved[1], (1.0, "Speaker 2", "b"))
+ self.assertEqual(resolved[2], (2.0, "Speaker 3", "c"))
+ # Same placeholder key reuses the same number on a later turn.
+ self.assertEqual(resolved[3], (3.0, "Speaker 2", "d"))
+
+
+class TagChannelSegmentsTests(unittest.TestCase):
+ def test_empty_asr_segments_returns_empty_without_diarizing(self):
+ with patch("src.transcriber._run_steno_diarize") as mock_run:
+ result = _tag_channel_segments([], Path("/fake/mic.wav"), 5.0, "You")
+ mock_run.assert_not_called()
+ self.assertEqual(result, [])
+
+ def test_no_channel_path_uses_legacy_labeling(self):
+ asr_segments = [{"text": "Hi.", "start": 0.0, "end": 1.0}]
+ result = _tag_channel_segments(asr_segments, None, 5.0, "You")
+ self.assertEqual(result, [(0.0, "You", "Hi.")])
+
+
+class _FakePopen:
+ """Stand-in for subprocess.Popen, matching only the surface
+ _run_steno_diarize actually uses: .stdout/.stderr as readable byte
+ streams (plain io.BytesIO works fine -- the two reader threads each
+ only ever touch their own stream, so there's no real cross-thread
+ contention to simulate), .wait(timeout=...), .kill(), .returncode.
+ """
+
+ def __init__(self, stdout=b"", stderr=b"", returncode=0, raise_timeout_once=False):
+ self.stdout = io.BytesIO(stdout)
+ self.stderr = io.BytesIO(stderr)
+ self._final_returncode = returncode
+ self._raise_timeout_once = raise_timeout_once
+ self.returncode = None
+ self.killed = False
+
+ def wait(self, timeout=None):
+ if self._raise_timeout_once and timeout is not None:
+ self._raise_timeout_once = False
+ raise subprocess.TimeoutExpired(cmd="steno-diarize", timeout=timeout)
+ self.returncode = self._final_returncode
+ return self.returncode
+
+ def kill(self):
+ self.killed = True
+
+
+def _patch_popen(**kwargs):
+ return patch("subprocess.Popen", return_value=_FakePopen(**kwargs))
+
+
+class RunStenoDiarizeTests(unittest.TestCase):
+ """_run_steno_diarize must survive the sidecar's real quirks: a
+ diagnostic warning printed to stdout ahead of the JSON payload, and any
+ kind of failure (missing binary, timeout, bad exit, bad JSON)."""
+
+ def test_returns_none_when_binary_unresolved(self):
+ with patch("src.transcriber._resolve_steno_diarize", return_value=None):
+ self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60))
+
+ def test_parses_json_with_e5rt_warning_prefix_on_stdout(self):
+ payload = json.dumps([
+ {"speakerId": "SPEAKER_1", "start": 1.0, "end": 2.0},
+ {"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9},
+ ]).encode()
+ stdout = b"E5RT encountered an STL exception. msg = unordered_map::at: key not found." + payload
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=stdout, stderr=b"", returncode=0):
+ result = _run_steno_diarize(Path("/fake/mic.wav"), 60)
+ self.assertEqual(
+ result,
+ [
+ {"start": 0.0, "end": 0.9, "speaker": "SPEAKER_0"},
+ {"start": 1.0, "end": 2.0, "speaker": "SPEAKER_1"},
+ ],
+ )
+
+ def test_parses_json_with_trailing_warning_after_payload(self):
+ # A late CoreML/Metal warning printed to stdout AFTER the JSON
+ # payload (at teardown) -- json.loads() requires the entire
+ # remaining string to be clean JSON and raises "Extra data" on
+ # trailing text, discarding an otherwise-successful diarization
+ # result. raw_decode() must tolerate this.
+ payload = json.dumps([{"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}]).encode()
+ stdout = payload + b"\nMetal warning: some late teardown message"
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=stdout, stderr=b"", returncode=0):
+ result = _run_steno_diarize(Path("/fake/mic.wav"), 60)
+ self.assertEqual(result, [{"start": 0.0, "end": 0.9, "speaker": "SPEAKER_0"}])
+
+ def test_skips_an_interstitial_array_that_is_not_segment_shaped(self):
+ # A non-payload array (no "speakerId" keys) printed BEFORE the real
+ # payload must not be mistaken for it -- keep scanning for an array
+ # of segment-shaped dicts.
+ real_payload = json.dumps([{"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}]).encode()
+ stdout = b'["not", "a", "segment"]\n' + real_payload
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=stdout, stderr=b"", returncode=0):
+ result = _run_steno_diarize(Path("/fake/mic.wav"), 60)
+ self.assertEqual(result, [{"start": 0.0, "end": 0.9, "speaker": "SPEAKER_0"}])
+
+ def test_prefers_the_last_matching_payload_when_multiple_exist(self):
+ # If more than one array in stdout looks segment-shaped (shouldn't
+ # normally happen, but the scan must have a defined, sane tie-break
+ # rather than an arbitrary one) -- the real payload is printed once,
+ # at the end, when diarization actually finishes, so prefer the
+ # LAST match.
+ first_payload = json.dumps([{"speakerId": "SPEAKER_0", "start": 0.0, "end": 0.9}]).encode()
+ second_payload = json.dumps([{"speakerId": "SPEAKER_1", "start": 5.0, "end": 6.0}]).encode()
+ stdout = first_payload + b"\n" + second_payload
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=stdout, stderr=b"", returncode=0):
+ result = _run_steno_diarize(Path("/fake/mic.wav"), 60)
+ self.assertEqual(result, [{"start": 5.0, "end": 6.0, "speaker": "SPEAKER_1"}])
+
+ def test_nonzero_exit_returns_none(self):
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=b"", stderr=b"boom", returncode=1):
+ self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60))
+
+ def test_unparseable_json_returns_none(self):
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=b"[not json", stderr=b"", returncode=0):
+ self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60))
+
+ def test_no_bracket_in_stdout_returns_none(self):
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(stdout=b"nothing useful", stderr=b"", returncode=0):
+ self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60))
+
+ def test_timeout_returns_none(self):
+ with patch("src.transcriber._resolve_steno_diarize", return_value="/fake/steno-diarize"), \
+ _patch_popen(raise_timeout_once=True):
+ self.assertIsNone(_run_steno_diarize(Path("/fake/mic.wav"), 60))
+
+
if __name__ == '__main__':
unittest.main()