From 3f88ff70404728e4fcb695f7d0f3e6c23aeb617f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Apr 2026 23:10:32 +0000 Subject: [PATCH 01/11] Improve TTS streaming chunking for better prosody and gapless playback Chunk 0 fires at the first punctuation boundary (fast start), while subsequent chunks accumulate tokens and trigger synthesis based on a rolling-average synthesis time vs remaining playback window. This maximises text per synthesis call (better Chirp3-HD prosody) while keeping playback gapless. Key changes in speak_streaming_async: - Two-phase chunking: fast-start chunk 0 + timing-aware chunk 1+ - Rolling average synthesis time (seeded at 400ms) drives trigger - Playback duration tracking via _get_audio_duration helper - Stall detection with logged warnings for self-correcting average - All state is per-call (reset each invocation) https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 200 ++++++++++++++++++++++++--------- 1 file changed, 148 insertions(+), 52 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index e10d250..0c92a87 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -461,22 +461,83 @@ def speech_worker(): speech_queue.put(None) # Stop the worker speech_thread.join() + @staticmethod + def _get_audio_duration(file_path: str) -> float: + """Get the duration of an audio file in seconds using pygame. + + Falls back to a file-size estimate (~4 KB/s for Google TTS MP3) if + pygame.mixer.Sound cannot determine the length. + """ + try: + sound = pygame.mixer.Sound(file_path) + duration = sound.get_length() + del sound + if duration > 0: + return duration + except Exception: + pass + # Fallback: estimate from file size assuming ~32 kbps MP3 (4 KB/s) + try: + return os.path.getsize(file_path) / 4000.0 + except Exception: + return 0.0 + def speak_streaming_async(self, text_generator, chunk_on: str = ".", print_text: bool = True, min_chunk_size: int = 15): """ Speak text as it's being generated with truly parallel processing. Multiple sentences can be synthesized and queued while others are playing. This provides the fastest response time. - + + Chunking strategy + ----------------- + * **Chunk 0 (fast start):** fire synthesis at the first punctuation + boundary (including commas) to minimise time-to-first-sound. + * **Chunk 1+ (quality chunks):** accumulate tokens and trigger synthesis + at the first sentence-ending boundary (``.!?``) where the remaining + playback time of previously queued audio drops to within + ``avg_synthesis_time + 200 ms``. This maximises input length for + better prosody while keeping playback gapless. + :param text_generator: Generator that yields text tokens - :param chunk_on: Character to chunk on (default: "." for sentences) + :param chunk_on: Character to chunk on for chunk 0 (default: ".") :param print_text: If True, print the text as it's being spoken :param min_chunk_size: Minimum characters before considering a chunk (prevents tiny fragments, default 15) """ buffer = "" synthesis_queue = queue.Queue() playback_queue = queue.Queue() - + + # ---- per-call timing state (reset each invocation) ---- + chunk_index = 0 + synthesis_times: list[float] = [] + avg_synthesis_time = 0.4 # seeded at 400 ms + + # Shared playback state – accessed from main + playback threads + state_lock = threading.Lock() + state = { + "playback_active": False, + "play_start": 0.0, # time.monotonic() when current file started + "play_duration": 0.0, # duration in seconds of current file + "queued_durations": [], # durations of files waiting in playback_queue + } + + def _remaining_playback_time() -> float: + """Total seconds of audio still to play (current + queued).""" + with state_lock: + remaining = 0.0 + if state["playback_active"]: + elapsed = time.monotonic() - state["play_start"] + remaining = max(0.0, state["play_duration"] - elapsed) + remaining += sum(state["queued_durations"]) + return remaining + + def _update_avg(duration: float): + nonlocal avg_synthesis_time + synthesis_times.append(duration) + avg_synthesis_time = sum(synthesis_times) / len(synthesis_times) + + # ---- worker threads ---- def synthesis_worker(): """Worker thread that synthesizes speech.""" while True: @@ -485,98 +546,133 @@ def synthesis_worker(): synthesis_queue.task_done() playback_queue.put(None) # Signal playback worker break - + text_to_speak = item try: - # Create a temporary file for the audio with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as temp_file: temp_filename = temp_file.name - - # Synthesize speech to the temp file + + t0 = time.monotonic() audio_file = self.synthesize_to_file(text_to_speak, temp_filename) + synth_dur = time.monotonic() - t0 + if audio_file: - playback_queue.put(audio_file) + audio_dur = self._get_audio_duration(audio_file) + _update_avg(synth_dur) + + with state_lock: + state["queued_durations"].append(audio_dur) + + playback_queue.put((audio_file, audio_dur)) except Exception as e: print(f"\n❌ Synthesis error: {e}") - + synthesis_queue.task_done() - + def playback_worker(): """Worker thread that plays synthesized audio.""" + last_end: float = 0.0 # monotonic time previous chunk finished while True: - audio_file = playback_queue.get() - if audio_file is None: + item = playback_queue.get() + if item is None: playback_queue.task_done() break - + + audio_file, audio_dur = item try: + # Detect stalls (playback queue went empty) + now = time.monotonic() + if last_end > 0 and (now - last_end) > 0.15: + print(f"\n⚠️ Playback stall: waited {now - last_end:.1f}s for next chunk") + + with state_lock: + if state["queued_durations"]: + state["queued_durations"].pop(0) + state["play_start"] = time.monotonic() + state["play_duration"] = audio_dur + state["playback_active"] = True + pygame.mixer.music.load(audio_file) pygame.mixer.music.play() - - # Wait for playback to finish + while pygame.mixer.music.get_busy(): - time.sleep(0.1) - - # Clean up the temporary file + time.sleep(0.05) + + last_end = time.monotonic() + + with state_lock: + state["playback_active"] = False + try: os.remove(audio_file) - except: + except Exception: pass except Exception as e: print(f"\n❌ Playback error: {e}") - + with state_lock: + state["playback_active"] = False + playback_queue.task_done() - + # Start worker threads synthesis_thread = threading.Thread(target=synthesis_worker, daemon=True) playback_thread = threading.Thread(target=playback_worker, daemon=True) synthesis_thread.start() playback_thread.start() - + try: - # Process tokens from the LLM for token in text_generator: if print_text: print(token.content, end="", flush=True) buffer += token.content - - # Check if we have potential sentence boundaries - if any(c in buffer for c in chunk_on): - # Find the last valid sentence boundary - last_chunk_idx = self._find_sentence_boundary(buffer, chunk_on) - - if last_chunk_idx >= 0: - # Extract the complete sentence(s) - to_speak = buffer[:last_chunk_idx + 1].strip() - - # Smart chunking: use different min sizes based on delimiter type - # Complete sentences (.!?) can be shorter, commas need more context - delimiter = buffer[last_chunk_idx] if last_chunk_idx < len(buffer) else '' - effective_min = 5 if delimiter in '.!?' else min_chunk_size - - # Only chunk if we have substantial content (prevents tiny fragments) - # This ensures we don't speak very short incomplete phrases - if len(to_speak) >= effective_min: - # Keep the remainder for the next iteration - buffer = buffer[last_chunk_idx + 1:] - - # Queue for synthesis (non-blocking) - synthesis_queue.put(to_speak) - - # Process any remaining text + + if chunk_index == 0: + # --- Chunk 0: fast start – fire at first boundary (incl. commas) --- + if any(c in buffer for c in chunk_on): + last_chunk_idx = self._find_sentence_boundary(buffer, chunk_on) + if last_chunk_idx >= 0: + to_speak = buffer[:last_chunk_idx + 1].strip() + delimiter = buffer[last_chunk_idx] + effective_min = 5 if delimiter in '.!?' else min_chunk_size + if len(to_speak) >= effective_min: + buffer = buffer[last_chunk_idx + 1:] + synthesis_queue.put(to_speak) + chunk_index += 1 + else: + # --- Chunk 1+: quality chunks – sentence boundaries only --- + sentence_boundary = self._find_sentence_boundary(buffer, ".!?") + if sentence_boundary < 0: + continue + + remaining = _remaining_playback_time() + threshold = avg_synthesis_time + 0.2 # 200 ms margin + + if remaining <= threshold: + to_speak = buffer[:sentence_boundary + 1].strip() + + # Edge case: chunk 0 still playing with time to spare + # and buffer text is very short – hold for more tokens + if remaining > threshold * 2 and len(to_speak) < min_chunk_size: + continue + + buffer = buffer[sentence_boundary + 1:] + synthesis_queue.put(to_speak) + chunk_index += 1 + + # LLM stream ended – synthesize whatever is buffered if buffer.strip(): synthesis_queue.put(buffer.strip()) - + # Signal workers to stop synthesis_queue.put(None) - + # Wait for all work to complete synthesis_thread.join() playback_thread.join() - + if print_text: print() # New line at the end - + except Exception as e: print(f"\n❌ Error in async streaming speech: {e}") synthesis_queue.put(None) From fc73c27f1a3146e81647aaa070b388147e75c3fd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Apr 2026 23:37:16 +0000 Subject: [PATCH 02/11] Address Copilot review feedback on TTS chunking PR - Fix chunk_on default to ",.!?" so chunk 0 fires on commas as documented - Replace unbounded synthesis_times list with O(1) incremental mean - Replace queued_durations list with a single queued_total float for O(1) remaining-time computation - Gate chunk 1+ trigger on has_audio flag so we don't fire prematurely before chunk 0 synthesis completes - Move playback temp-file cleanup and state reset into finally block to prevent file leaks on exceptions https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 40 ++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index 0c92a87..9d0a958 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -482,7 +482,7 @@ def _get_audio_duration(file_path: str) -> float: except Exception: return 0.0 - def speak_streaming_async(self, text_generator, chunk_on: str = ".", print_text: bool = True, + def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_text: bool = True, min_chunk_size: int = 15): """ Speak text as it's being generated with truly parallel processing. @@ -510,7 +510,8 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ".", print_text: # ---- per-call timing state (reset each invocation) ---- chunk_index = 0 - synthesis_times: list[float] = [] + synthesis_count = 0 + synthesis_sum = 0.0 avg_synthesis_time = 0.4 # seeded at 400 ms # Shared playback state – accessed from main + playback threads @@ -519,7 +520,8 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ".", print_text: "playback_active": False, "play_start": 0.0, # time.monotonic() when current file started "play_duration": 0.0, # duration in seconds of current file - "queued_durations": [], # durations of files waiting in playback_queue + "queued_total": 0.0, # total duration of files waiting in playback_queue + "has_audio": False, # True once first synthesis result is queued } def _remaining_playback_time() -> float: @@ -529,13 +531,14 @@ def _remaining_playback_time() -> float: if state["playback_active"]: elapsed = time.monotonic() - state["play_start"] remaining = max(0.0, state["play_duration"] - elapsed) - remaining += sum(state["queued_durations"]) + remaining += state["queued_total"] return remaining def _update_avg(duration: float): - nonlocal avg_synthesis_time - synthesis_times.append(duration) - avg_synthesis_time = sum(synthesis_times) / len(synthesis_times) + nonlocal avg_synthesis_time, synthesis_count, synthesis_sum + synthesis_count += 1 + synthesis_sum += duration + avg_synthesis_time = synthesis_sum / synthesis_count # ---- worker threads ---- def synthesis_worker(): @@ -561,7 +564,8 @@ def synthesis_worker(): _update_avg(synth_dur) with state_lock: - state["queued_durations"].append(audio_dur) + state["queued_total"] += audio_dur + state["has_audio"] = True playback_queue.put((audio_file, audio_dur)) except Exception as e: @@ -586,8 +590,7 @@ def playback_worker(): print(f"\n⚠️ Playback stall: waited {now - last_end:.1f}s for next chunk") with state_lock: - if state["queued_durations"]: - state["queued_durations"].pop(0) + state["queued_total"] = max(0.0, state["queued_total"] - audio_dur) state["play_start"] = time.monotonic() state["play_duration"] = audio_dur state["playback_active"] = True @@ -599,18 +602,15 @@ def playback_worker(): time.sleep(0.05) last_end = time.monotonic() - + except Exception as e: + print(f"\n❌ Playback error: {e}") + finally: with state_lock: state["playback_active"] = False - try: os.remove(audio_file) except Exception: pass - except Exception as e: - print(f"\n❌ Playback error: {e}") - with state_lock: - state["playback_active"] = False playback_queue.task_done() @@ -644,6 +644,14 @@ def playback_worker(): if sentence_boundary < 0: continue + # Wait until chunk 0 has been synthesised and we have a real + # playback window to compare against; otherwise remaining==0 + # would trigger immediately before any audio is ready. + with state_lock: + has_audio = state["has_audio"] + if not has_audio: + continue + remaining = _remaining_playback_time() threshold = avg_synthesis_time + 0.2 # 200 ms margin From 469f70a69104c8166ea9bf65b05a6fbbae020ac9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Apr 2026 12:07:26 +0000 Subject: [PATCH 03/11] Address second round of Copilot review feedback - Chunk 0 now uses first_only=True boundary search so synthesis fires at the earliest valid boundary, minimising time-to-first-audio - Moved synthesis timing state (count/sum/avg) into shared state dict under state_lock to eliminate cross-thread data race - Short-circuit chunk 1+ boundary scan: skip full buffer rescan when the incoming token contains no sentence-ending punctuation (.!?) - Clean up temp MP3 file when synthesis returns falsy or throws, preventing file leaks on quota blocks or errors https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/chunking.py | 21 ++++++++++++---- src/myai/tts/text_to_speech.py | 46 +++++++++++++++++++++++----------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/myai/tts/chunking.py b/src/myai/tts/chunking.py index af56f87..16a46bd 100644 --- a/src/myai/tts/chunking.py +++ b/src/myai/tts/chunking.py @@ -95,26 +95,37 @@ def is_weak_comma(text: str, position: int) -> bool: return False -def find_sentence_boundary(text: str, chunk_chars: str = ".!?") -> int: - """Find the last valid boundary index in text, or -1 if none found.""" +def find_sentence_boundary(text: str, chunk_chars: str = ".!?", first_only: bool = False) -> int: + """Find a valid boundary index in text, or -1 if none found. + + When *first_only* is True the first valid boundary is returned (useful for + fast-start chunking). Otherwise the last valid boundary is returned + (maximises chunk length for better prosody). + """ last_valid_boundary = -1 for i, char in enumerate(text): if char not in chunk_chars: continue + valid = False if char == ".": if is_sentence_boundary(text, i): - last_valid_boundary = i + valid = True elif char == ",": if not is_weak_comma(text, i): - last_valid_boundary = i + valid = True elif char == "—": if i > 0 and (i + 1 >= len(text) or text[i + 1] != "—"): - last_valid_boundary = i + valid = True else: if i > 0 and text[i - 1] in "!?": continue + valid = True + + if valid: + if first_only: + return i last_valid_boundary = i return last_valid_boundary diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index 9d0a958..9cd5129 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -510,11 +510,8 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_te # ---- per-call timing state (reset each invocation) ---- chunk_index = 0 - synthesis_count = 0 - synthesis_sum = 0.0 - avg_synthesis_time = 0.4 # seeded at 400 ms - # Shared playback state – accessed from main + playback threads + # Shared state – accessed from main + worker threads under state_lock state_lock = threading.Lock() state = { "playback_active": False, @@ -522,6 +519,9 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_te "play_duration": 0.0, # duration in seconds of current file "queued_total": 0.0, # total duration of files waiting in playback_queue "has_audio": False, # True once first synthesis result is queued + "synth_count": 0, # number of completed syntheses + "synth_sum": 0.0, # cumulative synthesis duration + "avg_synth": 0.4, # rolling average, seeded at 400 ms } def _remaining_playback_time() -> float: @@ -535,10 +535,10 @@ def _remaining_playback_time() -> float: return remaining def _update_avg(duration: float): - nonlocal avg_synthesis_time, synthesis_count, synthesis_sum - synthesis_count += 1 - synthesis_sum += duration - avg_synthesis_time = synthesis_sum / synthesis_count + with state_lock: + state["synth_count"] += 1 + state["synth_sum"] += duration + state["avg_synth"] = state["synth_sum"] / state["synth_count"] # ---- worker threads ---- def synthesis_worker(): @@ -551,6 +551,7 @@ def synthesis_worker(): break text_to_speak = item + temp_filename = None try: with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as temp_file: temp_filename = temp_file.name @@ -568,8 +569,20 @@ def synthesis_worker(): state["has_audio"] = True playback_queue.put((audio_file, audio_dur)) + else: + # Synthesis returned nothing (e.g. quota block) – clean up + if temp_filename: + try: + os.remove(temp_filename) + except Exception: + pass except Exception as e: print(f"\n❌ Synthesis error: {e}") + if temp_filename: + try: + os.remove(temp_filename) + except Exception: + pass synthesis_queue.task_done() @@ -629,17 +642,21 @@ def playback_worker(): if chunk_index == 0: # --- Chunk 0: fast start – fire at first boundary (incl. commas) --- if any(c in buffer for c in chunk_on): - last_chunk_idx = self._find_sentence_boundary(buffer, chunk_on) - if last_chunk_idx >= 0: - to_speak = buffer[:last_chunk_idx + 1].strip() - delimiter = buffer[last_chunk_idx] + first_idx = find_sentence_boundary(buffer, chunk_on, first_only=True) + if first_idx >= 0: + to_speak = buffer[:first_idx + 1].strip() + delimiter = buffer[first_idx] effective_min = 5 if delimiter in '.!?' else min_chunk_size if len(to_speak) >= effective_min: - buffer = buffer[last_chunk_idx + 1:] + buffer = buffer[first_idx + 1:] synthesis_queue.put(to_speak) chunk_index += 1 else: # --- Chunk 1+: quality chunks – sentence boundaries only --- + # Short-circuit: skip the full buffer scan when the new + # token contains no sentence-ending punctuation. + if not any(c in token.content for c in ".!?"): + continue sentence_boundary = self._find_sentence_boundary(buffer, ".!?") if sentence_boundary < 0: continue @@ -649,11 +666,12 @@ def playback_worker(): # would trigger immediately before any audio is ready. with state_lock: has_audio = state["has_audio"] + current_avg_synth = state["avg_synth"] if not has_audio: continue remaining = _remaining_playback_time() - threshold = avg_synthesis_time + 0.2 # 200 ms margin + threshold = current_avg_synth + 0.2 # 200 ms margin if remaining <= threshold: to_speak = buffer[:sentence_boundary + 1].strip() From 406b95f420b34c6e97123a3cb77fdcc6bf3f580c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 3 Apr 2026 12:27:40 +0000 Subject: [PATCH 04/11] Fix cached-boundary timing and cross-thread pygame safety - Chunk 1+ now caches `pending_boundary` and re-evaluates the remaining-time condition on every token, even when the token has no punctuation. Only the buffer scan is skipped (not the timing check), so an existing boundary fires as soon as the window closes. - Replaced `_get_audio_duration` (which called pygame.mixer.Sound from the synthesis thread) with `_estimate_audio_duration` (file-size only, thread-safe). Accurate duration via pygame.mixer.Sound is now measured in the playback worker, keeping all pygame/SDL_mixer calls on a single thread. https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 59 ++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index 9cd5129..cdc7b5f 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -462,21 +462,13 @@ def speech_worker(): speech_thread.join() @staticmethod - def _get_audio_duration(file_path: str) -> float: - """Get the duration of an audio file in seconds using pygame. + def _estimate_audio_duration(file_path: str) -> float: + """Estimate the duration of an MP3 file from its size. - Falls back to a file-size estimate (~4 KB/s for Google TTS MP3) if - pygame.mixer.Sound cannot determine the length. + Uses ~32 kbps (4 KB/s), which is typical for Google Cloud TTS MP3 + output. This avoids calling any pygame API so it is safe to use + from any thread. """ - try: - sound = pygame.mixer.Sound(file_path) - duration = sound.get_length() - del sound - if duration > 0: - return duration - except Exception: - pass - # Fallback: estimate from file size assuming ~32 kbps MP3 (4 KB/s) try: return os.path.getsize(file_path) / 4000.0 except Exception: @@ -510,6 +502,7 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_te # ---- per-call timing state (reset each invocation) ---- chunk_index = 0 + pending_boundary = -1 # cached sentence boundary index for chunk 1+ # Shared state – accessed from main + worker threads under state_lock state_lock = threading.Lock() @@ -561,7 +554,7 @@ def synthesis_worker(): synth_dur = time.monotonic() - t0 if audio_file: - audio_dur = self._get_audio_duration(audio_file) + audio_dur = self._estimate_audio_duration(audio_file) _update_avg(synth_dur) with state_lock: @@ -595,17 +588,31 @@ def playback_worker(): playback_queue.task_done() break - audio_file, audio_dur = item + audio_file, estimated_dur = item try: # Detect stalls (playback queue went empty) now = time.monotonic() if last_end > 0 and (now - last_end) > 0.15: print(f"\n⚠️ Playback stall: waited {now - last_end:.1f}s for next chunk") + # Get accurate duration via pygame (all pygame calls + # happen on this single thread to avoid cross-thread + # SDL_mixer issues). + try: + snd = pygame.mixer.Sound(audio_file) + actual_dur = snd.get_length() + del snd + if actual_dur <= 0: + actual_dur = estimated_dur + except Exception: + actual_dur = estimated_dur + with state_lock: - state["queued_total"] = max(0.0, state["queued_total"] - audio_dur) + # Correct queued_total: remove the estimate, the + # play_duration field now uses the real value. + state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) state["play_start"] = time.monotonic() - state["play_duration"] = audio_dur + state["play_duration"] = actual_dur state["playback_active"] = True pygame.mixer.music.load(audio_file) @@ -653,12 +660,13 @@ def playback_worker(): chunk_index += 1 else: # --- Chunk 1+: quality chunks – sentence boundaries only --- - # Short-circuit: skip the full buffer scan when the new - # token contains no sentence-ending punctuation. - if not any(c in token.content for c in ".!?"): - continue - sentence_boundary = self._find_sentence_boundary(buffer, ".!?") - if sentence_boundary < 0: + # Only rescan the buffer when the new token contains + # sentence-ending punctuation; otherwise reuse the cached + # boundary so we still re-evaluate timing each token. + if any(c in token.content for c in ".!?"): + pending_boundary = self._find_sentence_boundary(buffer, ".!?") + + if pending_boundary < 0: continue # Wait until chunk 0 has been synthesised and we have a real @@ -674,16 +682,17 @@ def playback_worker(): threshold = current_avg_synth + 0.2 # 200 ms margin if remaining <= threshold: - to_speak = buffer[:sentence_boundary + 1].strip() + to_speak = buffer[:pending_boundary + 1].strip() # Edge case: chunk 0 still playing with time to spare # and buffer text is very short – hold for more tokens if remaining > threshold * 2 and len(to_speak) < min_chunk_size: continue - buffer = buffer[sentence_boundary + 1:] + buffer = buffer[pending_boundary + 1:] synthesis_queue.put(to_speak) chunk_index += 1 + pending_boundary = -1 # reset after consuming # LLM stream ended – synthesize whatever is buffered if buffer.strip(): From bce706168abda9dacf16032c9a0f3237db8cfd23 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Apr 2026 02:44:57 +0000 Subject: [PATCH 05/11] Fix docstring default and play_start timing accuracy - Update chunk_on docstring to reflect actual default ",.!?" - Move play_start assignment to after pygame.mixer.music.play() so elapsed-time calculations reflect real playback, not load latency https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index cdc7b5f..4aeed15 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -492,7 +492,7 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_te better prosody while keeping playback gapless. :param text_generator: Generator that yields text tokens - :param chunk_on: Character to chunk on for chunk 0 (default: ".") + :param chunk_on: Characters used to detect chunk 0 boundaries only (default: ``",.!?"``) :param print_text: If True, print the text as it's being spoken :param min_chunk_size: Minimum characters before considering a chunk (prevents tiny fragments, default 15) """ @@ -611,13 +611,17 @@ def playback_worker(): # Correct queued_total: remove the estimate, the # play_duration field now uses the real value. state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) - state["play_start"] = time.monotonic() state["play_duration"] = actual_dur state["playback_active"] = True pygame.mixer.music.load(audio_file) pygame.mixer.music.play() + # Set play_start after .play() so elapsed time reflects + # actual playback, not load latency. + with state_lock: + state["play_start"] = time.monotonic() + while pygame.mixer.music.get_busy(): time.sleep(0.05) From a4ed68b5c7c5f1c6ca710f3e1419d3399b43a0cc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Apr 2026 02:54:58 +0000 Subject: [PATCH 06/11] Fix chunk 0 boundary stall, seed pending_boundary, align docstring, add doctests - Chunk 0 now iterates over boundaries to find the first one meeting the min-length requirement, avoiding infinite stall on short first boundaries - After chunk 0 fires, immediately scan the remainder for sentence- ending boundaries and seed pending_boundary so chunk 1+ doesn't wait for a future token containing punctuation - Align chunk 1+ docstring: triggers at the *last* available sentence boundary (not "first") to maximise chunk length - Add doctests to find_sentence_boundary covering first_only, decimals, initials/abbreviations, and weak commas https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/chunking.py | 25 +++++++++++++++++++++ src/myai/tts/text_to_speech.py | 40 +++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 13 deletions(-) diff --git a/src/myai/tts/chunking.py b/src/myai/tts/chunking.py index 16a46bd..11edaff 100644 --- a/src/myai/tts/chunking.py +++ b/src/myai/tts/chunking.py @@ -101,6 +101,31 @@ def find_sentence_boundary(text: str, chunk_chars: str = ".!?", first_only: bool When *first_only* is True the first valid boundary is returned (useful for fast-start chunking). Otherwise the last valid boundary is returned (maximises chunk length for better prosody). + + >>> text = "Hello world. Another sentence!" + >>> find_sentence_boundary(text) + 29 + >>> find_sentence_boundary(text, first_only=True) + 11 + + Decimals should not be split at the period: + + >>> find_sentence_boundary("The value is 3.14 and rising.") + 28 + >>> find_sentence_boundary("Version 2.0") + -1 + + Initials / abbreviations stay connected: + + >>> find_sentence_boundary("Meet J. T. Smith.") + 16 + >>> find_sentence_boundary("Meet J. T. Smith.", first_only=True) + 16 + + When commas are enabled, weak commas are skipped: + + >>> find_sentence_boundary("Count 1, 2, 3", chunk_chars=",") + -1 """ last_valid_boundary = -1 diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index 4aeed15..d711544 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -485,11 +485,11 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_te ----------------- * **Chunk 0 (fast start):** fire synthesis at the first punctuation boundary (including commas) to minimise time-to-first-sound. - * **Chunk 1+ (quality chunks):** accumulate tokens and trigger synthesis - at the first sentence-ending boundary (``.!?``) where the remaining - playback time of previously queued audio drops to within - ``avg_synthesis_time + 200 ms``. This maximises input length for - better prosody while keeping playback gapless. + * **Chunk 1+ (quality chunks):** accumulate tokens and, once the + remaining playback time of previously queued audio drops to within + ``avg_synthesis_time + 200 ms``, trigger synthesis at the last + available sentence-ending boundary (``.!?``). This maximises input + length for better prosody while keeping playback gapless. :param text_generator: Generator that yields text tokens :param chunk_on: Characters used to detect chunk 0 boundaries only (default: ``",.!?"``) @@ -651,17 +651,31 @@ def playback_worker(): buffer += token.content if chunk_index == 0: - # --- Chunk 0: fast start – fire at first boundary (incl. commas) --- + # --- Chunk 0: fast start – fire at earliest acceptable boundary --- if any(c in buffer for c in chunk_on): - first_idx = find_sentence_boundary(buffer, chunk_on, first_only=True) - if first_idx >= 0: - to_speak = buffer[:first_idx + 1].strip() - delimiter = buffer[first_idx] + search_start = 0 + while search_start < len(buffer): + rel_idx = find_sentence_boundary( + buffer[search_start:], chunk_on, first_only=True, + ) + if rel_idx < 0: + break + idx = search_start + rel_idx + candidate = buffer[:idx + 1].strip() + delimiter = buffer[idx] effective_min = 5 if delimiter in '.!?' else min_chunk_size - if len(to_speak) >= effective_min: - buffer = buffer[first_idx + 1:] - synthesis_queue.put(to_speak) + if len(candidate) >= effective_min: + buffer = buffer[idx + 1:] + synthesis_queue.put(candidate) chunk_index += 1 + # Remainder may already contain sentence- + # ending punctuation; seed pending_boundary + # so chunk 1+ doesn't wait for a future token. + pending_boundary = self._find_sentence_boundary( + buffer, ".!?", + ) + break + search_start = idx + 1 else: # --- Chunk 1+: quality chunks – sentence boundaries only --- # Only rescan the buffer when the new token contains From afeb431e8cc0f5fef40f8f61d1fbd02e903fcfea Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Apr 2026 03:04:08 +0000 Subject: [PATCH 07/11] Add pytest test file for chunking boundary heuristics Addresses Copilot feedback that doctests won't run in CI without --doctest-modules. Creates tests/tts/test_chunking.py with 11 pytest cases covering first_only, last boundary, decimals, initials, abbreviations, weak commas, and edge cases (empty string, no boundary). https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- tests/tts/__init__.py | 0 tests/tts/test_chunking.py | 59 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 tests/tts/__init__.py create mode 100644 tests/tts/test_chunking.py diff --git a/tests/tts/__init__.py b/tests/tts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/tts/test_chunking.py b/tests/tts/test_chunking.py new file mode 100644 index 0000000..a9a0718 --- /dev/null +++ b/tests/tts/test_chunking.py @@ -0,0 +1,59 @@ +"""Tests for TTS chunk-boundary heuristics.""" + +import importlib +import sys +from pathlib import Path + +# Import the chunking module directly to avoid pulling in google.cloud +# via myai.tts.__init__.py (which imports TextToSpeech). +_chunking_path = Path(__file__).resolve().parents[2] / "src" / "myai" / "tts" / "chunking.py" +_spec = importlib.util.spec_from_file_location("myai.tts.chunking", _chunking_path) +_mod = importlib.util.module_from_spec(_spec) +sys.modules.setdefault("myai.tts.chunking", _mod) +_spec.loader.exec_module(_mod) +find_sentence_boundary = _mod.find_sentence_boundary + + +class TestFindSentenceBoundary: + """Regression tests for find_sentence_boundary (last vs first_only).""" + + def test_last_boundary_default(self): + text = "Hello world. Another sentence!" + assert find_sentence_boundary(text) == 29 + + def test_first_only_boundary(self): + text = "Hello world. Another sentence!" + assert find_sentence_boundary(text, first_only=True) == 11 + + def test_decimal_not_split(self): + assert find_sentence_boundary("The value is 3.14 and rising.") == 28 + + def test_decimal_only_no_boundary(self): + assert find_sentence_boundary("Version 2.0") == -1 + + def test_initials_stay_connected(self): + assert find_sentence_boundary("Meet J. T. Smith.") == 16 + + def test_initials_first_only(self): + assert find_sentence_boundary("Meet J. T. Smith.", first_only=True) == 16 + + def test_weak_commas_skipped(self): + assert find_sentence_boundary("Count 1, 2, 3", chunk_chars=",") == -1 + + def test_no_boundary_returns_negative(self): + assert find_sentence_boundary("no punctuation here") == -1 + + def test_empty_string(self): + assert find_sentence_boundary("") == -1 + + def test_first_only_with_commas(self): + text = "Alpha, beta gamma delta epsilon zeta, final stop." + # First strong comma is at index 36 ("zeta,") + idx = find_sentence_boundary(text, chunk_chars=",.!?", first_only=True) + assert idx == 36 + + def test_last_with_commas(self): + text = "Alpha, beta gamma delta epsilon zeta, final stop." + # Last boundary is the period at index 48 + idx = find_sentence_boundary(text, chunk_chars=",.!?") + assert idx == 48 From 6894cfea0a452b5629438a7e5908abb353e7956f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Apr 2026 03:08:59 +0000 Subject: [PATCH 08/11] Fix race between playback_active and play_start Set playback_active=True and play_start together in a single lock acquisition after .play() returns, so _remaining_playback_time() never sees playback_active=True with a stale play_start of 0.0. https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index d711544..4666c8f 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -612,15 +612,16 @@ def playback_worker(): # play_duration field now uses the real value. state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) state["play_duration"] = actual_dur - state["playback_active"] = True pygame.mixer.music.load(audio_file) pygame.mixer.music.play() - # Set play_start after .play() so elapsed time reflects - # actual playback, not load latency. + # Set play_start and playback_active together after + # .play() so _remaining_playback_time() never sees + # playback_active=True with a stale play_start. with state_lock: state["play_start"] = time.monotonic() + state["playback_active"] = True while pygame.mixer.music.get_busy(): time.sleep(0.05) From 8b89b98972825e1ce8060a98df2dec205e6521b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Apr 2026 12:26:47 +0000 Subject: [PATCH 09/11] Fix queued_total undercount during load/play window Defer the queued_total decrement until the same lock block where playback_active and play_start are set, so _remaining_playback_time() never sees a gap where the loading chunk is unaccounted for. https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index 4666c8f..c112cbd 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -607,19 +607,15 @@ def playback_worker(): except Exception: actual_dur = estimated_dur - with state_lock: - # Correct queued_total: remove the estimate, the - # play_duration field now uses the real value. - state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) - state["play_duration"] = actual_dur - pygame.mixer.music.load(audio_file) pygame.mixer.music.play() - # Set play_start and playback_active together after - # .play() so _remaining_playback_time() never sees - # playback_active=True with a stale play_start. + # Atomically move the chunk from queued_total into + # active playback tracking so _remaining_playback_time() + # never undercounts during the load/play window. with state_lock: + state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) + state["play_duration"] = actual_dur state["play_start"] = time.monotonic() state["playback_active"] = True From 5e0eaa12613dab5e3f83ddd1b98172d5db79bb4b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 4 Apr 2026 12:57:27 +0000 Subject: [PATCH 10/11] Fix queued_total leak on playback failure Decrement queued_total immediately on dequeue and track the chunk via play_duration. _remaining_playback_time() now accounts for the chunk during loading (play_duration when !playback_active) and during playback (play_duration - elapsed). play_duration is reset in finally so it never lingers after a chunk finishes or fails. https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index c112cbd..fc01b3a 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -520,11 +520,15 @@ def speak_streaming_async(self, text_generator, chunk_on: str = ",.!?", print_te def _remaining_playback_time() -> float: """Total seconds of audio still to play (current + queued).""" with state_lock: - remaining = 0.0 + remaining = state["queued_total"] if state["playback_active"]: elapsed = time.monotonic() - state["play_start"] - remaining = max(0.0, state["play_duration"] - elapsed) - remaining += state["queued_total"] + remaining += max(0.0, state["play_duration"] - elapsed) + else: + # Chunk being loaded but not yet playing; its + # duration was moved from queued_total into + # play_duration on dequeue. + remaining += state["play_duration"] return remaining def _update_avg(duration: float): @@ -589,6 +593,14 @@ def playback_worker(): break audio_file, estimated_dur = item + + # Immediately move the chunk out of queued_total and + # into play_duration so _remaining_playback_time() + # always accounts for it (either as queued or active). + with state_lock: + state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) + state["play_duration"] = estimated_dur + try: # Detect stalls (playback queue went empty) now = time.monotonic() @@ -610,11 +622,7 @@ def playback_worker(): pygame.mixer.music.load(audio_file) pygame.mixer.music.play() - # Atomically move the chunk from queued_total into - # active playback tracking so _remaining_playback_time() - # never undercounts during the load/play window. with state_lock: - state["queued_total"] = max(0.0, state["queued_total"] - estimated_dur) state["play_duration"] = actual_dur state["play_start"] = time.monotonic() state["playback_active"] = True @@ -628,6 +636,7 @@ def playback_worker(): finally: with state_lock: state["playback_active"] = False + state["play_duration"] = 0.0 try: os.remove(audio_file) except Exception: From f275fe643b99aa273cb1bbe3d0e34739d64bf7c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Apr 2026 09:01:05 +0000 Subject: [PATCH 11/11] Fix unreachable short-chunk suppression guard Move the edge-case check (hold short chunks when remaining playback time is ample) before the `remaining <= threshold` gate so it can actually trigger. https://claude.ai/code/session_01E7isyc2DwdLiHppMT8gvA1 --- src/myai/tts/text_to_speech.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/myai/tts/text_to_speech.py b/src/myai/tts/text_to_speech.py index fc01b3a..09c0a3d 100644 --- a/src/myai/tts/text_to_speech.py +++ b/src/myai/tts/text_to_speech.py @@ -705,14 +705,14 @@ def playback_worker(): remaining = _remaining_playback_time() threshold = current_avg_synth + 0.2 # 200 ms margin - if remaining <= threshold: - to_speak = buffer[:pending_boundary + 1].strip() + to_speak = buffer[:pending_boundary + 1].strip() - # Edge case: chunk 0 still playing with time to spare - # and buffer text is very short – hold for more tokens - if remaining > threshold * 2 and len(to_speak) < min_chunk_size: - continue + # Edge case: chunk 0 still playing with time to spare + # and buffer text is very short – hold for more tokens + if remaining > threshold * 2 and len(to_speak) < min_chunk_size: + continue + if remaining <= threshold: buffer = buffer[pending_boundary + 1:] synthesis_queue.put(to_speak) chunk_index += 1