feat: update ElevenLabs TTS model ID and add prebuffering for improved audio streaming

services/ai_voice_runtime_service/media_runtime.py

@@ -24,6 +24,7 @@ from services.ai_voice_runtime_service.audiosocket import (
     pcm16le_to_wav_bytes,
     read_packet,
     resample_pcm16le,
+    resample_pcm16le_stateful,
 )
 from services.ai_voice_runtime_service.providers.asr import (
     ASRTranscription,
@@ -199,6 +200,12 @@ class AudioSocketMediaRuntime:
         self._v2_ack_post_gap_seconds = 0.10
         self._v1_ack_wait_seconds = 0.6
         self._partial_poll_interval_seconds = 0.20
+        # Small startup cushion for streamed TTS playback: absorb ElevenLabs
+        # network delivery jitter before we start pacing frames out to the
+        # caller, so a brief mid-download stall doesn't read as a dead-air gap.
+        self._tts_stream_prebuffer_ms = max(
+            int(os.getenv("AI_VOICE_TTS_STREAM_PREBUFFER_MS", "200") or "200"), 0
+        )
         self._streaming_asr_partial_poll_enabled = (
             str(os.getenv("AI_VOICE_V2_STREAMING_ASR_PARTIAL_POLL_ENABLED", "0")).strip().lower()
             in {"1", "true", "yes", "on"}
@@ -471,6 +478,10 @@ class AudioSocketMediaRuntime:
     def _immediate_ack_min_bytes(self) -> int:
         return self._immediate_ack_min_ms * 16
 
+    @property
+    def _tts_stream_prebuffer_bytes(self) -> int:
+        return self._tts_stream_prebuffer_ms * 16
+
     @staticmethod
     def _reset_live_turn_state(actor: MediaActor) -> None:
         actor.utterance_generation += 1
@@ -1861,9 +1872,13 @@ class AudioSocketMediaRuntime:
         await self._set_actor_state(actor, "speaking")
         actor.current_reply_phase = reply_phase
         synth_started_at = time.monotonic()
+        first_chunk_logged = False
         first_frame_sent = False
         total_audio_bytes = 0
         interrupted = False
+        pending_pcm = bytearray()
+        resample_state: object | None = None
+        prebuffered = False
         await self._record_reply_status(
             actor,
             text=text,
@@ -1871,11 +1886,30 @@ class AudioSocketMediaRuntime:
             phase=reply_phase,
             status="started",
         )
+
+        async def _write_pcm_frames(pcm_bytes: bytes) -> bool:
+            nonlocal first_frame_sent
+            for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes):
+                if actor.closed or actor.playback_interrupt.is_set():
+                    return True
+                await self._write_audio_packet(actor, frame)
+                if not first_frame_sent:
+                    first_frame_sent = True
+                    logger.info(
+                        "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s",
+                        actor.registration.voice_session_id,
+                        is_greeting,
+                        len(frame),
+                    )
+                await asyncio.sleep(actor.frame_ms / 1000.0)
+            return False
+
         async for synthesis in self._stream_tts_chunks(actor, text, style_hints=style_hints):
             if not synthesis.audio_bytes:
                 continue
             total_audio_bytes += len(synthesis.audio_bytes)
-            if not first_frame_sent:
+            if not first_chunk_logged:
+                first_chunk_logged = True
                 logger.info(
                     "audiosocket.tts_ready session_id=%s greeting=%s synth_ms=%s audio_bytes=%s",
                     actor.registration.voice_session_id,
@@ -1888,29 +1922,29 @@ class AudioSocketMediaRuntime:
                     await self._record_latency_metric(actor, "speech_start_to_ack_start", actor.speech_started_monotonic)
                 elif reply_phase == "main":
                     await self._record_latency_metric(actor, "speech_end_to_main_reply_start", actor.speech_ended_monotonic)
-            pcm_8k = resample_pcm16le(
+            # Carry ratecv state across chunks: resampling each network chunk
+            # independently introduces an audible click/discontinuity at every
+            # chunk boundary, which gets much more noticeable with a fast model
+            # that streams many small chunks (e.g. eleven_flash_v2_5).
+            pcm_8k, resample_state = resample_pcm16le_stateful(
                 synthesis.audio_bytes,
                 input_rate_hz=synthesis.sample_rate_hz,
                 output_rate_hz=8000,
+                state=resample_state,
             )
-            for frame in chunk_audio(pcm_8k, frame_bytes=actor.frame_bytes):
-                if actor.closed or actor.playback_interrupt.is_set():
-                    interrupted = True
-                    break
-                await self._write_audio_packet(actor, frame)
-                if not first_frame_sent:
-                    first_frame_sent = True
-                    logger.info(
-                        "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s",
-                        actor.registration.voice_session_id,
-                        is_greeting,
-                        len(frame),
-                    )
-                await asyncio.sleep(actor.frame_ms / 1000.0)
-            if actor.closed or actor.playback_interrupt.is_set():
-                interrupted = True
+            pending_pcm.extend(pcm_8k)
+            if not prebuffered and len(pending_pcm) < self._tts_stream_prebuffer_bytes:
+                continue
+            prebuffered = True
+            interrupted = await _write_pcm_frames(bytes(pending_pcm))
+            pending_pcm.clear()
+            if interrupted:
                 break
 
+        if not interrupted and pending_pcm:
+            interrupted = await _write_pcm_frames(bytes(pending_pcm))
+            pending_pcm.clear()
+
         if total_audio_bytes <= 0:
             raise RuntimeError("TTS provider returned empty audio")