feat: update AI voice settings for improved responsiveness and pacing

services/ai_voice_runtime_service/media_runtime.py

@@ -23,7 +23,6 @@ from services.ai_voice_runtime_service.audiosocket import (
     normalize_media_uuid,
     pcm16le_to_wav_bytes,
     read_packet,
-    resample_pcm16le,
     resample_pcm16le_stateful,
 )
 from services.ai_voice_runtime_service.providers.asr import (
@@ -126,7 +125,26 @@ class MediaActor:
     listening_since_monotonic: float = 0.0
     reprompt_attempts: int = 0
     consecutive_low_signal_turns: int = 0
-    reprompt_watchdog_task: asyncio.Task | None = None
+
+
+@dataclass(slots=True)
+class _FramePacer:
+    """Wall-clock-anchored pacing for one playback: paces frames to the first
+    frame's send time instead of naive per-frame relative sleeps, whose drift
+    accumulates under event-loop scheduling pressure (audible as stutter)."""
+
+    frame_seconds: float
+    start_monotonic: float | None = None
+    frames_written: int = 0
+
+    async def wait_for_next_frame(self) -> None:
+        if self.start_monotonic is None:
+            self.start_monotonic = time.monotonic()
+        self.frames_written += 1
+        target_monotonic = self.start_monotonic + self.frames_written * self.frame_seconds
+        sleep_seconds = target_monotonic - time.monotonic()
+        if sleep_seconds > 0:
+            await asyncio.sleep(sleep_seconds)
 
 
 class AudioSocketMediaRuntime:
@@ -203,9 +221,7 @@ class AudioSocketMediaRuntime:
         # Small startup cushion for streamed TTS playback: absorb ElevenLabs
         # network delivery jitter before we start pacing frames out to the
         # caller, so a brief mid-download stall doesn't read as a dead-air gap.
-        self._tts_stream_prebuffer_ms = max(
-            int(os.getenv("AI_VOICE_TTS_STREAM_PREBUFFER_MS", "200") or "200"), 0
-        )
+        self._tts_stream_prebuffer_ms = 200
         self._streaming_asr_partial_poll_enabled = (
             str(os.getenv("AI_VOICE_V2_STREAMING_ASR_PARTIAL_POLL_ENABLED", "0")).strip().lower()
             in {"1", "true", "yes", "on"}
@@ -230,16 +246,9 @@ class AudioSocketMediaRuntime:
             str(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_ENABLED", "1")).strip().lower()
             in {"1", "true", "yes", "on"}
         )
-        self._no_speech_reprompt_seconds = max(
-            float(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_SECONDS", "7") or "7"), 2.0
-        )
-        self._no_speech_reprompt_low_signal_turns = max(
-            int(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_LOW_SIGNAL_TURNS", "2") or "2"), 1
-        )
-        self._no_speech_reprompt_max_attempts = max(
-            int(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_MAX_ATTEMPTS", "2") or "2"), 1
-        )
-        self._no_speech_reprompt_poll_seconds = 1.0
+        self._no_speech_reprompt_seconds = 7.0
+        self._no_speech_reprompt_low_signal_turns = 2
+        self._no_speech_reprompt_max_attempts = 2
         raw_early_plan_intents = str(
             os.getenv(
                 "AI_VOICE_V2_EARLY_PLAN_INTENTS",
@@ -1139,7 +1148,6 @@ class AudioSocketMediaRuntime:
         actor.barge_in_speech_ms = 0
         await self._set_actor_state(actor, "speaking")
         actor.current_reply_phase = reply_phase
-        interrupted = False
         await self._record_reply_status(
             actor,
             text=text,
@@ -1152,12 +1160,8 @@ class AudioSocketMediaRuntime:
             await self._record_latency_metric(actor, "speech_start_to_ack_start", actor.speech_started_monotonic)
         elif reply_phase == "main":
             await self._record_latency_metric(actor, "speech_end_to_main_reply_start", actor.speech_ended_monotonic)
-        for frame in chunk_audio(pcm_8k, frame_bytes=actor.frame_bytes):
-            if actor.closed or actor.playback_interrupt.is_set():
-                interrupted = True
-                break
-            await self._write_audio_packet(actor, frame)
-            await asyncio.sleep(actor.frame_ms / 1000.0)
+        pacer = _FramePacer(frame_seconds=actor.frame_ms / 1000.0)
+        interrupted = await self._write_paced_pcm_frames(actor, pcm_8k, pacer)
         if actor.playback_interrupt.is_set():
             interrupted = True
         actor.playback_interrupt.clear()
@@ -1352,7 +1356,6 @@ class AudioSocketMediaRuntime:
             await asyncio.to_thread(self._mark_media_connected, registration.voice_session_id, media_uuid)
             actor.keepalive_task = asyncio.create_task(self._keepalive_loop(actor))
             actor.worker_task = asyncio.create_task(self._worker(actor))
-            actor.reprompt_watchdog_task = asyncio.create_task(self._no_speech_reprompt_loop(actor))
 
             while not actor.closed:
                 packet_type, payload = await read_packet(reader, timeout=self._idle_timeout_seconds)
@@ -1873,7 +1876,6 @@ class AudioSocketMediaRuntime:
         actor.current_reply_phase = reply_phase
         synth_started_at = time.monotonic()
         first_chunk_logged = False
-        first_frame_sent = False
         total_audio_bytes = 0
         interrupted = False
         pending_pcm = bytearray()
@@ -1887,22 +1889,18 @@ class AudioSocketMediaRuntime:
             status="started",
         )
 
+        pacer = _FramePacer(frame_seconds=actor.frame_ms / 1000.0)
+
+        def _log_first_frame(frame: bytes) -> None:
+            logger.info(
+                "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s",
+                actor.registration.voice_session_id,
+                is_greeting,
+                len(frame),
+            )
+
         async def _write_pcm_frames(pcm_bytes: bytes) -> bool:
-            nonlocal first_frame_sent
-            for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes):
-                if actor.closed or actor.playback_interrupt.is_set():
-                    return True
-                await self._write_audio_packet(actor, frame)
-                if not first_frame_sent:
-                    first_frame_sent = True
-                    logger.info(
-                        "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s",
-                        actor.registration.voice_session_id,
-                        is_greeting,
-                        len(frame),
-                    )
-                await asyncio.sleep(actor.frame_ms / 1000.0)
-            return False
+            return await self._write_paced_pcm_frames(actor, pcm_bytes, pacer, on_first_frame=_log_first_frame)
 
         async for synthesis in self._stream_tts_chunks(actor, text, style_hints=style_hints):
             if not synthesis.audio_bytes:
@@ -1932,8 +1930,15 @@ class AudioSocketMediaRuntime:
                 output_rate_hz=8000,
                 state=resample_state,
             )
+            if prebuffered:
+                # Past the initial cushion: feed each resampled chunk straight
+                # through instead of round-tripping it through pending_pcm.
+                interrupted = await _write_pcm_frames(pcm_8k)
+                if interrupted:
+                    break
+                continue
             pending_pcm.extend(pcm_8k)
-            if not prebuffered and len(pending_pcm) < self._tts_stream_prebuffer_bytes:
+            if len(pending_pcm) < self._tts_stream_prebuffer_bytes:
                 continue
             prebuffered = True
             interrupted = await _write_pcm_frames(bytes(pending_pcm))
@@ -1985,12 +1990,34 @@ class AudioSocketMediaRuntime:
             await actor.writer.drain()
             actor.last_outbound_audio_monotonic = time.monotonic()
 
+    async def _write_paced_pcm_frames(
+        self,
+        actor: MediaActor,
+        pcm_bytes: bytes,
+        pacer: _FramePacer,
+        *,
+        on_first_frame: Callable[[bytes], None] | None = None,
+    ) -> bool:
+        """Write pcm_bytes out as frame_bytes-sized packets, paced by `pacer`.
+
+        Returns True if playback was interrupted (barge-in or actor closed).
+        """
+        for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes):
+            if actor.closed or actor.playback_interrupt.is_set():
+                return True
+            await self._write_audio_packet(actor, frame)
+            if pacer.frames_written == 0 and on_first_frame is not None:
+                on_first_frame(frame)
+            await pacer.wait_for_next_frame()
+        return False
+
     async def _keepalive_loop(self, actor: MediaActor) -> None:
         silence_frame = b"\x00" * actor.frame_bytes
         while not actor.closed:
             await asyncio.sleep(0.25)
             if actor.closed:
                 break
+            await self._maybe_reprompt_on_silence(actor)
             if (time.monotonic() - actor.last_outbound_audio_monotonic) < self._outbound_keepalive_interval_seconds:
                 continue
             if not actor.keepalive_loop_logged:
@@ -2021,18 +2048,14 @@ class AudioSocketMediaRuntime:
         if not actor.closed:
             await self._set_actor_state(actor, "listening")
 
-    async def _no_speech_reprompt_loop(self, actor: MediaActor) -> None:
+    async def _maybe_reprompt_on_silence(self, actor: MediaActor) -> None:
         if not self._no_speech_reprompt_enabled:
             return
-        while not actor.closed:
-            await asyncio.sleep(self._no_speech_reprompt_poll_seconds)
-            if actor.closed:
-                break
-            if actor.state != "listening" or actor.listening_since_monotonic <= 0:
-                continue
-            if time.monotonic() - actor.listening_since_monotonic < self._no_speech_reprompt_seconds:
-                continue
-            await self._maybe_play_no_speech_reprompt(actor, trigger="silence_timeout")
+        if actor.state != "listening" or actor.listening_since_monotonic <= 0:
+            return
+        if time.monotonic() - actor.listening_since_monotonic < self._no_speech_reprompt_seconds:
+            return
+        await self._maybe_play_no_speech_reprompt(actor, trigger="silence_timeout")
 
     async def _set_actor_state(
         self,
@@ -2086,10 +2109,6 @@ class AudioSocketMediaRuntime:
             actor.keepalive_task.cancel()
             with contextlib.suppress(asyncio.CancelledError, Exception):
                 await actor.keepalive_task
-        if actor.reprompt_watchdog_task is not None:
-            actor.reprompt_watchdog_task.cancel()
-            with contextlib.suppress(asyncio.CancelledError, Exception):
-                await actor.reprompt_watchdog_task
         if actor.handoff_task is not None:
             actor.handoff_task.cancel()
             with contextlib.suppress(asyncio.CancelledError, Exception):