feat: update AI voice settings for improved responsiveness and pacing

deployment/aimaq.env.production

@@ -39,10 +39,10 @@ AI_VOICE_MEDIA_IDLE_TIMEOUT_SECONDS=15
 AI_VOICE_GREETING_BARGE_IN_TRIGGER_MS=350
 AI_VOICE_V2_ENABLED=1
 AI_VOICE_V2_QUEUE_CODES=ivr_aimaq_ai_ru
-AI_VOICE_V2_ACK_MODE=disabled
+AI_VOICE_V2_ACK_MODE=immediate_short
 AI_VOICE_V2_DUPLEX_ENABLED=1
 AI_VOICE_V2_PARTIAL_ASR=1
-AI_VOICE_V2_PREBAKED_ACK_ENABLED=0
+AI_VOICE_V2_PREBAKED_ACK_ENABLED=1
 AI_VOICE_V2_EMOTIVE_ACK_ENABLED=0
 AI_VOICE_V2_EMOTIVE_ACK_RU_ONLY=1
 AI_VOICE_V2_STREAMING_TTS=1
@@ -60,9 +60,6 @@ AI_VOICE_VAD_MIN_SPEECH_MS=300
 AI_VOICE_VAD_TRAILING_SILENCE_MS=500
 
 AI_VOICE_NO_SPEECH_REPROMPT_ENABLED=1
-AI_VOICE_NO_SPEECH_REPROMPT_SECONDS=7
-AI_VOICE_NO_SPEECH_REPROMPT_LOW_SIGNAL_TURNS=2
-AI_VOICE_NO_SPEECH_REPROMPT_MAX_ATTEMPTS=2
 
 AI_VOICE_ASR_PROVIDER=elevenlabs
 AI_VOICE_ASR_MODEL=gpt-4o-transcribe
@@ -85,13 +82,12 @@ AI_VOICE_TTS_CACHE_ENABLED=1
 AI_VOICE_TTS_CACHE_DIR=/app/.data_local/ai_voice_tts_cache
 AI_VOICE_TTS_ELEVENLABS_API_KEY=sk_eb9bd36750e2a7fb378259b7144e978a044cfb412e6a2ac9
 AI_VOICE_TTS_ELEVENLABS_API_BASE=https://api.elevenlabs.io
-AI_VOICE_TTS_ELEVENLABS_MODEL_ID=eleven_turbo_v2_5
+AI_VOICE_TTS_ELEVENLABS_MODEL_ID=eleven_multilingual_v2
 AI_VOICE_TTS_ELEVENLABS_RU_VOICE_ID=4O1sYUnmtThcBoSBrri7
 AI_VOICE_TTS_ELEVENLABS_RU_LANGUAGE_CODE=ru
 AI_VOICE_TTS_ELEVENLABS_KK_VOICE_ID=4O1sYUnmtThcBoSBrri7
 AI_VOICE_TTS_ELEVENLABS_KK_LANGUAGE_CODE=kk
 AI_VOICE_TTS_ELEVENLABS_OUTPUT_FORMAT=pcm_16000
-AI_VOICE_TTS_STREAM_PREBUFFER_MS=200
 AI_VOICE_TTS_YANDEX_API_KEY=AQWJUMiUaXmbegxN4kgvM2XIlNqAPoBR5Wtq-40
 AI_VOICE_TTS_YANDEX_FOLDER_ID=ao7hkif5pvc7vfnmbl0d
 

services/ai_voice_runtime_service/media_runtime.py

@@ -23,7 +23,6 @@ from services.ai_voice_runtime_service.audiosocket import (
     normalize_media_uuid,
     pcm16le_to_wav_bytes,
     read_packet,
-    resample_pcm16le,
     resample_pcm16le_stateful,
 )
 from services.ai_voice_runtime_service.providers.asr import (
@@ -126,7 +125,26 @@ class MediaActor:
     listening_since_monotonic: float = 0.0
     reprompt_attempts: int = 0
     consecutive_low_signal_turns: int = 0
-    reprompt_watchdog_task: asyncio.Task | None = None
+
+
+@dataclass(slots=True)
+class _FramePacer:
+    """Wall-clock-anchored pacing for one playback: paces frames to the first
+    frame's send time instead of naive per-frame relative sleeps, whose drift
+    accumulates under event-loop scheduling pressure (audible as stutter)."""
+
+    frame_seconds: float
+    start_monotonic: float | None = None
+    frames_written: int = 0
+
+    async def wait_for_next_frame(self) -> None:
+        if self.start_monotonic is None:
+            self.start_monotonic = time.monotonic()
+        self.frames_written += 1
+        target_monotonic = self.start_monotonic + self.frames_written * self.frame_seconds
+        sleep_seconds = target_monotonic - time.monotonic()
+        if sleep_seconds > 0:
+            await asyncio.sleep(sleep_seconds)
 
 
 class AudioSocketMediaRuntime:
@@ -203,9 +221,7 @@ class AudioSocketMediaRuntime:
         # Small startup cushion for streamed TTS playback: absorb ElevenLabs
         # network delivery jitter before we start pacing frames out to the
         # caller, so a brief mid-download stall doesn't read as a dead-air gap.
-        self._tts_stream_prebuffer_ms = max(
-            int(os.getenv("AI_VOICE_TTS_STREAM_PREBUFFER_MS", "200") or "200"), 0
-        )
+        self._tts_stream_prebuffer_ms = 200
         self._streaming_asr_partial_poll_enabled = (
             str(os.getenv("AI_VOICE_V2_STREAMING_ASR_PARTIAL_POLL_ENABLED", "0")).strip().lower()
             in {"1", "true", "yes", "on"}
@@ -230,16 +246,9 @@ class AudioSocketMediaRuntime:
             str(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_ENABLED", "1")).strip().lower()
             in {"1", "true", "yes", "on"}
         )
-        self._no_speech_reprompt_seconds = max(
-            float(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_SECONDS", "7") or "7"), 2.0
-        )
-        self._no_speech_reprompt_low_signal_turns = max(
-            int(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_LOW_SIGNAL_TURNS", "2") or "2"), 1
-        )
-        self._no_speech_reprompt_max_attempts = max(
-            int(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_MAX_ATTEMPTS", "2") or "2"), 1
-        )
-        self._no_speech_reprompt_poll_seconds = 1.0
+        self._no_speech_reprompt_seconds = 7.0
+        self._no_speech_reprompt_low_signal_turns = 2
+        self._no_speech_reprompt_max_attempts = 2
         raw_early_plan_intents = str(
             os.getenv(
                 "AI_VOICE_V2_EARLY_PLAN_INTENTS",
@@ -1139,7 +1148,6 @@ class AudioSocketMediaRuntime:
         actor.barge_in_speech_ms = 0
         await self._set_actor_state(actor, "speaking")
         actor.current_reply_phase = reply_phase
-        interrupted = False
         await self._record_reply_status(
             actor,
             text=text,
@@ -1152,12 +1160,8 @@ class AudioSocketMediaRuntime:
             await self._record_latency_metric(actor, "speech_start_to_ack_start", actor.speech_started_monotonic)
         elif reply_phase == "main":
             await self._record_latency_metric(actor, "speech_end_to_main_reply_start", actor.speech_ended_monotonic)
-        for frame in chunk_audio(pcm_8k, frame_bytes=actor.frame_bytes):
-            if actor.closed or actor.playback_interrupt.is_set():
-                interrupted = True
-                break
-            await self._write_audio_packet(actor, frame)
-            await asyncio.sleep(actor.frame_ms / 1000.0)
+        pacer = _FramePacer(frame_seconds=actor.frame_ms / 1000.0)
+        interrupted = await self._write_paced_pcm_frames(actor, pcm_8k, pacer)
         if actor.playback_interrupt.is_set():
             interrupted = True
         actor.playback_interrupt.clear()
@@ -1352,7 +1356,6 @@ class AudioSocketMediaRuntime:
             await asyncio.to_thread(self._mark_media_connected, registration.voice_session_id, media_uuid)
             actor.keepalive_task = asyncio.create_task(self._keepalive_loop(actor))
             actor.worker_task = asyncio.create_task(self._worker(actor))
-            actor.reprompt_watchdog_task = asyncio.create_task(self._no_speech_reprompt_loop(actor))
 
             while not actor.closed:
                 packet_type, payload = await read_packet(reader, timeout=self._idle_timeout_seconds)
@@ -1873,7 +1876,6 @@ class AudioSocketMediaRuntime:
         actor.current_reply_phase = reply_phase
         synth_started_at = time.monotonic()
         first_chunk_logged = False
-        first_frame_sent = False
         total_audio_bytes = 0
         interrupted = False
         pending_pcm = bytearray()
@@ -1887,22 +1889,18 @@ class AudioSocketMediaRuntime:
             status="started",
         )
 
+        pacer = _FramePacer(frame_seconds=actor.frame_ms / 1000.0)
+
+        def _log_first_frame(frame: bytes) -> None:
+            logger.info(
+                "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s",
+                actor.registration.voice_session_id,
+                is_greeting,
+                len(frame),
+            )
+
         async def _write_pcm_frames(pcm_bytes: bytes) -> bool:
-            nonlocal first_frame_sent
-            for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes):
-                if actor.closed or actor.playback_interrupt.is_set():
-                    return True
-                await self._write_audio_packet(actor, frame)
-                if not first_frame_sent:
-                    first_frame_sent = True
-                    logger.info(
-                        "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s",
-                        actor.registration.voice_session_id,
-                        is_greeting,
-                        len(frame),
-                    )
-                await asyncio.sleep(actor.frame_ms / 1000.0)
-            return False
+            return await self._write_paced_pcm_frames(actor, pcm_bytes, pacer, on_first_frame=_log_first_frame)
 
         async for synthesis in self._stream_tts_chunks(actor, text, style_hints=style_hints):
             if not synthesis.audio_bytes:
@@ -1932,8 +1930,15 @@ class AudioSocketMediaRuntime:
                 output_rate_hz=8000,
                 state=resample_state,
             )
+            if prebuffered:
+                # Past the initial cushion: feed each resampled chunk straight
+                # through instead of round-tripping it through pending_pcm.
+                interrupted = await _write_pcm_frames(pcm_8k)
+                if interrupted:
+                    break
+                continue
             pending_pcm.extend(pcm_8k)
-            if not prebuffered and len(pending_pcm) < self._tts_stream_prebuffer_bytes:
+            if len(pending_pcm) < self._tts_stream_prebuffer_bytes:
                 continue
             prebuffered = True
             interrupted = await _write_pcm_frames(bytes(pending_pcm))
@@ -1985,12 +1990,34 @@ class AudioSocketMediaRuntime:
             await actor.writer.drain()
             actor.last_outbound_audio_monotonic = time.monotonic()
 
+    async def _write_paced_pcm_frames(
+        self,
+        actor: MediaActor,
+        pcm_bytes: bytes,
+        pacer: _FramePacer,
+        *,
+        on_first_frame: Callable[[bytes], None] | None = None,
+    ) -> bool:
+        """Write pcm_bytes out as frame_bytes-sized packets, paced by `pacer`.
+
+        Returns True if playback was interrupted (barge-in or actor closed).
+        """
+        for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes):
+            if actor.closed or actor.playback_interrupt.is_set():
+                return True
+            await self._write_audio_packet(actor, frame)
+            if pacer.frames_written == 0 and on_first_frame is not None:
+                on_first_frame(frame)
+            await pacer.wait_for_next_frame()
+        return False
+
     async def _keepalive_loop(self, actor: MediaActor) -> None:
         silence_frame = b"\x00" * actor.frame_bytes
         while not actor.closed:
             await asyncio.sleep(0.25)
             if actor.closed:
                 break
+            await self._maybe_reprompt_on_silence(actor)
             if (time.monotonic() - actor.last_outbound_audio_monotonic) < self._outbound_keepalive_interval_seconds:
                 continue
             if not actor.keepalive_loop_logged:
@@ -2021,18 +2048,14 @@ class AudioSocketMediaRuntime:
         if not actor.closed:
             await self._set_actor_state(actor, "listening")
 
-    async def _no_speech_reprompt_loop(self, actor: MediaActor) -> None:
+    async def _maybe_reprompt_on_silence(self, actor: MediaActor) -> None:
         if not self._no_speech_reprompt_enabled:
             return
-        while not actor.closed:
-            await asyncio.sleep(self._no_speech_reprompt_poll_seconds)
-            if actor.closed:
-                break
-            if actor.state != "listening" or actor.listening_since_monotonic <= 0:
-                continue
-            if time.monotonic() - actor.listening_since_monotonic < self._no_speech_reprompt_seconds:
-                continue
-            await self._maybe_play_no_speech_reprompt(actor, trigger="silence_timeout")
+        if actor.state != "listening" or actor.listening_since_monotonic <= 0:
+            return
+        if time.monotonic() - actor.listening_since_monotonic < self._no_speech_reprompt_seconds:
+            return
+        await self._maybe_play_no_speech_reprompt(actor, trigger="silence_timeout")
 
     async def _set_actor_state(
         self,
@@ -2086,10 +2109,6 @@ class AudioSocketMediaRuntime:
             actor.keepalive_task.cancel()
             with contextlib.suppress(asyncio.CancelledError, Exception):
                 await actor.keepalive_task
-        if actor.reprompt_watchdog_task is not None:
-            actor.reprompt_watchdog_task.cancel()
-            with contextlib.suppress(asyncio.CancelledError, Exception):
-                await actor.reprompt_watchdog_task
         if actor.handoff_task is not None:
             actor.handoff_task.cancel()
             with contextlib.suppress(asyncio.CancelledError, Exception):

tests/test_ai_voice_media_runtime.py

@@ -2861,7 +2861,6 @@ def test_media_runtime_no_speech_watchdog_reprompts_on_silence_timeout():
         handle_media_error=lambda session_id, message, metadata: None,
     )
     runtime._no_speech_reprompt_seconds = 0.05
-    runtime._no_speech_reprompt_poll_seconds = 0.02
 
     spoken: list[str] = []
 
@@ -2887,16 +2886,21 @@ def test_media_runtime_no_speech_watchdog_reprompts_on_silence_timeout():
             frame_ms=20,
             frame_bytes=320,
         )
+        # The no-speech watchdog check now rides the existing keepalive loop
+        # instead of its own dedicated task; keep last_outbound_audio_monotonic
+        # fresh so the loop's silence-keepalive-audio branch (which needs a
+        # real writer) doesn't fire during this test.
+        actor.last_outbound_audio_monotonic = time.monotonic()
         await runtime._set_actor_state(actor, "listening")
-        watchdog_task = asyncio.create_task(runtime._no_speech_reprompt_loop(actor))
-        for _ in range(50):
+        keepalive_task = asyncio.create_task(runtime._keepalive_loop(actor))
+        for _ in range(20):
             if spoken:
                 break
-            await asyncio.sleep(0.02)
+            await asyncio.sleep(0.05)
         actor.closed = True
-        watchdog_task.cancel()
+        keepalive_task.cancel()
         with contextlib.suppress(asyncio.CancelledError, Exception):
-            await watchdog_task
+            await keepalive_task
         assert spoken, "watchdog should reprompt after prolonged silence in listening state"
 
     asyncio.run(_scenario())