From 2ea6e6f4bbd44a558b32edbec8f1e9d4d0ba3434 Mon Sep 17 00:00:00 2001 From: didar Date: Tue, 25 Aug 2026 01:42:10 +0500 Subject: [PATCH] feat: update AI voice settings for improved responsiveness and pacing --- deployment/aimaq.env.production | 10 +- .../ai_voice_runtime_service/media_runtime.py | 127 ++++++++++-------- tests/test_ai_voice_media_runtime.py | 16 ++- 3 files changed, 86 insertions(+), 67 deletions(-) diff --git a/deployment/aimaq.env.production b/deployment/aimaq.env.production index db6c462..3a73199 100644 --- a/deployment/aimaq.env.production +++ b/deployment/aimaq.env.production @@ -39,10 +39,10 @@ AI_VOICE_MEDIA_IDLE_TIMEOUT_SECONDS=15 AI_VOICE_GREETING_BARGE_IN_TRIGGER_MS=350 AI_VOICE_V2_ENABLED=1 AI_VOICE_V2_QUEUE_CODES=ivr_aimaq_ai_ru -AI_VOICE_V2_ACK_MODE=disabled +AI_VOICE_V2_ACK_MODE=immediate_short AI_VOICE_V2_DUPLEX_ENABLED=1 AI_VOICE_V2_PARTIAL_ASR=1 -AI_VOICE_V2_PREBAKED_ACK_ENABLED=0 +AI_VOICE_V2_PREBAKED_ACK_ENABLED=1 AI_VOICE_V2_EMOTIVE_ACK_ENABLED=0 AI_VOICE_V2_EMOTIVE_ACK_RU_ONLY=1 AI_VOICE_V2_STREAMING_TTS=1 @@ -60,9 +60,6 @@ AI_VOICE_VAD_MIN_SPEECH_MS=300 AI_VOICE_VAD_TRAILING_SILENCE_MS=500 AI_VOICE_NO_SPEECH_REPROMPT_ENABLED=1 -AI_VOICE_NO_SPEECH_REPROMPT_SECONDS=7 -AI_VOICE_NO_SPEECH_REPROMPT_LOW_SIGNAL_TURNS=2 -AI_VOICE_NO_SPEECH_REPROMPT_MAX_ATTEMPTS=2 AI_VOICE_ASR_PROVIDER=elevenlabs AI_VOICE_ASR_MODEL=gpt-4o-transcribe @@ -85,13 +82,12 @@ AI_VOICE_TTS_CACHE_ENABLED=1 AI_VOICE_TTS_CACHE_DIR=/app/.data_local/ai_voice_tts_cache AI_VOICE_TTS_ELEVENLABS_API_KEY=sk_eb9bd36750e2a7fb378259b7144e978a044cfb412e6a2ac9 AI_VOICE_TTS_ELEVENLABS_API_BASE=https://api.elevenlabs.io -AI_VOICE_TTS_ELEVENLABS_MODEL_ID=eleven_turbo_v2_5 +AI_VOICE_TTS_ELEVENLABS_MODEL_ID=eleven_multilingual_v2 AI_VOICE_TTS_ELEVENLABS_RU_VOICE_ID=4O1sYUnmtThcBoSBrri7 AI_VOICE_TTS_ELEVENLABS_RU_LANGUAGE_CODE=ru AI_VOICE_TTS_ELEVENLABS_KK_VOICE_ID=4O1sYUnmtThcBoSBrri7 AI_VOICE_TTS_ELEVENLABS_KK_LANGUAGE_CODE=kk AI_VOICE_TTS_ELEVENLABS_OUTPUT_FORMAT=pcm_16000 -AI_VOICE_TTS_STREAM_PREBUFFER_MS=200 AI_VOICE_TTS_YANDEX_API_KEY=AQWJUMiUaXmbegxN4kgvM2XIlNqAPoBR5Wtq-40 AI_VOICE_TTS_YANDEX_FOLDER_ID=ao7hkif5pvc7vfnmbl0d diff --git a/services/ai_voice_runtime_service/media_runtime.py b/services/ai_voice_runtime_service/media_runtime.py index a1e1554..2bf5c16 100644 --- a/services/ai_voice_runtime_service/media_runtime.py +++ b/services/ai_voice_runtime_service/media_runtime.py @@ -23,7 +23,6 @@ from services.ai_voice_runtime_service.audiosocket import ( normalize_media_uuid, pcm16le_to_wav_bytes, read_packet, - resample_pcm16le, resample_pcm16le_stateful, ) from services.ai_voice_runtime_service.providers.asr import ( @@ -126,7 +125,26 @@ class MediaActor: listening_since_monotonic: float = 0.0 reprompt_attempts: int = 0 consecutive_low_signal_turns: int = 0 - reprompt_watchdog_task: asyncio.Task | None = None + + +@dataclass(slots=True) +class _FramePacer: + """Wall-clock-anchored pacing for one playback: paces frames to the first + frame's send time instead of naive per-frame relative sleeps, whose drift + accumulates under event-loop scheduling pressure (audible as stutter).""" + + frame_seconds: float + start_monotonic: float | None = None + frames_written: int = 0 + + async def wait_for_next_frame(self) -> None: + if self.start_monotonic is None: + self.start_monotonic = time.monotonic() + self.frames_written += 1 + target_monotonic = self.start_monotonic + self.frames_written * self.frame_seconds + sleep_seconds = target_monotonic - time.monotonic() + if sleep_seconds > 0: + await asyncio.sleep(sleep_seconds) class AudioSocketMediaRuntime: @@ -203,9 +221,7 @@ class AudioSocketMediaRuntime: # Small startup cushion for streamed TTS playback: absorb ElevenLabs # network delivery jitter before we start pacing frames out to the # caller, so a brief mid-download stall doesn't read as a dead-air gap. - self._tts_stream_prebuffer_ms = max( - int(os.getenv("AI_VOICE_TTS_STREAM_PREBUFFER_MS", "200") or "200"), 0 - ) + self._tts_stream_prebuffer_ms = 200 self._streaming_asr_partial_poll_enabled = ( str(os.getenv("AI_VOICE_V2_STREAMING_ASR_PARTIAL_POLL_ENABLED", "0")).strip().lower() in {"1", "true", "yes", "on"} @@ -230,16 +246,9 @@ class AudioSocketMediaRuntime: str(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_ENABLED", "1")).strip().lower() in {"1", "true", "yes", "on"} ) - self._no_speech_reprompt_seconds = max( - float(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_SECONDS", "7") or "7"), 2.0 - ) - self._no_speech_reprompt_low_signal_turns = max( - int(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_LOW_SIGNAL_TURNS", "2") or "2"), 1 - ) - self._no_speech_reprompt_max_attempts = max( - int(os.getenv("AI_VOICE_NO_SPEECH_REPROMPT_MAX_ATTEMPTS", "2") or "2"), 1 - ) - self._no_speech_reprompt_poll_seconds = 1.0 + self._no_speech_reprompt_seconds = 7.0 + self._no_speech_reprompt_low_signal_turns = 2 + self._no_speech_reprompt_max_attempts = 2 raw_early_plan_intents = str( os.getenv( "AI_VOICE_V2_EARLY_PLAN_INTENTS", @@ -1139,7 +1148,6 @@ class AudioSocketMediaRuntime: actor.barge_in_speech_ms = 0 await self._set_actor_state(actor, "speaking") actor.current_reply_phase = reply_phase - interrupted = False await self._record_reply_status( actor, text=text, @@ -1152,12 +1160,8 @@ class AudioSocketMediaRuntime: await self._record_latency_metric(actor, "speech_start_to_ack_start", actor.speech_started_monotonic) elif reply_phase == "main": await self._record_latency_metric(actor, "speech_end_to_main_reply_start", actor.speech_ended_monotonic) - for frame in chunk_audio(pcm_8k, frame_bytes=actor.frame_bytes): - if actor.closed or actor.playback_interrupt.is_set(): - interrupted = True - break - await self._write_audio_packet(actor, frame) - await asyncio.sleep(actor.frame_ms / 1000.0) + pacer = _FramePacer(frame_seconds=actor.frame_ms / 1000.0) + interrupted = await self._write_paced_pcm_frames(actor, pcm_8k, pacer) if actor.playback_interrupt.is_set(): interrupted = True actor.playback_interrupt.clear() @@ -1352,7 +1356,6 @@ class AudioSocketMediaRuntime: await asyncio.to_thread(self._mark_media_connected, registration.voice_session_id, media_uuid) actor.keepalive_task = asyncio.create_task(self._keepalive_loop(actor)) actor.worker_task = asyncio.create_task(self._worker(actor)) - actor.reprompt_watchdog_task = asyncio.create_task(self._no_speech_reprompt_loop(actor)) while not actor.closed: packet_type, payload = await read_packet(reader, timeout=self._idle_timeout_seconds) @@ -1873,7 +1876,6 @@ class AudioSocketMediaRuntime: actor.current_reply_phase = reply_phase synth_started_at = time.monotonic() first_chunk_logged = False - first_frame_sent = False total_audio_bytes = 0 interrupted = False pending_pcm = bytearray() @@ -1887,22 +1889,18 @@ class AudioSocketMediaRuntime: status="started", ) + pacer = _FramePacer(frame_seconds=actor.frame_ms / 1000.0) + + def _log_first_frame(frame: bytes) -> None: + logger.info( + "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s", + actor.registration.voice_session_id, + is_greeting, + len(frame), + ) + async def _write_pcm_frames(pcm_bytes: bytes) -> bool: - nonlocal first_frame_sent - for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes): - if actor.closed or actor.playback_interrupt.is_set(): - return True - await self._write_audio_packet(actor, frame) - if not first_frame_sent: - first_frame_sent = True - logger.info( - "audiosocket.first_frame session_id=%s greeting=%s frame_bytes=%s", - actor.registration.voice_session_id, - is_greeting, - len(frame), - ) - await asyncio.sleep(actor.frame_ms / 1000.0) - return False + return await self._write_paced_pcm_frames(actor, pcm_bytes, pacer, on_first_frame=_log_first_frame) async for synthesis in self._stream_tts_chunks(actor, text, style_hints=style_hints): if not synthesis.audio_bytes: @@ -1932,8 +1930,15 @@ class AudioSocketMediaRuntime: output_rate_hz=8000, state=resample_state, ) + if prebuffered: + # Past the initial cushion: feed each resampled chunk straight + # through instead of round-tripping it through pending_pcm. + interrupted = await _write_pcm_frames(pcm_8k) + if interrupted: + break + continue pending_pcm.extend(pcm_8k) - if not prebuffered and len(pending_pcm) < self._tts_stream_prebuffer_bytes: + if len(pending_pcm) < self._tts_stream_prebuffer_bytes: continue prebuffered = True interrupted = await _write_pcm_frames(bytes(pending_pcm)) @@ -1985,12 +1990,34 @@ class AudioSocketMediaRuntime: await actor.writer.drain() actor.last_outbound_audio_monotonic = time.monotonic() + async def _write_paced_pcm_frames( + self, + actor: MediaActor, + pcm_bytes: bytes, + pacer: _FramePacer, + *, + on_first_frame: Callable[[bytes], None] | None = None, + ) -> bool: + """Write pcm_bytes out as frame_bytes-sized packets, paced by `pacer`. + + Returns True if playback was interrupted (barge-in or actor closed). + """ + for frame in chunk_audio(pcm_bytes, frame_bytes=actor.frame_bytes): + if actor.closed or actor.playback_interrupt.is_set(): + return True + await self._write_audio_packet(actor, frame) + if pacer.frames_written == 0 and on_first_frame is not None: + on_first_frame(frame) + await pacer.wait_for_next_frame() + return False + async def _keepalive_loop(self, actor: MediaActor) -> None: silence_frame = b"\x00" * actor.frame_bytes while not actor.closed: await asyncio.sleep(0.25) if actor.closed: break + await self._maybe_reprompt_on_silence(actor) if (time.monotonic() - actor.last_outbound_audio_monotonic) < self._outbound_keepalive_interval_seconds: continue if not actor.keepalive_loop_logged: @@ -2021,18 +2048,14 @@ class AudioSocketMediaRuntime: if not actor.closed: await self._set_actor_state(actor, "listening") - async def _no_speech_reprompt_loop(self, actor: MediaActor) -> None: + async def _maybe_reprompt_on_silence(self, actor: MediaActor) -> None: if not self._no_speech_reprompt_enabled: return - while not actor.closed: - await asyncio.sleep(self._no_speech_reprompt_poll_seconds) - if actor.closed: - break - if actor.state != "listening" or actor.listening_since_monotonic <= 0: - continue - if time.monotonic() - actor.listening_since_monotonic < self._no_speech_reprompt_seconds: - continue - await self._maybe_play_no_speech_reprompt(actor, trigger="silence_timeout") + if actor.state != "listening" or actor.listening_since_monotonic <= 0: + return + if time.monotonic() - actor.listening_since_monotonic < self._no_speech_reprompt_seconds: + return + await self._maybe_play_no_speech_reprompt(actor, trigger="silence_timeout") async def _set_actor_state( self, @@ -2086,10 +2109,6 @@ class AudioSocketMediaRuntime: actor.keepalive_task.cancel() with contextlib.suppress(asyncio.CancelledError, Exception): await actor.keepalive_task - if actor.reprompt_watchdog_task is not None: - actor.reprompt_watchdog_task.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await actor.reprompt_watchdog_task if actor.handoff_task is not None: actor.handoff_task.cancel() with contextlib.suppress(asyncio.CancelledError, Exception): diff --git a/tests/test_ai_voice_media_runtime.py b/tests/test_ai_voice_media_runtime.py index 2ced970..0d86128 100644 --- a/tests/test_ai_voice_media_runtime.py +++ b/tests/test_ai_voice_media_runtime.py @@ -2861,7 +2861,6 @@ def test_media_runtime_no_speech_watchdog_reprompts_on_silence_timeout(): handle_media_error=lambda session_id, message, metadata: None, ) runtime._no_speech_reprompt_seconds = 0.05 - runtime._no_speech_reprompt_poll_seconds = 0.02 spoken: list[str] = [] @@ -2887,16 +2886,21 @@ def test_media_runtime_no_speech_watchdog_reprompts_on_silence_timeout(): frame_ms=20, frame_bytes=320, ) + # The no-speech watchdog check now rides the existing keepalive loop + # instead of its own dedicated task; keep last_outbound_audio_monotonic + # fresh so the loop's silence-keepalive-audio branch (which needs a + # real writer) doesn't fire during this test. + actor.last_outbound_audio_monotonic = time.monotonic() await runtime._set_actor_state(actor, "listening") - watchdog_task = asyncio.create_task(runtime._no_speech_reprompt_loop(actor)) - for _ in range(50): + keepalive_task = asyncio.create_task(runtime._keepalive_loop(actor)) + for _ in range(20): if spoken: break - await asyncio.sleep(0.02) + await asyncio.sleep(0.05) actor.closed = True - watchdog_task.cancel() + keepalive_task.cancel() with contextlib.suppress(asyncio.CancelledError, Exception): - await watchdog_task + await keepalive_task assert spoken, "watchdog should reprompt after prolonged silence in listening state" asyncio.run(_scenario())