diff --git a/services/ai_voice_runtime_service/media_runtime.py b/services/ai_voice_runtime_service/media_runtime.py index 75c55ba..5bd0936 100644 --- a/services/ai_voice_runtime_service/media_runtime.py +++ b/services/ai_voice_runtime_service/media_runtime.py @@ -81,6 +81,7 @@ class MediaActor: partial_asr_task: asyncio.Task | None = None partial_asr_attempted: bool = False last_ack_text: str | None = None + last_ack_completed_monotonic: float = 0.0 class AudioSocketMediaRuntime: @@ -139,7 +140,8 @@ class AudioSocketMediaRuntime: self._actors: dict[str, MediaActor] = {} self._v2_ack_wait_seconds = 0.18 self._partial_asr_min_ms = 650 - self._immediate_ack_min_ms = 450 + self._immediate_ack_min_ms = 280 + self._v2_ack_post_gap_seconds = 0.12 @staticmethod def _normalize_intent_text(text: str) -> str: @@ -378,6 +380,7 @@ class AudioSocketMediaRuntime: }, ) await self._speak_text(actor, ack_text, is_greeting=False, style_hints=style_hints) + actor.last_ack_completed_monotonic = time.monotonic() if not actor.closed: await self._set_actor_state(actor, "thinking") @@ -708,6 +711,13 @@ class AudioSocketMediaRuntime: await self._set_actor_state(actor, "handoff_requested", decision.handoff_reason) handoff_task = self._start_handoff_request(actor, transcript_text, decision) if decision.reply_text: + if self._should_use_voice_v2(actor.registration) and actor.early_ack_started: + remaining_gap = self._v2_ack_post_gap_seconds - max( + 0.0, + time.monotonic() - actor.last_ack_completed_monotonic, + ) + if remaining_gap > 0: + await asyncio.sleep(remaining_gap) if self._should_use_voice_v2(actor.registration): await self._plan_reply_segment( actor, diff --git a/tests/test_ai_voice_media_runtime.py b/tests/test_ai_voice_media_runtime.py index 138ac8a..f77acc0 100644 --- a/tests/test_ai_voice_media_runtime.py +++ b/tests/test_ai_voice_media_runtime.py @@ -846,6 +846,199 @@ def test_media_runtime_voice_v2_emits_generic_ack_before_full_asr_without_partia assert speak_events[0][1] < timings["full_finished"] +def test_media_runtime_voice_v2_emits_ack_for_short_utterance_after_reduced_threshold(): + timings: dict[str, float] = {} + speak_events: list[tuple[str, float]] = [] + + class _FastASRProvider(ASRProvider): + name = "fast-asr" + + def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription: + assert audio_bytes + timings["full_started"] = time.monotonic() + time.sleep(0.12) + timings["full_finished"] = time.monotonic() + return ASRTranscription(text="hello there", language=language_hint or "ru", confidence=0.9) + + runtime = AudioSocketMediaRuntime( + enabled=True, + host="127.0.0.1", + port=0, + frame_ms=20, + idle_timeout_seconds=2.0, + registration_wait_timeout_seconds=0.5, + min_speech_ms=40, + trailing_silence_ms=40, + max_turn_ms=2000, + asr_provider=_FastASRProvider(), + tts_provider=_StubTTSProvider(), + load_registration_by_media_uuid=lambda value: None, + mark_media_connected=lambda session_id, value: None, + mark_media_ended=lambda session_id, reason: None, + touch_media_frame=lambda session_id: None, + set_state=lambda session_id, state, handoff_reason, metadata: None, + get_pending_greeting=lambda session_id: None, + mark_reply_delivered=lambda session_id, text, is_greeting: None, + plan_reply=lambda session_id, text, metadata, kind: None, + process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut( + language=language or "ru", + intent="clarification", + reply_text="Подскажите подробнее, пожалуйста.", + confidence=0.9, + needs_handoff=False, + handoff_reason=None, + case_action="keep_open", + kb_refs=[], + summary_text="reply ready", + model="stub-voice", + latency_ms=1, + status="active", + ), + request_handoff=lambda session_id, customer_request_text, decision: None, + handle_media_error=lambda session_id, message, metadata: None, + ) + + async def _fake_speak_text( + current_actor, + text: str, + *, + is_greeting: bool, + style_hints: dict[str, object] | None = None, + ) -> None: + del current_actor, is_greeting, style_hints + speak_events.append((text, time.monotonic())) + await asyncio.sleep(0) + + runtime._speak_text = _fake_speak_text # type: ignore[method-assign] + pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160 + + async def _scenario() -> None: + actor = MediaActor( + registration=MediaRegistration( + voice_session_id="avs_media_runtime_v2_short_ack", + call_id="call_media_runtime_v2_short_ack", + interaction_id="int_media_runtime_v2_short_ack", + ai_session_id="ais_media_runtime_v2_short_ack", + language="ru", + media_uuid=str(uuid.uuid4()), + queue_code="voice_lab_ai", + queue_id="que_voice_lab_ai", + agent_profile="voice_support", + voice_v2_enabled=True, + voice_v2_ack_mode="immediate_short", + voice_v2_streaming_tts=True, + voice_v2_partial_asr=False, + ), + reader=asyncio.StreamReader(), + writer=None, # type: ignore[arg-type] + vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000), + frame_ms=20, + frame_bytes=320, + ) + await runtime._process_utterance(actor, pcm_frame * 20, False) + + asyncio.run(_scenario()) + + assert speak_events + assert speak_events[0][0] == runtime._ack_text("ru", "generic") + assert speak_events[0][1] < timings["full_finished"] + + +def test_media_runtime_voice_v2_inserts_small_gap_between_ack_and_main_reply(): + speak_events: list[tuple[str, float]] = [] + + class _InstantASRProvider(ASRProvider): + name = "instant-asr" + + def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription: + assert audio_bytes + return ASRTranscription(text="hello there", language=language_hint or "ru", confidence=0.9) + + runtime = AudioSocketMediaRuntime( + enabled=True, + host="127.0.0.1", + port=0, + frame_ms=20, + idle_timeout_seconds=2.0, + registration_wait_timeout_seconds=0.5, + min_speech_ms=40, + trailing_silence_ms=40, + max_turn_ms=2000, + asr_provider=_InstantASRProvider(), + tts_provider=_StubTTSProvider(), + load_registration_by_media_uuid=lambda value: None, + mark_media_connected=lambda session_id, value: None, + mark_media_ended=lambda session_id, reason: None, + touch_media_frame=lambda session_id: None, + set_state=lambda session_id, state, handoff_reason, metadata: None, + get_pending_greeting=lambda session_id: None, + mark_reply_delivered=lambda session_id, text, is_greeting: None, + plan_reply=lambda session_id, text, metadata, kind: None, + process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut( + language=language or "ru", + intent="clarification", + reply_text="Подскажите подробнее, пожалуйста.", + confidence=0.9, + needs_handoff=False, + handoff_reason=None, + case_action="keep_open", + kb_refs=[], + summary_text="reply ready", + model="stub-voice", + latency_ms=1, + status="active", + ), + request_handoff=lambda session_id, customer_request_text, decision: None, + handle_media_error=lambda session_id, message, metadata: None, + ) + + async def _fake_speak_text( + current_actor, + text: str, + *, + is_greeting: bool, + style_hints: dict[str, object] | None = None, + ) -> None: + del current_actor, is_greeting, style_hints + speak_events.append((text, time.monotonic())) + await asyncio.sleep(0) + + runtime._speak_text = _fake_speak_text # type: ignore[method-assign] + pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160 + + async def _scenario() -> None: + actor = MediaActor( + registration=MediaRegistration( + voice_session_id="avs_media_runtime_v2_gap", + call_id="call_media_runtime_v2_gap", + interaction_id="int_media_runtime_v2_gap", + ai_session_id="ais_media_runtime_v2_gap", + language="ru", + media_uuid=str(uuid.uuid4()), + queue_code="voice_lab_ai", + queue_id="que_voice_lab_ai", + agent_profile="voice_support", + voice_v2_enabled=True, + voice_v2_ack_mode="immediate_short", + voice_v2_streaming_tts=True, + voice_v2_partial_asr=False, + ), + reader=asyncio.StreamReader(), + writer=None, # type: ignore[arg-type] + vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000), + frame_ms=20, + frame_bytes=320, + ) + await runtime._process_utterance(actor, pcm_frame * 20, False) + + asyncio.run(_scenario()) + + assert len(speak_events) == 2 + assert speak_events[0][0] == runtime._ack_text("ru", "generic") + assert speak_events[1][0] == "Подскажите подробнее, пожалуйста." + assert (speak_events[1][1] - speak_events[0][1]) >= (runtime._v2_ack_post_gap_seconds - 0.02) + + def test_media_runtime_voice_v2_emotive_ack_uses_ru_variants_and_style_hints_only_for_ack(): synth_calls: list[tuple[str, dict[str, object] | None]] = []