From 9e4c47eddd20a66bf46a4cd61efbbd2cc0d3fb27 Mon Sep 17 00:00:00 2001 From: didar Date: Sun, 30 Aug 2026 11:30:55 +0500 Subject: [PATCH] feat: enhance AudioSocketMediaRuntime to skip filler acks for closing intents and throttle repeated filler acks --- .../ai_voice_runtime_service/media_runtime.py | 57 +++- tests/test_ai_voice_media_runtime.py | 282 ++++++++++++++++++ 2 files changed, 336 insertions(+), 3 deletions(-) diff --git a/services/ai_voice_runtime_service/media_runtime.py b/services/ai_voice_runtime_service/media_runtime.py index 7a788c6..c53d248 100644 --- a/services/ai_voice_runtime_service/media_runtime.py +++ b/services/ai_voice_runtime_service/media_runtime.py @@ -217,6 +217,10 @@ class AudioSocketMediaRuntime: self._immediate_ack_min_ms = 700 self._v2_ack_post_gap_seconds = 0.10 self._v1_ack_wait_seconds = 0.6 + # Skip a would-be filler if the previous one finished too recently, so rapid + # back-and-forth turns (e.g. a caller spelling out a phone number field by + # field) don't get a filler read before every single fragment. + self._ack_min_repeat_gap_seconds = 2.5 self._partial_poll_interval_seconds = 0.20 # Small startup cushion for streamed TTS playback: absorb ElevenLabs # network delivery jitter before we start pacing frames out to the @@ -314,10 +318,32 @@ class AudioSocketMediaRuntime: return True return normalized in cls._FINAL_LOW_SIGNAL_PHRASES + _CLOSING_INTENT_PHRASES = ( + "до свидания", + "всего доброго", + "хорошего дня", + "хорошего вечера", + "прощайте", + "созвонимся", + "это все спасибо", + "это всё спасибо", + "у меня все спасибо", + "у меня всё спасибо", + "больше вопросов нет", + "вопросов больше нет", + "спасибо за помощь", + "спасибо большое до свидания", + "сау болыңыз", + "келесіге дейін", + "рахмет көп", + ) + def _detect_early_intent(self, text: str) -> str: normalized = self._normalize_intent_text(text) if not normalized: return "unknown" + if any(token in normalized for token in self._CLOSING_INTENT_PHRASES): + return "closing" if any(token in normalized for token in ("оператор", "оператором", "человеком", "менеджер", "сотрудник")): return "operator_request" if any(token in normalized for token in ("график", "распис", "время работы", "work schedule", "жұмыс")): @@ -347,6 +373,8 @@ class AudioSocketMediaRuntime: @staticmethod def _ack_kind_for_intent(intent: str) -> str: + if intent == "closing": + return "closing" if intent == "operator_request": return "handoff" if intent in {"schedule", "address", "price", "status", "problem"}: @@ -406,8 +434,18 @@ class AudioSocketMediaRuntime: return True return normalized_intent != "unknown" - def _should_emit_blind_ack(self, actor: MediaActor, pcm_bytes: bytes) -> bool: + def _should_emit_blind_ack(self, actor: MediaActor, pcm_bytes: bytes, partial_transcript: str) -> bool: + """Duration-only fallback for when no usable partial transcript exists yet. + + Must defer to the transcript when one *is* available: otherwise a caller + who already said a recognized filler-answer ("да"/"нет"/"хорошо") still + gets a blind ack just because the audio happened to cross the length + threshold, even though `_should_emit_partial_ack` correctly said no. + """ del actor + transcript_text = str(partial_transcript or "").strip() + if transcript_text and self._is_low_signal_partial_transcript(transcript_text): + return False return len(pcm_bytes) >= self._immediate_ack_min_bytes @staticmethod @@ -1116,10 +1154,22 @@ class AudioSocketMediaRuntime: metadata: dict[str, Any], ack_source: str, ack_kind: str | None = None, + intent: str | None = None, ) -> None: if actor.closed or actor.early_ack_started: return - ack_kind = ack_kind or self._ack_kind_for_intent(actor.partial_intent or "unknown") + effective_intent = str( + intent or actor.stable_partial_intent or actor.partial_intent or "unknown" + ).strip() or "unknown" + if effective_intent == "closing": + # The caller is wrapping up; a "thinking" filler right before the + # closing reply reads as robotic, so skip it and go straight to the reply. + return + if actor.last_ack_completed_monotonic and ( + time.monotonic() - actor.last_ack_completed_monotonic + ) < self._ack_min_repeat_gap_seconds: + return + ack_kind = ack_kind or self._ack_kind_for_intent(effective_intent) ack_text, style_hints, ack_variant = self._select_ack_payload( actor, language=language, @@ -1620,8 +1670,9 @@ class AudioSocketMediaRuntime: language=actor.registration.language, metadata=base_metadata, ack_source="streaming_partial" if actor.asr_streaming_enabled else "precomputed_partial_asr", + intent=partial_intent, ) - elif self._should_emit_blind_ack(actor, pcm_bytes): + elif self._should_emit_blind_ack(actor, pcm_bytes, partial_transcript): await self._emit_early_ack( actor, language=actor.registration.language, diff --git a/tests/test_ai_voice_media_runtime.py b/tests/test_ai_voice_media_runtime.py index 0d86128..6b3eb70 100644 --- a/tests/test_ai_voice_media_runtime.py +++ b/tests/test_ai_voice_media_runtime.py @@ -1249,6 +1249,288 @@ def test_media_runtime_voice_v2_emits_blind_ack_on_first_turn_without_partial_si assert speak_events[1][0] == "Подскажите подробнее, пожалуйста." +def test_media_runtime_voice_v2_skips_filler_ack_when_caller_says_goodbye(): + planned: list[tuple[str, str, str, dict | None]] = [] + + class _GoodbyeASRProvider(ASRProvider): + name = "goodbye-asr" + + def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription: + assert audio_bytes + return ASRTranscription(text="Спасибо, до свидания", language=language_hint or "ru", confidence=0.9) + + runtime = AudioSocketMediaRuntime( + enabled=True, + host="127.0.0.1", + port=0, + frame_ms=20, + idle_timeout_seconds=2.0, + registration_wait_timeout_seconds=0.5, + min_speech_ms=40, + trailing_silence_ms=40, + max_turn_ms=400, + asr_provider=_GoodbyeASRProvider(), + tts_provider=_StubTTSProvider(), + load_registration_by_media_uuid=lambda value: None, + mark_media_connected=lambda session_id, value: None, + mark_media_ended=lambda session_id, reason: None, + touch_media_frame=lambda session_id: None, + set_state=lambda session_id, state, handoff_reason, metadata: None, + get_pending_greeting=lambda session_id: None, + mark_reply_delivered=lambda session_id, text, is_greeting: None, + plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)), + process_turn=lambda session_id, transcript_text, language, barge_in, metadata: ( + time.sleep(0.25) + or VoiceAITurnDecisionOut( + language=language or "ru", + intent="closing", + reply_text="Хорошо, всего доброго!", + confidence=0.9, + needs_handoff=False, + handoff_reason=None, + case_action="close", + kb_refs=[], + summary_text="call wrapped up", + model="stub-voice", + latency_ms=1, + status="active", + ) + ), + request_handoff=lambda session_id, customer_request_text, decision: None, + handle_media_error=lambda session_id, message, metadata: None, + ) + + async def _fake_speak_text( + current_actor, + text: str, + *, + is_greeting: bool, + style_hints: dict[str, object] | None = None, + ) -> None: + del current_actor, is_greeting, style_hints + await asyncio.sleep(0) + + runtime._speak_text = _fake_speak_text # type: ignore[method-assign] + pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160 + + async def _scenario() -> None: + actor = MediaActor( + registration=MediaRegistration( + voice_session_id="avs_media_runtime_v2_goodbye", + call_id="call_media_runtime_v2_goodbye", + interaction_id="int_media_runtime_v2_goodbye", + ai_session_id="ais_media_runtime_v2_goodbye", + language="ru", + media_uuid=str(uuid.uuid4()), + queue_code="voice_lab_ai", + queue_id="que_voice_lab_ai", + agent_profile="voice_support", + voice_v2_enabled=True, + voice_v2_ack_mode="immediate_short", + voice_v2_streaming_tts=True, + voice_v2_partial_asr=False, + ), + reader=asyncio.StreamReader(), + writer=None, # type: ignore[arg-type] + vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=400), + frame_ms=20, + frame_bytes=320, + ) + actor.finalized_caller_turn_count = 1 + await runtime._process_utterance(actor, pcm_frame, False) + + asyncio.run(_scenario()) + + assert [item[2] for item in planned] == ["reply"] + assert planned[0][1] == "Хорошо, всего доброго!" + + +def test_media_runtime_voice_v2_throttles_repeated_filler_ack_within_gap(): + planned: list[tuple[str, str, str, dict | None]] = [] + + class _SlowASRProvider(ASRProvider): + name = "slow-asr" + + def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription: + assert audio_bytes + return ASRTranscription(text="Хочу узнать график работы", language=language_hint or "ru", confidence=0.9) + + runtime = AudioSocketMediaRuntime( + enabled=True, + host="127.0.0.1", + port=0, + frame_ms=20, + idle_timeout_seconds=2.0, + registration_wait_timeout_seconds=0.5, + min_speech_ms=40, + trailing_silence_ms=40, + max_turn_ms=400, + asr_provider=_SlowASRProvider(), + tts_provider=_StubTTSProvider(), + load_registration_by_media_uuid=lambda value: None, + mark_media_connected=lambda session_id, value: None, + mark_media_ended=lambda session_id, reason: None, + touch_media_frame=lambda session_id: None, + set_state=lambda session_id, state, handoff_reason, metadata: None, + get_pending_greeting=lambda session_id: None, + mark_reply_delivered=lambda session_id, text, is_greeting: None, + plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)), + process_turn=lambda session_id, transcript_text, language, barge_in, metadata: ( + time.sleep(0.25) + or VoiceAITurnDecisionOut( + language=language or "ru", + intent="clarification", + reply_text="Подскажите, пожалуйста, какой город вас интересует?", + confidence=0.9, + needs_handoff=False, + handoff_reason=None, + case_action="keep_open", + kb_refs=[], + summary_text="reply ready", + model="stub-voice", + latency_ms=1, + status="active", + ) + ), + request_handoff=lambda session_id, customer_request_text, decision: None, + handle_media_error=lambda session_id, message, metadata: None, + ) + + async def _fake_speak_text( + current_actor, + text: str, + *, + is_greeting: bool, + style_hints: dict[str, object] | None = None, + ) -> None: + del current_actor, is_greeting, style_hints + await asyncio.sleep(0) + + runtime._speak_text = _fake_speak_text # type: ignore[method-assign] + pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160 + + async def _scenario() -> None: + actor = MediaActor( + registration=MediaRegistration( + voice_session_id="avs_media_runtime_v2_throttle", + call_id="call_media_runtime_v2_throttle", + interaction_id="int_media_runtime_v2_throttle", + ai_session_id="ais_media_runtime_v2_throttle", + language="ru", + media_uuid=str(uuid.uuid4()), + queue_code="voice_lab_ai", + queue_id="que_voice_lab_ai", + agent_profile="voice_support", + voice_v2_enabled=True, + voice_v2_ack_mode="immediate_short", + voice_v2_streaming_tts=True, + voice_v2_partial_asr=False, + ), + reader=asyncio.StreamReader(), + writer=None, # type: ignore[arg-type] + vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=400), + frame_ms=20, + frame_bytes=320, + ) + actor.finalized_caller_turn_count = 1 + await runtime._process_utterance(actor, pcm_frame, False) + runtime._reset_live_turn_state(actor) + await runtime._process_utterance(actor, pcm_frame, False) + + asyncio.run(_scenario()) + + assert [item[2] for item in planned] == ["ack", "reply", "reply"] + + +def test_media_runtime_voice_v2_blind_ack_defers_to_known_low_signal_partial_transcript(): + speak_events: list[str] = [] + + runtime = AudioSocketMediaRuntime( + enabled=True, + host="127.0.0.1", + port=0, + frame_ms=20, + idle_timeout_seconds=2.0, + registration_wait_timeout_seconds=0.5, + min_speech_ms=40, + trailing_silence_ms=40, + max_turn_ms=2000, + asr_provider=_StubASRProvider(), + tts_provider=_StubTTSProvider(), + load_registration_by_media_uuid=lambda value: None, + mark_media_connected=lambda session_id, value: None, + mark_media_ended=lambda session_id, reason: None, + touch_media_frame=lambda session_id: None, + set_state=lambda session_id, state, handoff_reason, metadata: None, + get_pending_greeting=lambda session_id: None, + mark_reply_delivered=lambda session_id, text, is_greeting: None, + plan_reply=lambda session_id, text, metadata, kind: None, + process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut( + language=language or "ru", + intent="clarification", + reply_text="Подскажите подробнее, пожалуйста.", + confidence=0.9, + needs_handoff=False, + handoff_reason=None, + case_action="keep_open", + kb_refs=[], + summary_text="reply ready", + model="stub-voice", + latency_ms=1, + status="active", + ), + request_handoff=lambda session_id, customer_request_text, decision: None, + handle_media_error=lambda session_id, message, metadata: None, + ) + + async def _fake_speak_text( + current_actor, + text: str, + *, + is_greeting: bool, + style_hints: dict[str, object] | None = None, + ) -> None: + del current_actor, is_greeting, style_hints + speak_events.append(text) + await asyncio.sleep(0) + + runtime._speak_text = _fake_speak_text # type: ignore[method-assign] + pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160 + + async def _scenario() -> None: + actor = MediaActor( + registration=MediaRegistration( + voice_session_id="avs_media_runtime_v2_low_signal_blind", + call_id="call_media_runtime_v2_low_signal_blind", + interaction_id="int_media_runtime_v2_low_signal_blind", + ai_session_id="ais_media_runtime_v2_low_signal_blind", + language="ru", + media_uuid=str(uuid.uuid4()), + queue_code="voice_lab_ai", + queue_id="que_voice_lab_ai", + agent_profile="voice_support", + voice_v2_enabled=True, + voice_v2_ack_mode="immediate_short", + voice_v2_streaming_tts=True, + voice_v2_partial_asr=True, + ), + reader=asyncio.StreamReader(), + writer=None, # type: ignore[arg-type] + vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000), + frame_ms=20, + frame_bytes=320, + ) + actor.finalized_caller_turn_count = 1 + # A caller who already said a recognized filler-answer ("да") should not + # get a blind ack just because the audio clip crossed the length threshold. + actor.stable_partial_transcript = "да" + await runtime._process_utterance(actor, pcm_frame * 40, False) + + asyncio.run(_scenario()) + + assert speak_events == ["Подскажите подробнее, пожалуйста."] + + def test_media_runtime_low_signal_filter_catches_short_asr_noise(): assert AudioSocketMediaRuntime._is_low_signal_partial_transcript("Давай") assert AudioSocketMediaRuntime._is_low_signal_partial_transcript("твой")