feat: enhance AudioSocketMediaRuntime to skip filler acks for closing intents and throttle repeated filler acks

services/ai_voice_runtime_service/media_runtime.py

@@ -217,6 +217,10 @@ class AudioSocketMediaRuntime:
         self._immediate_ack_min_ms = 700
         self._v2_ack_post_gap_seconds = 0.10
         self._v1_ack_wait_seconds = 0.6
+        # Skip a would-be filler if the previous one finished too recently, so rapid
+        # back-and-forth turns (e.g. a caller spelling out a phone number field by
+        # field) don't get a filler read before every single fragment.
+        self._ack_min_repeat_gap_seconds = 2.5
         self._partial_poll_interval_seconds = 0.20
         # Small startup cushion for streamed TTS playback: absorb ElevenLabs
         # network delivery jitter before we start pacing frames out to the
@@ -314,10 +318,32 @@ class AudioSocketMediaRuntime:
             return True
         return normalized in cls._FINAL_LOW_SIGNAL_PHRASES
 
+    _CLOSING_INTENT_PHRASES = (
+        "до свидания",
+        "всего доброго",
+        "хорошего дня",
+        "хорошего вечера",
+        "прощайте",
+        "созвонимся",
+        "это все спасибо",
+        "это всё спасибо",
+        "у меня все спасибо",
+        "у меня всё спасибо",
+        "больше вопросов нет",
+        "вопросов больше нет",
+        "спасибо за помощь",
+        "спасибо большое до свидания",
+        "сау болыңыз",
+        "келесіге дейін",
+        "рахмет көп",
+    )
+
     def _detect_early_intent(self, text: str) -> str:
         normalized = self._normalize_intent_text(text)
         if not normalized:
             return "unknown"
+        if any(token in normalized for token in self._CLOSING_INTENT_PHRASES):
+            return "closing"
         if any(token in normalized for token in ("оператор", "оператором", "человеком", "менеджер", "сотрудник")):
             return "operator_request"
         if any(token in normalized for token in ("график", "распис", "время работы", "work schedule", "жұмыс")):
@@ -347,6 +373,8 @@ class AudioSocketMediaRuntime:
 
     @staticmethod
     def _ack_kind_for_intent(intent: str) -> str:
+        if intent == "closing":
+            return "closing"
         if intent == "operator_request":
             return "handoff"
         if intent in {"schedule", "address", "price", "status", "problem"}:
@@ -406,8 +434,18 @@ class AudioSocketMediaRuntime:
             return True
         return normalized_intent != "unknown"
 
-    def _should_emit_blind_ack(self, actor: MediaActor, pcm_bytes: bytes) -> bool:
+    def _should_emit_blind_ack(self, actor: MediaActor, pcm_bytes: bytes, partial_transcript: str) -> bool:
+        """Duration-only fallback for when no usable partial transcript exists yet.
+
+        Must defer to the transcript when one *is* available: otherwise a caller
+        who already said a recognized filler-answer ("да"/"нет"/"хорошо") still
+        gets a blind ack just because the audio happened to cross the length
+        threshold, even though `_should_emit_partial_ack` correctly said no.
+        """
         del actor
+        transcript_text = str(partial_transcript or "").strip()
+        if transcript_text and self._is_low_signal_partial_transcript(transcript_text):
+            return False
         return len(pcm_bytes) >= self._immediate_ack_min_bytes
 
     @staticmethod
@@ -1116,10 +1154,22 @@ class AudioSocketMediaRuntime:
         metadata: dict[str, Any],
         ack_source: str,
         ack_kind: str | None = None,
+        intent: str | None = None,
     ) -> None:
         if actor.closed or actor.early_ack_started:
             return
-        ack_kind = ack_kind or self._ack_kind_for_intent(actor.partial_intent or "unknown")
+        effective_intent = str(
+            intent or actor.stable_partial_intent or actor.partial_intent or "unknown"
+        ).strip() or "unknown"
+        if effective_intent == "closing":
+            # The caller is wrapping up; a "thinking" filler right before the
+            # closing reply reads as robotic, so skip it and go straight to the reply.
+            return
+        if actor.last_ack_completed_monotonic and (
+            time.monotonic() - actor.last_ack_completed_monotonic
+        ) < self._ack_min_repeat_gap_seconds:
+            return
+        ack_kind = ack_kind or self._ack_kind_for_intent(effective_intent)
         ack_text, style_hints, ack_variant = self._select_ack_payload(
             actor,
             language=language,
@@ -1620,8 +1670,9 @@ class AudioSocketMediaRuntime:
                     language=actor.registration.language,
                     metadata=base_metadata,
                     ack_source="streaming_partial" if actor.asr_streaming_enabled else "precomputed_partial_asr",
+                    intent=partial_intent,
                 )
-            elif self._should_emit_blind_ack(actor, pcm_bytes):
+            elif self._should_emit_blind_ack(actor, pcm_bytes, partial_transcript):
                 await self._emit_early_ack(
                     actor,
                     language=actor.registration.language,

tests/test_ai_voice_media_runtime.py

@@ -1249,6 +1249,288 @@ def test_media_runtime_voice_v2_emits_blind_ack_on_first_turn_without_partial_si
     assert speak_events[1][0] == "Подскажите подробнее, пожалуйста."
 
 
+def test_media_runtime_voice_v2_skips_filler_ack_when_caller_says_goodbye():
+    planned: list[tuple[str, str, str, dict | None]] = []
+
+    class _GoodbyeASRProvider(ASRProvider):
+        name = "goodbye-asr"
+
+        def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
+            assert audio_bytes
+            return ASRTranscription(text="Спасибо, до свидания", language=language_hint or "ru", confidence=0.9)
+
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=400,
+        asr_provider=_GoodbyeASRProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)),
+        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
+            time.sleep(0.25)
+            or VoiceAITurnDecisionOut(
+                language=language or "ru",
+                intent="closing",
+                reply_text="Хорошо, всего доброго!",
+                confidence=0.9,
+                needs_handoff=False,
+                handoff_reason=None,
+                case_action="close",
+                kb_refs=[],
+                summary_text="call wrapped up",
+                model="stub-voice",
+                latency_ms=1,
+                status="active",
+            )
+        ),
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_speak_text(
+        current_actor,
+        text: str,
+        *,
+        is_greeting: bool,
+        style_hints: dict[str, object] | None = None,
+    ) -> None:
+        del current_actor, is_greeting, style_hints
+        await asyncio.sleep(0)
+
+    runtime._speak_text = _fake_speak_text  # type: ignore[method-assign]
+    pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_v2_goodbye",
+                call_id="call_media_runtime_v2_goodbye",
+                interaction_id="int_media_runtime_v2_goodbye",
+                ai_session_id="ais_media_runtime_v2_goodbye",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=False,
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=400),
+            frame_ms=20,
+            frame_bytes=320,
+        )
+        actor.finalized_caller_turn_count = 1
+        await runtime._process_utterance(actor, pcm_frame, False)
+
+    asyncio.run(_scenario())
+
+    assert [item[2] for item in planned] == ["reply"]
+    assert planned[0][1] == "Хорошо, всего доброго!"
+
+
+def test_media_runtime_voice_v2_throttles_repeated_filler_ack_within_gap():
+    planned: list[tuple[str, str, str, dict | None]] = []
+
+    class _SlowASRProvider(ASRProvider):
+        name = "slow-asr"
+
+        def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
+            assert audio_bytes
+            return ASRTranscription(text="Хочу узнать график работы", language=language_hint or "ru", confidence=0.9)
+
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=400,
+        asr_provider=_SlowASRProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)),
+        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
+            time.sleep(0.25)
+            or VoiceAITurnDecisionOut(
+                language=language or "ru",
+                intent="clarification",
+                reply_text="Подскажите, пожалуйста, какой город вас интересует?",
+                confidence=0.9,
+                needs_handoff=False,
+                handoff_reason=None,
+                case_action="keep_open",
+                kb_refs=[],
+                summary_text="reply ready",
+                model="stub-voice",
+                latency_ms=1,
+                status="active",
+            )
+        ),
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_speak_text(
+        current_actor,
+        text: str,
+        *,
+        is_greeting: bool,
+        style_hints: dict[str, object] | None = None,
+    ) -> None:
+        del current_actor, is_greeting, style_hints
+        await asyncio.sleep(0)
+
+    runtime._speak_text = _fake_speak_text  # type: ignore[method-assign]
+    pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_v2_throttle",
+                call_id="call_media_runtime_v2_throttle",
+                interaction_id="int_media_runtime_v2_throttle",
+                ai_session_id="ais_media_runtime_v2_throttle",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=False,
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=400),
+            frame_ms=20,
+            frame_bytes=320,
+        )
+        actor.finalized_caller_turn_count = 1
+        await runtime._process_utterance(actor, pcm_frame, False)
+        runtime._reset_live_turn_state(actor)
+        await runtime._process_utterance(actor, pcm_frame, False)
+
+    asyncio.run(_scenario())
+
+    assert [item[2] for item in planned] == ["ack", "reply", "reply"]
+
+
+def test_media_runtime_voice_v2_blind_ack_defers_to_known_low_signal_partial_transcript():
+    speak_events: list[str] = []
+
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=2000,
+        asr_provider=_StubASRProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: None,
+        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
+            language=language or "ru",
+            intent="clarification",
+            reply_text="Подскажите подробнее, пожалуйста.",
+            confidence=0.9,
+            needs_handoff=False,
+            handoff_reason=None,
+            case_action="keep_open",
+            kb_refs=[],
+            summary_text="reply ready",
+            model="stub-voice",
+            latency_ms=1,
+            status="active",
+        ),
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_speak_text(
+        current_actor,
+        text: str,
+        *,
+        is_greeting: bool,
+        style_hints: dict[str, object] | None = None,
+    ) -> None:
+        del current_actor, is_greeting, style_hints
+        speak_events.append(text)
+        await asyncio.sleep(0)
+
+    runtime._speak_text = _fake_speak_text  # type: ignore[method-assign]
+    pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_v2_low_signal_blind",
+                call_id="call_media_runtime_v2_low_signal_blind",
+                interaction_id="int_media_runtime_v2_low_signal_blind",
+                ai_session_id="ais_media_runtime_v2_low_signal_blind",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=True,
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
+            frame_ms=20,
+            frame_bytes=320,
+        )
+        actor.finalized_caller_turn_count = 1
+        # A caller who already said a recognized filler-answer ("да") should not
+        # get a blind ack just because the audio clip crossed the length threshold.
+        actor.stable_partial_transcript = "да"
+        await runtime._process_utterance(actor, pcm_frame * 40, False)
+
+    asyncio.run(_scenario())
+
+    assert speak_events == ["Подскажите подробнее, пожалуйста."]
+
+
 def test_media_runtime_low_signal_filter_catches_short_asr_noise():
     assert AudioSocketMediaRuntime._is_low_signal_partial_transcript("Давай")
     assert AudioSocketMediaRuntime._is_low_signal_partial_transcript("твой")