From 57bd5cad4b4d2a0aeea24fb5a7a9fbb18e0c2b25 Mon Sep 17 00:00:00 2001 From: Yera All Date: Sun, 12 Apr 2026 20:40:25 +0500 Subject: [PATCH] fix(voice): suppress low-signal turns and false lookup replies --- services/ai_orchestrator_service/voice.py | 144 +++++++++++++++++- .../ai_voice_runtime_service/media_runtime.py | 41 ++++- services/streaming_asr_sidecar_service/app.py | 13 ++ tests/test_ai_orchestrator_service.py | 46 ++++++ tests/test_ai_voice_media_runtime.py | 90 ++++++++++- 5 files changed, 323 insertions(+), 11 deletions(-) diff --git a/services/ai_orchestrator_service/voice.py b/services/ai_orchestrator_service/voice.py index 7c31475..d12a8a4 100644 --- a/services/ai_orchestrator_service/voice.py +++ b/services/ai_orchestrator_service/voice.py @@ -806,6 +806,7 @@ def _voice_is_low_signal_caller_text(text: str | None) -> bool: "ладно", "неа", "нет", + "ой", "ок", "понял", "поняла", @@ -819,6 +820,21 @@ def _voice_is_low_signal_caller_text(text: str | None) -> bool: return normalized in low_signal_phrases +def _voice_is_hearing_check_caller_text(text: str | None) -> bool: + normalized = _voice_text_key(text) + if not normalized: + return False + markers = ( + "алло ты меня слышишь", + "вы меня слышите", + "меня слышно", + "меня слышишь", + "ты меня слышишь", + "слышишь меня", + ) + return any(marker in normalized for marker in markers) + + def _voice_is_confused_caller_text(text: str | None) -> bool: normalized = _voice_text_key(text) confusion_markers = ( @@ -878,6 +894,25 @@ def _voice_topic_prompt(language: str, caller_texts: list[str]) -> str | None: if not context: return None if any(marker in context for marker in ("график", "время работы", "режим работы", "часы работы", "работаете")): + city_scope = any( + marker in context + for marker in ( + "алмат", + "астан", + "в городе", + "город", + "караганд", + "кызылорд", + "павлодар", + "семей", + "тараз", + "шымкент", + ) + ) + if city_scope: + if language == "kz": + return "Osy qaladagy qaysy filial nemese mekenjai qyzyqtyratynyn aitnyz." + return "Подскажите, какой филиал или адрес в этом городе вас интересует?" if language == "kz": return "Qai filialdyn, mekendyng nemese qalanyng jumys uaqyty qyzyqtyratynyn aitnyz." return "Подскажите, график работы какого филиала, адреса или города вас интересует?" @@ -1005,11 +1040,11 @@ def _voice_confusion_prompt(language: str, caller_texts: list[str]) -> str: topic_prompt = _voice_topic_prompt(language, caller_texts) if topic_prompt: if language == "kz": - return f"Qongyrau taqyrybyn naqtylap jatyrmyn. {topic_prompt}" - return f"Сейчас уточняю цель звонка. {topic_prompt}" + return f"Naqtylap alaiyn. {topic_prompt}" + return f"Подскажите точнее. {topic_prompt}" if language == "kz": - return "Qazir qongyraudyn maqratyn anyqtap jatyrmyn. Qysqasha aitnyz: jumys uaqyty, otinish statusy, tarif nemese operator." - return "Сейчас уточняю цель звонка. Скажите коротко, что именно нужно: график работы, статус заявки, тариф или оператор." + return "Naqtylap alaiyn: jumys uaqyty, otinish statusy, tarif nemese operator degenniń birin aitnyz." + return "Подскажите точнее: график работы, статус заявки, тариф или оператор." def _voice_loop_handoff(language: str) -> tuple[str, str, str]: @@ -1212,6 +1247,79 @@ def _voice_compact_reply_text(text: str, *, language: str) -> str: return f"{shortened}{ending}" +def _voice_caller_context_before_current(caller_texts: list[str], transcript_text: str) -> list[str]: + if not caller_texts: + return [] + current_key = _voice_text_key(transcript_text) + if current_key and _voice_text_key(caller_texts[-1]) == current_key: + return caller_texts[:-1] + return caller_texts + + +def _voice_is_midcall_greeting_reply(text: str | None) -> bool: + normalized = _voice_text_key(text) + if not normalized: + return False + greeting_markers = ("здравствуйте", "сәлеметсіз", "сәлем") + followup_markers = ("чем помочь", "как я могу помочь", "коротко расскажите", "қалай көмектесе") + return any(marker in normalized for marker in greeting_markers) and any( + marker in normalized for marker in followup_markers + ) + + +def _voice_contains_false_lookup_promise(text: str | None) -> bool: + normalized = _voice_text_key(text) + if not normalized: + return False + markers = ( + "сейчас уточню", + "я уточню", + "уточню", + "минуточ", + "подождите", + "подождите пожалуйста", + "проверю", + "сейчас проверю", + "я проверю", + "посмотрю", + "сейчас посмотрю", + ) + return any(marker in normalized for marker in markers) + + +def _voice_hearing_check_reply(language: str, caller_texts: list[str]) -> str: + followup = _voice_topic_prompt(language, caller_texts) or _voice_generic_prompt(language) + if language == "kz": + return f"Ia, estip turmyn. {followup}" + return f"Да, вас слышу. {followup}" + + +def _voice_postprocess_reply_text( + *, + language: str, + transcript_text: str, + transcript_window: list[VoiceTranscriptSegmentRow], + reply_text: str, + kb_results: list[Any], + needs_handoff: bool, +) -> str: + normalized_reply = str(reply_text or "").strip() + if not normalized_reply or needs_handoff: + return normalized_reply + caller_texts = _voice_recent_caller_texts(transcript_window) + prior_caller_texts = _voice_caller_context_before_current(caller_texts, transcript_text) + if _voice_is_hearing_check_caller_text(transcript_text): + return _voice_hearing_check_reply(language, prior_caller_texts or caller_texts) + if _voice_contains_false_lookup_promise(normalized_reply) and not kb_results: + return _voice_topic_prompt(language, caller_texts) or _voice_generic_prompt(language) + if _voice_is_midcall_greeting_reply(normalized_reply) and any( + segment.speaker == "assistant" for segment in transcript_window + ): + context_texts = prior_caller_texts if _voice_is_low_signal_caller_text(transcript_text) else caller_texts + return _voice_topic_prompt(language, context_texts) or _voice_generic_prompt(language) + return normalized_reply + + def _voice_v2_metadata( transcript_text: str, request_metadata: dict[str, Any] | None = None, @@ -1653,6 +1761,26 @@ def _voice_decision( "latency_ms": 1, } + if _voice_is_hearing_check_caller_text(normalized): + prior_caller_texts = _voice_caller_context_before_current(caller_texts, transcript_text) + decision = { + "language": language, + "intent": "clarification", + "reply_text": _voice_hearing_check_reply(language, prior_caller_texts or caller_texts), + "confidence": 0.66, + "needs_handoff": False, + "handoff_reason": None, + "case_action": "keep_open", + "kb_refs": [], + "summary_text": "AI подтвердил, что слышит клиента, и продолжил активный сценарий без сброса диалога.", + "model": "voice_policy_hearing_check", + "latency_ms": 1, + } + if v2_metadata: + decision["reply_text"] = _voice_compact_reply_text(decision["reply_text"], language=language) + decision["metadata"] = {**v2_metadata, "reply_phase": "final"} + return decision + clarification_count = _voice_recent_clarification_count(transcript_window) repeated_reply_count = _voice_repeated_assistant_reply_count(transcript_window) recent_caller_window = caller_texts[-3:] @@ -2283,6 +2411,14 @@ def turn_voice_session(session_id: str, payload: VoiceAITurnIn) -> VoiceAITurnDe customer_name_status=effective_name_status, request_metadata=request_metadata, ) + decision["reply_text"] = _voice_postprocess_reply_text( + language=decision["language"], + transcript_text=payload.transcript_text, + transcript_window=transcript_window, + reply_text=str(decision.get("reply_text") or ""), + kb_results=kb_results, + needs_handoff=bool(decision.get("needs_handoff")), + ) if decision.get("extracted_name") and not early_plan_only: effective_name_status = "name_obtained" diff --git a/services/ai_voice_runtime_service/media_runtime.py b/services/ai_voice_runtime_service/media_runtime.py index 777ace8..8682ddf 100644 --- a/services/ai_voice_runtime_service/media_runtime.py +++ b/services/ai_voice_runtime_service/media_runtime.py @@ -174,7 +174,7 @@ class AudioSocketMediaRuntime: self._actors: dict[str, MediaActor] = {} self._v2_ack_wait_seconds = 0.18 self._partial_asr_min_ms = 320 - self._immediate_ack_min_ms = 280 + self._immediate_ack_min_ms = 700 self._v2_ack_post_gap_seconds = 0.10 self._partial_poll_interval_seconds = 0.20 self._stable_partial_hold_seconds = 0.40 @@ -184,6 +184,32 @@ class AudioSocketMediaRuntime: def _normalize_intent_text(text: str) -> str: return " ".join(str(text or "").strip().lower().split()) + @classmethod + def _is_low_signal_transcript(cls, text: str | None) -> bool: + normalized = cls._normalize_intent_text(text) + if not normalized: + return True + return normalized in { + "ага", + "алло", + "да", + "добрый день", + "здравствуйте", + "ладно", + "неа", + "нет", + "ой", + "ок", + "понял", + "поняла", + "привет", + "слышу", + "слышно", + "угу", + "хорошо", + "ясно", + } + def _detect_early_intent(self, text: str) -> str: normalized = self._normalize_intent_text(text) if not normalized: @@ -363,8 +389,6 @@ class AudioSocketMediaRuntime: ) except StreamingASRUnavailable as exc: actor.asr_streaming_enabled = False - actor.registration.voice_v2_duplex = False - actor.registration.voice_v2_partial_asr = False logger.warning( "audiosocket.streaming_asr_unavailable session_id=%s backend=%s error=%s", actor.registration.voice_session_id, @@ -913,8 +937,6 @@ class AudioSocketMediaRuntime: str(exc)[:500], ) await self._close_streaming_asr(actor) - actor.registration.voice_v2_duplex = False - actor.registration.voice_v2_partial_asr = False elif actor.registration.voice_v2_partial_asr: self._maybe_schedule_partial_asr(actor) if vad_result.utterance_pcm: @@ -1008,6 +1030,15 @@ class AudioSocketMediaRuntime: if not transcript_text: await self._set_actor_state(actor, "listening") return + if self._is_low_signal_transcript(transcript_text): + actor.finalized_utterance_generation = max(actor.finalized_utterance_generation, utterance_generation) + logger.warning( + "audiosocket.low_signal_ignored session_id=%s transcript=%s", + actor.registration.voice_session_id, + transcript_text[:120], + ) + await self._set_actor_state(actor, "listening") + return actor.finalized_utterance_generation = max(actor.finalized_utterance_generation, utterance_generation) actor.partial_transcript = transcript_text if actor.registration.voice_v2_partial_asr else None diff --git a/services/streaming_asr_sidecar_service/app.py b/services/streaming_asr_sidecar_service/app.py index e394282..00d2c12 100644 --- a/services/streaming_asr_sidecar_service/app.py +++ b/services/streaming_asr_sidecar_service/app.py @@ -236,6 +236,19 @@ def _get_engine() -> TranscriptionEngine: return _ENGINE_INSTANCE +def _warmup_engine() -> None: + try: + _get_engine() + LOGGER.info("streaming_asr.engine_ready") + except Exception as exc: # noqa: BLE001 + LOGGER.warning("streaming_asr.engine_warmup_failed error=%s", str(exc)[:500]) + + +@app.on_event("startup") +def _startup_warmup_engine() -> None: + threading.Thread(target=_warmup_engine, daemon=True).start() + + @dataclass(slots=True) class StreamState: stream_id: str diff --git a/tests/test_ai_orchestrator_service.py b/tests/test_ai_orchestrator_service.py index 4611834..8dbdc62 100644 --- a/tests/test_ai_orchestrator_service.py +++ b/tests/test_ai_orchestrator_service.py @@ -1451,6 +1451,52 @@ def test_voice_v2_streaming_duplex_early_plan_returns_fast_safe_reply_without_ll assert "оператор" in decision["reply_text"].lower() +def test_voice_decision_hearing_check_keeps_active_topic_without_llm(monkeypatch): + def _unexpected_llm(messages): + raise AssertionError(f"LLM should not be called for hearing check: {messages!r}") + + monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm) + + decision = voice_module._voice_decision( + language="ru", + customer=None, + interaction=SimpleNamespace(interaction_id="int_voice_hearing", status="new", queue_id="que_voice", subject="schedule"), + transcript_text="Алло, ты меня слышишь?", + transcript_window=[ + SimpleNamespace(speaker="caller", text="Мне надо узнать график работы.", sequence_no=1, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()), + SimpleNamespace(speaker="assistant", text="Подскажите, какой именно график работы вас интересует?", sequence_no=2, source_type="voice_policy", barge_in_interrupted=False, created_at=utc_now_iso()), + SimpleNamespace(speaker="caller", text="В городе Алмата.", sequence_no=3, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()), + SimpleNamespace(speaker="caller", text="Алло, ты меня слышишь?", sequence_no=4, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()), + ], + kb_results=[], + disclosure_required=False, + request_metadata={"voice_v2_enabled": True, "response_plan_id": "rsp_hearing"}, + ) + + assert decision["model"] == "voice_policy_hearing_check" + assert decision["reply_text"].startswith("Да, вас слышу.") + assert "Как я могу помочь" not in decision["reply_text"] + assert "филиал" in decision["reply_text"] or "адрес" in decision["reply_text"] + + +def test_voice_postprocess_reply_rewrites_false_lookup_promise(): + reply_text = voice_module._voice_postprocess_reply_text( + language="ru", + transcript_text="В городе Алмата.", + transcript_window=[ + SimpleNamespace(speaker="caller", text="Мне надо узнать график работы.", sequence_no=1, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()), + SimpleNamespace(speaker="caller", text="В городе Алмата.", sequence_no=2, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()), + ], + reply_text="Спасибо. Я уточню график работы в Алмате. Минуточку, пожалуйста.", + kb_results=[], + needs_handoff=False, + ) + + assert "уточню" not in reply_text.lower() + assert "минуточ" not in reply_text.lower() + assert "филиал" in reply_text.lower() or "адрес" in reply_text.lower() + + def test_turn_voice_session_early_plan_does_not_persist_partial_turns(monkeypatch): monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_streaming_duplex") diff --git a/tests/test_ai_voice_media_runtime.py b/tests/test_ai_voice_media_runtime.py index 7a69f6b..370248c 100644 --- a/tests/test_ai_voice_media_runtime.py +++ b/tests/test_ai_voice_media_runtime.py @@ -842,7 +842,7 @@ def test_media_runtime_voice_v2_emits_generic_ack_before_full_asr_without_partia frame_ms=20, frame_bytes=320, ) - await runtime._process_utterance(actor, pcm_frame * 30, False) + await runtime._process_utterance(actor, pcm_frame * 45, False) asyncio.run(_scenario()) @@ -940,7 +940,7 @@ def test_media_runtime_voice_v2_emits_ack_for_short_utterance_after_reduced_thre frame_ms=20, frame_bytes=320, ) - await runtime._process_utterance(actor, pcm_frame * 20, False) + await runtime._process_utterance(actor, pcm_frame * 40, False) asyncio.run(_scenario()) @@ -949,6 +949,92 @@ def test_media_runtime_voice_v2_emits_ack_for_short_utterance_after_reduced_thre assert speak_events[0][1] < timings["full_finished"] +def test_media_runtime_ignores_low_signal_utterance_without_ack_or_turn(): + planned: list[tuple[str, str, str, dict | None]] = [] + delivered: list[tuple[str, str, bool]] = [] + turns: list[str] = [] + + class _LowSignalASRProvider(ASRProvider): + name = "low-signal-asr" + + def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription: + assert audio_bytes + return ASRTranscription(text="Ой", language=language_hint or "ru", confidence=0.82) + + runtime = AudioSocketMediaRuntime( + enabled=True, + host="127.0.0.1", + port=0, + frame_ms=20, + idle_timeout_seconds=2.0, + registration_wait_timeout_seconds=0.5, + min_speech_ms=40, + trailing_silence_ms=40, + max_turn_ms=2000, + asr_provider=_LowSignalASRProvider(), + tts_provider=_StubTTSProvider(), + load_registration_by_media_uuid=lambda value: None, + mark_media_connected=lambda session_id, value: None, + mark_media_ended=lambda session_id, reason: None, + touch_media_frame=lambda session_id: None, + set_state=lambda session_id, state, handoff_reason, metadata: None, + get_pending_greeting=lambda session_id: None, + mark_reply_delivered=lambda session_id, text, is_greeting: delivered.append((session_id, text, is_greeting)), + plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)), + process_turn=lambda session_id, transcript_text, language, barge_in, metadata: ( + turns.append(transcript_text) + or VoiceAITurnDecisionOut( + language=language or "ru", + intent="clarification", + reply_text="Подскажите подробнее.", + confidence=0.9, + needs_handoff=False, + handoff_reason=None, + case_action="keep_open", + kb_refs=[], + summary_text="reply ready", + model="stub-voice", + latency_ms=1, + status="active", + ) + ), + request_handoff=lambda session_id, customer_request_text, decision: None, + handle_media_error=lambda session_id, message, metadata: None, + ) + + async def _scenario() -> None: + actor = MediaActor( + registration=MediaRegistration( + voice_session_id="avs_media_runtime_low_signal", + call_id="call_media_runtime_low_signal", + interaction_id="int_media_runtime_low_signal", + ai_session_id="ais_media_runtime_low_signal", + language="ru", + media_uuid=str(uuid.uuid4()), + queue_code="voice_lab_ai", + queue_id="que_voice_lab_ai", + agent_profile="voice_support", + voice_v2_enabled=True, + voice_v2_ack_mode="immediate_short", + voice_v2_streaming_tts=True, + voice_v2_partial_asr=False, + ), + reader=asyncio.StreamReader(), + writer=None, # type: ignore[arg-type] + vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000), + frame_ms=20, + frame_bytes=320, + ) + pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160 + await runtime._process_utterance(actor, pcm_frame * 12, False) + + asyncio.run(_scenario()) + + assert turns == [] + assert planned == [] + assert delivered == [] + + def test_media_runtime_voice_v2_inserts_small_gap_between_ack_and_main_reply(): speak_events: list[tuple[str, float]] = []