digiops/call-center
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(voice): suppress low-signal turns and false lookup replies

Browse Source
This commit is contained in:
Yera All
2026-04-12 20:40:25 +05:00
parent 18ab5aa2e2
commit 57bd5cad4b
5 changed files with 323 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/ai_orchestrator_service/voice.py
+140 -4
View File
@@ -806,6 +806,7 @@ def _voice_is_low_signal_caller_text(text: str | None) -> bool:
"ладно",
"неа",
"нет",
"ой",
"ок",
"понял",
"поняла",
@@ -819,6 +820,21 @@ def _voice_is_low_signal_caller_text(text: str | None) -> bool:
return normalized in low_signal_phrases
def _voice_is_hearing_check_caller_text(text: str | None) -> bool:
normalized = _voice_text_key(text)
if not normalized:
return False
markers = (
"алло ты меня слышишь",
"вы меня слышите",
"меня слышно",
"меня слышишь",
"ты меня слышишь",
"слышишь меня",
)
return any(marker in normalized for marker in markers)
def _voice_is_confused_caller_text(text: str | None) -> bool:
normalized = _voice_text_key(text)
confusion_markers = (
@@ -878,6 +894,25 @@ def _voice_topic_prompt(language: str, caller_texts: list[str]) -> str | None:
if not context:
return None
if any(marker in context for marker in ("график", "время работы", "режим работы", "часы работы", "работаете")):
city_scope = any(
marker in context
for marker in (
"алмат",
"астан",
"в городе",
"город",
"караганд",
"кызылорд",
"павлодар",
"семей",
"тараз",
"шымкент",
)
)
if city_scope:
if language == "kz":
return "Osy qaladagy qaysy filial nemese mekenjai qyzyqtyratynyn aitnyz."
return "Подскажите, какой филиал или адрес в этом городе вас интересует?"
if language == "kz":
return "Qai filialdyn, mekendyng nemese qalanyng jumys uaqyty qyzyqtyratynyn aitnyz."
return "Подскажите, график работы какого филиала, адреса или города вас интересует?"
@@ -1005,11 +1040,11 @@ def _voice_confusion_prompt(language: str, caller_texts: list[str]) -> str:
topic_prompt = _voice_topic_prompt(language, caller_texts)
if topic_prompt:
if language == "kz":
return f"Qongyrau taqyrybyn naqtylap jatyrmyn. {topic_prompt}"
return f"Сейчас уточняю цель звонка. {topic_prompt}"
return f"Naqtylap alaiyn. {topic_prompt}"
return f"Подскажите точнее. {topic_prompt}"
if language == "kz":
return "Qazir qongyraudyn maqratyn anyqtap jatyrmyn. Qysqasha aitnyz: jumys uaqyty, otinish statusy, tarif nemese operator."
return "Сейчас уточняю цель звонка. Скажите коротко, что именно нужно: график работы, статус заявки, тариф или оператор."
return "Naqtylap alaiyn: jumys uaqyty, otinish statusy, tarif nemese operator degenniń birin aitnyz."
return "Подскажите точнее: график работы, статус заявки, тариф или оператор."
def _voice_loop_handoff(language: str) -> tuple[str, str, str]:
@@ -1212,6 +1247,79 @@ def _voice_compact_reply_text(text: str, *, language: str) -> str:
return f"{shortened}{ending}"
def _voice_caller_context_before_current(caller_texts: list[str], transcript_text: str) -> list[str]:
if not caller_texts:
return []
current_key = _voice_text_key(transcript_text)
if current_key and _voice_text_key(caller_texts[-1]) == current_key:
return caller_texts[:-1]
return caller_texts
def _voice_is_midcall_greeting_reply(text: str | None) -> bool:
normalized = _voice_text_key(text)
if not normalized:
return False
greeting_markers = ("здравствуйте", "сәлеметсіз", "сәлем")
followup_markers = ("чем помочь", "как я могу помочь", "коротко расскажите", "қалай көмектесе")
return any(marker in normalized for marker in greeting_markers) and any(
marker in normalized for marker in followup_markers
)
def _voice_contains_false_lookup_promise(text: str | None) -> bool:
normalized = _voice_text_key(text)
if not normalized:
return False
markers = (
"сейчас уточню",
"я уточню",
"уточню",
"минуточ",
"подождите",
"подождите пожалуйста",
"проверю",
"сейчас проверю",
"я проверю",
"посмотрю",
"сейчас посмотрю",
)
return any(marker in normalized for marker in markers)
def _voice_hearing_check_reply(language: str, caller_texts: list[str]) -> str:
followup = _voice_topic_prompt(language, caller_texts) or _voice_generic_prompt(language)
if language == "kz":
return f"Ia, estip turmyn. {followup}"
return f"Да, вас слышу. {followup}"
def _voice_postprocess_reply_text(
*,
language: str,
transcript_text: str,
transcript_window: list[VoiceTranscriptSegmentRow],
reply_text: str,
kb_results: list[Any],
needs_handoff: bool,
) -> str:
normalized_reply = str(reply_text or "").strip()
if not normalized_reply or needs_handoff:
return normalized_reply
caller_texts = _voice_recent_caller_texts(transcript_window)
prior_caller_texts = _voice_caller_context_before_current(caller_texts, transcript_text)
if _voice_is_hearing_check_caller_text(transcript_text):
return _voice_hearing_check_reply(language, prior_caller_texts or caller_texts)
if _voice_contains_false_lookup_promise(normalized_reply) and not kb_results:
return _voice_topic_prompt(language, caller_texts) or _voice_generic_prompt(language)
if _voice_is_midcall_greeting_reply(normalized_reply) and any(
segment.speaker == "assistant" for segment in transcript_window
):
context_texts = prior_caller_texts if _voice_is_low_signal_caller_text(transcript_text) else caller_texts
return _voice_topic_prompt(language, context_texts) or _voice_generic_prompt(language)
return normalized_reply
def _voice_v2_metadata(
transcript_text: str,
request_metadata: dict[str, Any] | None = None,
@@ -1653,6 +1761,26 @@ def _voice_decision(
"latency_ms": 1,
}
if _voice_is_hearing_check_caller_text(normalized):
prior_caller_texts = _voice_caller_context_before_current(caller_texts, transcript_text)
decision = {
"language": language,
"intent": "clarification",
"reply_text": _voice_hearing_check_reply(language, prior_caller_texts or caller_texts),
"confidence": 0.66,
"needs_handoff": False,
"handoff_reason": None,
"case_action": "keep_open",
"kb_refs": [],
"summary_text": "AI подтвердил, что слышит клиента, и продолжил активный сценарий без сброса диалога.",
"model": "voice_policy_hearing_check",
"latency_ms": 1,
}
if v2_metadata:
decision["reply_text"] = _voice_compact_reply_text(decision["reply_text"], language=language)
decision["metadata"] = {**v2_metadata, "reply_phase": "final"}
return decision
clarification_count = _voice_recent_clarification_count(transcript_window)
repeated_reply_count = _voice_repeated_assistant_reply_count(transcript_window)
recent_caller_window = caller_texts[-3:]
@@ -2283,6 +2411,14 @@ def turn_voice_session(session_id: str, payload: VoiceAITurnIn) -> VoiceAITurnDe
customer_name_status=effective_name_status,
request_metadata=request_metadata,
)
decision["reply_text"] = _voice_postprocess_reply_text(
language=decision["language"],
transcript_text=payload.transcript_text,
transcript_window=transcript_window,
reply_text=str(decision.get("reply_text") or ""),
kb_results=kb_results,
needs_handoff=bool(decision.get("needs_handoff")),
)
if decision.get("extracted_name") and not early_plan_only:
effective_name_status = "name_obtained"
services/ai_voice_runtime_service/media_runtime.py
+36 -5
View File
@@ -174,7 +174,7 @@ class AudioSocketMediaRuntime:
self._actors: dict[str, MediaActor] = {}
self._v2_ack_wait_seconds = 0.18
self._partial_asr_min_ms = 320
self._immediate_ack_min_ms = 280
self._immediate_ack_min_ms = 700
self._v2_ack_post_gap_seconds = 0.10
self._partial_poll_interval_seconds = 0.20
self._stable_partial_hold_seconds = 0.40
@@ -184,6 +184,32 @@ class AudioSocketMediaRuntime:
def _normalize_intent_text(text: str) -> str:
return " ".join(str(text or "").strip().lower().split())
@classmethod
def _is_low_signal_transcript(cls, text: str | None) -> bool:
normalized = cls._normalize_intent_text(text)
if not normalized:
return True
return normalized in {
"ага",
"алло",
"да",
"добрый день",
"здравствуйте",
"ладно",
"неа",
"нет",
"ой",
"ок",
"понял",
"поняла",
"привет",
"слышу",
"слышно",
"угу",
"хорошо",
"ясно",
}
def _detect_early_intent(self, text: str) -> str:
normalized = self._normalize_intent_text(text)
if not normalized:
@@ -363,8 +389,6 @@ class AudioSocketMediaRuntime:
)
except StreamingASRUnavailable as exc:
actor.asr_streaming_enabled = False
actor.registration.voice_v2_duplex = False
actor.registration.voice_v2_partial_asr = False
logger.warning(
"audiosocket.streaming_asr_unavailable session_id=%s backend=%s error=%s",
actor.registration.voice_session_id,
@@ -913,8 +937,6 @@ class AudioSocketMediaRuntime:
str(exc)[:500],
)
await self._close_streaming_asr(actor)
actor.registration.voice_v2_duplex = False
actor.registration.voice_v2_partial_asr = False
elif actor.registration.voice_v2_partial_asr:
self._maybe_schedule_partial_asr(actor)
if vad_result.utterance_pcm:
@@ -1008,6 +1030,15 @@ class AudioSocketMediaRuntime:
if not transcript_text:
await self._set_actor_state(actor, "listening")
return
if self._is_low_signal_transcript(transcript_text):
actor.finalized_utterance_generation = max(actor.finalized_utterance_generation, utterance_generation)
logger.warning(
"audiosocket.low_signal_ignored session_id=%s transcript=%s",
actor.registration.voice_session_id,
transcript_text[:120],
)
await self._set_actor_state(actor, "listening")
return
actor.finalized_utterance_generation = max(actor.finalized_utterance_generation, utterance_generation)
actor.partial_transcript = transcript_text if actor.registration.voice_v2_partial_asr else None
services/streaming_asr_sidecar_service/app.py
+13
View File
@@ -236,6 +236,19 @@ def _get_engine() -> TranscriptionEngine:
return _ENGINE_INSTANCE
def _warmup_engine() -> None:
try:
_get_engine()
LOGGER.info("streaming_asr.engine_ready")
except Exception as exc: # noqa: BLE001
LOGGER.warning("streaming_asr.engine_warmup_failed error=%s", str(exc)[:500])
@app.on_event("startup")
def _startup_warmup_engine() -> None:
threading.Thread(target=_warmup_engine, daemon=True).start()
@dataclass(slots=True)
class StreamState:
stream_id: str
tests/test_ai_orchestrator_service.py
+46
View File
@@ -1451,6 +1451,52 @@ def test_voice_v2_streaming_duplex_early_plan_returns_fast_safe_reply_without_ll
assert "оператор" in decision["reply_text"].lower()
def test_voice_decision_hearing_check_keeps_active_topic_without_llm(monkeypatch):
def _unexpected_llm(messages):
raise AssertionError(f"LLM should not be called for hearing check: {messages!r}")
monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm)
decision = voice_module._voice_decision(
language="ru",
customer=None,
interaction=SimpleNamespace(interaction_id="int_voice_hearing", status="new", queue_id="que_voice", subject="schedule"),
transcript_text="Алло, ты меня слышишь?",
transcript_window=[
SimpleNamespace(speaker="caller", text="Мне надо узнать график работы.", sequence_no=1, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()),
SimpleNamespace(speaker="assistant", text="Подскажите, какой именно график работы вас интересует?", sequence_no=2, source_type="voice_policy", barge_in_interrupted=False, created_at=utc_now_iso()),
SimpleNamespace(speaker="caller", text="В городе Алмата.", sequence_no=3, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()),
SimpleNamespace(speaker="caller", text="Алло, ты меня слышишь?", sequence_no=4, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()),
],
kb_results=[],
disclosure_required=False,
request_metadata={"voice_v2_enabled": True, "response_plan_id": "rsp_hearing"},
)
assert decision["model"] == "voice_policy_hearing_check"
assert decision["reply_text"].startswith("Да, вас слышу.")
assert "Как я могу помочь" not in decision["reply_text"]
assert "филиал" in decision["reply_text"] or "адрес" in decision["reply_text"]
def test_voice_postprocess_reply_rewrites_false_lookup_promise():
reply_text = voice_module._voice_postprocess_reply_text(
language="ru",
transcript_text="В городе Алмата.",
transcript_window=[
SimpleNamespace(speaker="caller", text="Мне надо узнать график работы.", sequence_no=1, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()),
SimpleNamespace(speaker="caller", text="В городе Алмата.", sequence_no=2, source_type="voice_asr", barge_in_interrupted=False, created_at=utc_now_iso()),
],
reply_text="Спасибо. Я уточню график работы в Алмате. Минуточку, пожалуйста.",
kb_results=[],
needs_handoff=False,
)
assert "уточню" not in reply_text.lower()
assert "минуточ" not in reply_text.lower()
assert "филиал" in reply_text.lower() or "адрес" in reply_text.lower()
def test_turn_voice_session_early_plan_does_not_persist_partial_turns(monkeypatch):
monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_streaming_duplex")
tests/test_ai_voice_media_runtime.py
+88 -2
View File
@@ -842,7 +842,7 @@ def test_media_runtime_voice_v2_emits_generic_ack_before_full_asr_without_partia
frame_ms=20,
frame_bytes=320,
)
await runtime._process_utterance(actor, pcm_frame * 30, False)
await runtime._process_utterance(actor, pcm_frame * 45, False)
asyncio.run(_scenario())
@@ -940,7 +940,7 @@ def test_media_runtime_voice_v2_emits_ack_for_short_utterance_after_reduced_thre
frame_ms=20,
frame_bytes=320,
)
await runtime._process_utterance(actor, pcm_frame * 20, False)
await runtime._process_utterance(actor, pcm_frame * 40, False)
asyncio.run(_scenario())
@@ -949,6 +949,92 @@ def test_media_runtime_voice_v2_emits_ack_for_short_utterance_after_reduced_thre
assert speak_events[0][1] < timings["full_finished"]
def test_media_runtime_ignores_low_signal_utterance_without_ack_or_turn():
planned: list[tuple[str, str, str, dict | None]] = []
delivered: list[tuple[str, str, bool]] = []
turns: list[str] = []
class _LowSignalASRProvider(ASRProvider):
name = "low-signal-asr"
def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
assert audio_bytes
return ASRTranscription(text="Ой", language=language_hint or "ru", confidence=0.82)
runtime = AudioSocketMediaRuntime(
enabled=True,
host="127.0.0.1",
port=0,
frame_ms=20,
idle_timeout_seconds=2.0,
registration_wait_timeout_seconds=0.5,
min_speech_ms=40,
trailing_silence_ms=40,
max_turn_ms=2000,
asr_provider=_LowSignalASRProvider(),
tts_provider=_StubTTSProvider(),
load_registration_by_media_uuid=lambda value: None,
mark_media_connected=lambda session_id, value: None,
mark_media_ended=lambda session_id, reason: None,
touch_media_frame=lambda session_id: None,
set_state=lambda session_id, state, handoff_reason, metadata: None,
get_pending_greeting=lambda session_id: None,
mark_reply_delivered=lambda session_id, text, is_greeting: delivered.append((session_id, text, is_greeting)),
plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)),
process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
turns.append(transcript_text)
or VoiceAITurnDecisionOut(
language=language or "ru",
intent="clarification",
reply_text="Подскажите подробнее.",
confidence=0.9,
needs_handoff=False,
handoff_reason=None,
case_action="keep_open",
kb_refs=[],
summary_text="reply ready",
model="stub-voice",
latency_ms=1,
status="active",
)
),
request_handoff=lambda session_id, customer_request_text, decision: None,
handle_media_error=lambda session_id, message, metadata: None,
)
async def _scenario() -> None:
actor = MediaActor(
registration=MediaRegistration(
voice_session_id="avs_media_runtime_low_signal",
call_id="call_media_runtime_low_signal",
interaction_id="int_media_runtime_low_signal",
ai_session_id="ais_media_runtime_low_signal",
language="ru",
media_uuid=str(uuid.uuid4()),
queue_code="voice_lab_ai",
queue_id="que_voice_lab_ai",
agent_profile="voice_support",
voice_v2_enabled=True,
voice_v2_ack_mode="immediate_short",
voice_v2_streaming_tts=True,
voice_v2_partial_asr=False,
),
reader=asyncio.StreamReader(),
writer=None, # type: ignore[arg-type]
vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
frame_ms=20,
frame_bytes=320,
)
pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
await runtime._process_utterance(actor, pcm_frame * 12, False)
asyncio.run(_scenario())
assert turns == []
assert planned == []
assert delivered == []
def test_media_runtime_voice_v2_inserts_small_gap_between_ack_and_main_reply():
speak_events: list[tuple[str, float]] = []