feat: improve phone speech and reduce response gaps

2026-04-30 11:08:47 +05:00
parent cd1fde5175
commit fd89daf7a4
3 changed files with 108 additions and 11 deletions
@@ -9,7 +9,7 @@ LLM_PROVIDER=openai
 OPENAI_API_KEY=
 OPENAI_BASE_URL=
 OPENAI_LLM_MODEL=gpt-4o-mini
-OPENAI_LLM_SYSTEM_PROMPT=You are a concise voice assistant for the DigiOps contact center. Answer clearly and briefly.
+OPENAI_LLM_SYSTEM_PROMPT=You are Ainur, a concise Russian-speaking voice agent for the DigiOps contact center. Always answer in Russian, naturally and briefly for voice playback. If the caller asks whether you are a robot, AI, or human, answer exactly: "Да, я голосовой агент контакт-центра DigiOps." Read phone numbers digit by digit. Prefer clear stress-friendly wording. Do not use markdown, URLs, or tables.
 OPENAI_TIMEOUT_SECONDS=30
 OPENAI_STT_MODEL=whisper-1
 OPENAI_STT_LANGUAGE=ru
@@ -60,3 +60,6 @@ VAD_MIN_SPEECH_DURATION_MS=0
 VAD_USE_ONNX=false
 REALTIME_VOICE_INITIAL_GREETING_TEXT=Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться?
 SEMANTIC_ENDPOINTING_HOLD_MS=600
+REALTIME_VOICE_FILLER_DELAY_MS=250
+REALTIME_VOICE_TTS_CHUNK_SOFT_MIN_CHARS=10
+REALTIME_VOICE_TTS_CHUNK_SOFT_MIN_WORDS=2
@@ -16,12 +16,12 @@ LOGGER = logging.getLogger("uvicorn.error")

 DEFAULT_FILLER_TEXTS: dict[str, tuple[str, ...]] = {
    "serper": (
-        "Одну минуту.",
-        "Минуточку, проверяю данные.",
+        "Сейчас уточню.",
+        "Минуту, пожалуйста.",
    ),
    "generic": (
-        "Одну минуту.",
-        "Секундочку.",
+        "Секунду.",
+        "Минуту.",
    ),
 }

@@ -21,16 +21,37 @@ from realtime_voice_service.transports.base import BaseMediaTransport


 LOGGER = logging.getLogger("uvicorn.error")
-def _semantic_hold_ms() -> int:
-    raw = str(os.getenv("SEMANTIC_ENDPOINTING_HOLD_MS", "700")).strip()
+
+
+def _read_bounded_int_env(name: str, default: int, *, minimum: int, maximum: int) -> int:
+    raw = str(os.getenv(name, str(default))).strip()
    try:
        value = int(float(raw))
    except Exception:
-        value = 700
-    return max(200, min(value, 2000))
+        value = default
+    return max(minimum, min(value, maximum))
+
+
+def _semantic_hold_ms() -> int:
+    return _read_bounded_int_env("SEMANTIC_ENDPOINTING_HOLD_MS", 700, minimum=200, maximum=2000)
+
+
+def _filler_delay_ms() -> int:
+    return _read_bounded_int_env("REALTIME_VOICE_FILLER_DELAY_MS", 250, minimum=0, maximum=3000)
+
+
+def _tts_chunk_soft_min_chars() -> int:
+    return _read_bounded_int_env("REALTIME_VOICE_TTS_CHUNK_SOFT_MIN_CHARS", 10, minimum=1, maximum=80)
+
+
+def _tts_chunk_soft_min_words() -> int:
+    return _read_bounded_int_env("REALTIME_VOICE_TTS_CHUNK_SOFT_MIN_WORDS", 2, minimum=1, maximum=10)


 SEMANTIC_ENDPOINTING_HOLD_MS = _semantic_hold_ms()
+FILLER_AUDIO_DELAY_MS = _filler_delay_ms()
+TTS_CHUNK_SOFT_MIN_CHARS = _tts_chunk_soft_min_chars()
+TTS_CHUNK_SOFT_MIN_WORDS = _tts_chunk_soft_min_words()
 SEMANTIC_CONTINUATION_TOKENS = {
    "а",
    "в",
@@ -282,8 +303,39 @@ def _ru_date_words(day_raw: str, month_raw: str, year_raw: str) -> str:
    return f"{day_words} {month_words} {_ru_number_words(str(year))}"


+VOICE_PRONUNCIATION_REPLACEMENTS: tuple[tuple[str, str], ...] = (
+    (r"\bDigiOps\b", "ДиджиОпс"),
+    (r"\bA\.?I\.?\b", "эй-ай"),
+    (r"\bAI\b", "эй-ай"),
+    (r"\bCRM\b", "си-ар-эм"),
+    (r"\bB2B\b", "би-ту-би"),
+    (r"\bIT\b", "ай-ти"),
+)
+
+
+def _normalize_voice_pronunciation(text: str) -> str:
+    normalized = str(text or "")
+    for pattern, replacement in VOICE_PRONUNCIATION_REPLACEMENTS:
+        normalized = re.sub(pattern, replacement, normalized, flags=re.IGNORECASE)
+    return normalized
+
+
+def _ru_phone_words(raw: str) -> str:
+    payload = str(raw or "")
+    digits = re.sub(r"\D+", "", payload)
+    if len(digits) < 7:
+        return _ru_number_words(payload)
+    prefix = "плюс " if payload.strip().startswith("+") else ""
+    return f"{prefix}{_ru_digit_sequence(digits)}".strip()
+
+
 def _normalize_voice_numbers(text: str) -> str:
    normalized = str(text or "")
+    normalized = re.sub(
+        r"(?<!\w)(\+?\d(?:[\s()\-]*\d){6,})(?!\w)",
+        lambda match: _ru_phone_words(match.group(1)),
+        normalized,
+    )
    normalized = re.sub(
        r"(?<!\w)(\d{1,2})[.](\d{1,2})[.](\d{2,4})(?!\w)",
        lambda match: _ru_date_words(match.group(1), match.group(2), match.group(3)),
@@ -340,6 +392,7 @@ def _sanitize_voice_text(text: str) -> str:
    sanitized = re.sub(r"\[([^\]]+)\]\((?:https?://|www\.)[^)\s]+[^)]*\)", r"\1", sanitized)
    sanitized = re.sub(r"(?:https?://|www\.)\S+", "", sanitized)
    sanitized = sanitized.replace("[", "").replace("]", "").replace("(", "").replace(")", "")
+    sanitized = _normalize_voice_pronunciation(sanitized)
    sanitized = re.sub(r"\s+", " ", sanitized).strip()
    sanitized = _normalize_voice_numbers(sanitized)
    sanitized = re.sub(r"\s+", " ", sanitized).strip()
@@ -628,8 +681,8 @@ class TextChunker:
    def __init__(
        self,
        *,
-        soft_min_chars: int = 18,
-        soft_min_words: int = 3,
+        soft_min_chars: int = TTS_CHUNK_SOFT_MIN_CHARS,
+        soft_min_words: int = TTS_CHUNK_SOFT_MIN_WORDS,
    ) -> None:
        self._buffer = ""
        self._soft_min_chars = max(soft_min_chars, 1)
@@ -1008,6 +1061,14 @@ class CallSession:
            llm_started_monotonic = time.perf_counter()
            chunker = TextChunker()
            first_token_seen = False
+            filler_trigger_task = asyncio.create_task(
+                self._trigger_filler_audio_after_delay(
+                    epoch=epoch,
+                    tool_name=None,
+                    started_monotonic=llm_started_monotonic,
+                ),
+                name=f"{self.session_id}-filler-delay-{epoch}",
+            )
            async for event in self._llm.generate_stream(transcript, self._build_llm_context()):
                self._ensure_generation(epoch)
                if event.type == "tool_call_start":
@@ -1019,6 +1080,8 @@ class CallSession:
                        event.tool_call_id,
                        filler_started,
                    )
+                    if not filler_trigger_task.done():
+                        filler_trigger_task.cancel()
                    if not filler_started:
                        filler_started = True
                        self._start_filler_audio(
@@ -1033,6 +1096,8 @@ class CallSession:
                    continue
                if not first_token_seen:
                    first_token_seen = True
+                    if not filler_trigger_task.done():
+                        filler_trigger_task.cancel()
                    self._log_latency(
                        "ttft",
                        llm_started_monotonic,
@@ -1120,6 +1185,11 @@ class CallSession:
            if epoch == self.generation_epoch and not self._closed:
                self._set_state(SessionState.LISTENING, reason="assistant generation failed")
        finally:
+            filler_trigger_task = locals().get("filler_trigger_task")
+            if filler_trigger_task is not None and not filler_trigger_task.done():
+                filler_trigger_task.cancel()
+                with contextlib.suppress(asyncio.CancelledError):
+                    await filler_trigger_task
            if stt_stream is not None:
                with contextlib.suppress(Exception):
                    await stt_stream.cancel()
@@ -1542,6 +1612,30 @@ class CallSession:
        with contextlib.suppress(asyncio.CancelledError):
            await filler_task

+    async def _trigger_filler_audio_after_delay(
+        self,
+        *,
+        epoch: int,
+        tool_name: str | None,
+        started_monotonic: float,
+    ) -> None:
+        if FILLER_AUDIO_DELAY_MS <= 0:
+            self._start_filler_audio(
+                epoch=epoch,
+                tool_name=tool_name,
+                started_monotonic=started_monotonic,
+            )
+            return
+        await asyncio.sleep(FILLER_AUDIO_DELAY_MS / 1000.0)
+        self._ensure_generation(epoch)
+        if self._active_answer_audio_started:
+            return
+        self._start_filler_audio(
+            epoch=epoch,
+            tool_name=tool_name,
+            started_monotonic=started_monotonic,
+        )
+
    def _start_filler_audio(
        self,
        *,