diff --git a/.env.example b/.env.example index c16cabf..f2527a2 100644 --- a/.env.example +++ b/.env.example @@ -33,10 +33,14 @@ OLLAMA_LLM_NUM_CTX=1024 STT_PROVIDER=openai STT_FALLBACK_PROVIDER=elevenlabs -STT_NAME_CAPTURE_LANGUAGE_CODE=kz +# Empty means auto-detect the first answer, because the greeting asks for name and preferred language. +STT_NAME_CAPTURE_LANGUAGE_CODE= STT_PROMPT=Это русская речь по телефону. Точно распознавай короткие ответы, имена, фамилии, слова благодарности и прощания. Особое внимание именам: Айнур, Айгерим, Алия, Данияр, Азамат, Арман, Мадина, Нурсултан, Руслан, Асель. Если человек представился одним словом, сохраняй его как имя. Номера телефонов записывай цифрами. TTS_PROVIDER=elevenlabs +REALTIME_VOICE_INITIAL_GREETING_KK_TEXT=Сәлеметсіз бе! Менің атым Айнұр, мен ДиджиОпс компаниясының эй-ай ассистентімін. Сізге қалай жүгінсем болады және қай тілде сөйлескен ыңғайлы: қазақша ма, орысша ма? +REALTIME_VOICE_INITIAL_GREETING_RU_TEXT=Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться и на каком языке вам удобнее продолжить: на казахском или на русском? + ENABLE_AUDIO_DUMP=false AUDIO_DUMP_DIR=debug_audio @@ -68,7 +72,7 @@ VAD_SILENCE_TIMEOUT_MS=420 VAD_SPEECH_PAD_MS=180 VAD_MIN_SPEECH_DURATION_MS=0 VAD_USE_ONNX=false -REALTIME_VOICE_INITIAL_GREETING_TEXT=Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться? +REALTIME_VOICE_INITIAL_GREETING_TEXT= SEMANTIC_ENDPOINTING_HOLD_MS=600 REALTIME_VOICE_FILLER_DELAY_MS=300 REALTIME_VOICE_TTS_CHUNK_SOFT_MIN_CHARS=10 diff --git a/core/filler_audio.py b/core/filler_audio.py index a008eef..7a4215d 100644 --- a/core/filler_audio.py +++ b/core/filler_audio.py @@ -8,6 +8,7 @@ import os import random import wave from collections.abc import AsyncIterable +from collections.abc import Iterable from realtime_voice_service.providers.base import BaseTTS @@ -98,6 +99,18 @@ class FillerAudioLibrary: async def synthesize_text(self, tts: BaseTTS, text: str) -> bytes: return await self._synthesize_clip(tts, text) + async def synthesize_segments( + self, + tts: BaseTTS, + segments: Iterable[tuple[str, str | None]], + ) -> bytes: + clip = bytearray() + for text, language_code in segments: + segment = await self._synthesize_clip(tts, text, language_code=language_code) + if segment: + clip.extend(segment) + return bytes(clip) + def _load_from_files(self) -> None: raw_config = str(os.getenv(self._config_env, "")).strip() if not raw_config: @@ -149,12 +162,12 @@ class FillerAudioLibrary: if not self._clips.get(key): LOGGER.warning("no filler clips available for key=%s", key) - async def _synthesize_clip(self, tts: BaseTTS, text: str) -> bytes: + async def _synthesize_clip(self, tts: BaseTTS, text: str, *, language_code: str | None = None) -> bytes: async def one_shot_text_stream() -> AsyncIterable[str]: yield text clip = bytearray() - async for audio_chunk in tts.synthesize_stream(one_shot_text_stream()): + async for audio_chunk in tts.synthesize_stream(one_shot_text_stream(), language_code=language_code): if audio_chunk: clip.extend(audio_chunk) return bytes(clip) diff --git a/core/session.py b/core/session.py index 09ab7f9..eb7ffb1 100644 --- a/core/session.py +++ b/core/session.py @@ -247,12 +247,49 @@ def _preview_text(text: str, *, limit: int = 160) -> str: _KAZAKH_SPECIFIC_LETTERS = set("әғқңөұүһі") _RUSSIAN_SPECIFIC_LETTERS = set("ыэъё") +_KAZAKH_LANGUAGE_MARKERS = frozenset({ + "казахский", + "казахском", + "казахски", + "казакша", + "казак", + "қазақ", + "қазақша", +}) +_RUSSIAN_LANGUAGE_MARKERS = frozenset({ + "русский", + "русском", + "русски", + "орыс", + "орысша", +}) + + +def _detect_preferred_session_language(text: str) -> str | None: + normalized = _voice_text_key(text) + if not normalized: + return None + words = set(normalized.split()) + if words & _KAZAKH_LANGUAGE_MARKERS: + return "kk" + if words & _RUSSIAN_LANGUAGE_MARKERS: + return "ru" + if re.search(r"\bпо\s+русски\b", normalized): + return "ru" + if re.search(r"\bна\s+казахском\b", normalized): + return "kk" + if re.search(r"\bна\s+русском\b", normalized): + return "ru" + return None def _detect_session_language(text: str) -> str | None: normalized = str(text or "").strip().lower() if not normalized: return None + preferred_language = _detect_preferred_session_language(normalized) + if preferred_language: + return preferred_language has_kazakh = any(ch in _KAZAKH_SPECIFIC_LETTERS for ch in normalized) if has_kazakh: return "kk" @@ -605,6 +642,8 @@ def _sanitize_voice_text(text: str) -> str: DEFAULT_NAME_RETRY_TEXT = "Подскажите, пожалуйста, как я могу к вам обращаться?" DEFAULT_PERSONALIZED_GREETING_TEMPLATE = '{name}, я работаю на основе искусственного интеллекта и постараюсь максимально внимательно разобраться с вашим вопросом. Чем я могу помочь?' +KAZAKH_NAME_RETRY_TEXT = "Өтінемін, сізге қалай жүгінсем болады?" +KAZAKH_PERSONALIZED_GREETING_TEMPLATE = "{name}, мен жасанды интеллект негізінде жұмыс істеймін және сұрағыңызды мұқият түсінуге тырысамын. Қалай көмектесе аламын?" _NAME_CAPTURE_CONFUSION_ALIASES = { "парень": "Арнур", "арнер": "Арнур", @@ -693,7 +732,11 @@ def _normalize_name_candidate(text: str | None) -> str | None: "атым", "аты", "болады", + "язык", + "тіл", } + stop_words.update(_KAZAKH_LANGUAGE_MARKERS) + stop_words.update(_RUSSIAN_LANGUAGE_MARKERS) filtered = [word for word, lowered_word in zip(words, lowered) if lowered_word not in stop_words] if not filtered: return None @@ -756,6 +799,8 @@ def _normalize_name_candidate(text: str | None) -> str | None: "сұрақ", "көмек", } + invalid_tokens.update(_KAZAKH_LANGUAGE_MARKERS) + invalid_tokens.update(_RUSSIAN_LANGUAGE_MARKERS) invalid_letter_tokens = {_voice_letters_key(token) for token in invalid_tokens} filler_letters = {"а", "э", "е", "ё", "у", "о", "ы", "м", "m", "h"} for word in filtered: @@ -838,6 +883,8 @@ def _extract_name_candidate(text: str | None) -> tuple[str | None, bool]: "сұра", "көмек", } + cutoff_tokens.update(_KAZAKH_LANGUAGE_MARKERS) + cutoff_tokens.update(_RUSSIAN_LANGUAGE_MARKERS) for pattern in explicit_patterns: match = re.search(pattern, normalized, flags=re.IGNORECASE) if not match: @@ -1008,6 +1055,7 @@ class CallSession: tts: BaseTTS | None = None, filler_audio: FillerAudioLibrary | None = None, initial_greeting_text: str | None = None, + initial_greeting_segments: Iterable[tuple[str, str | None]] | None = None, ) -> None: self.session_id = session_id self.transport = transport @@ -1033,9 +1081,13 @@ class CallSession: self._sentence_queue: asyncio.Queue[str | None] | None = None self._closed = False self._filler_audio = filler_audio - self._initial_greeting_text = str(initial_greeting_text or "").strip() + self._initial_greeting_segments = self._normalize_initial_greeting_segments( + initial_greeting_segments, + fallback_text=initial_greeting_text, + ) + self._initial_greeting_text = _combine_user_transcripts(text for text, _ in self._initial_greeting_segments) self._initial_greeting_discarded_bytes = 0 - self._awaiting_customer_name = bool(self._initial_greeting_text) + self._awaiting_customer_name = bool(self._initial_greeting_segments) self._customer_name: str | None = None self._name_retry_text = DEFAULT_NAME_RETRY_TEXT self._personalized_greeting_template = DEFAULT_PERSONALIZED_GREETING_TEMPLATE @@ -1057,6 +1109,26 @@ class CallSession: self._semantic_hold_silence_timeout_ms, ) + @staticmethod + def _normalize_initial_greeting_segments( + segments: Iterable[tuple[str, str | None]] | None, + *, + fallback_text: str | None, + ) -> tuple[tuple[str, str | None], ...]: + normalized_segments: list[tuple[str, str | None]] = [] + for text, language_code in segments or (): + normalized_text = str(text or "").strip() + if not normalized_text: + continue + normalized_language = str(language_code or "").strip() or None + normalized_segments.append((normalized_text, normalized_language)) + if normalized_segments: + return tuple(normalized_segments) + normalized_fallback = str(fallback_text or "").strip() + if normalized_fallback: + return ((normalized_fallback, None),) + return () + @property def conversation(self) -> tuple[tuple[str, str], ...]: return tuple(self._conversation) @@ -1641,13 +1713,22 @@ class CallSession: raw = ( os.getenv("STT_NAME_CAPTURE_LANGUAGE_CODE", "").strip() or os.getenv("STT_NAME_CAPTURE_LANGUAGE", "").strip() - or "kz" ) return raw or None def _build_name_collection_response(self, transcript: str) -> str | None: if not self._awaiting_customer_name: return None + if self._session_language is None: + detected_language = _detect_session_language(transcript) + if detected_language: + self._session_language = detected_language + LOGGER.info( + "realtime session %s language locked during name collection: language=%s transcript=%r", + self.session_id, + detected_language, + _preview_text(transcript), + ) status, name_value = _voice_start_name_outcome(transcript) LOGGER.info( "realtime session %s name collection outcome: status=%s name=%r transcript=%r", @@ -1661,9 +1742,19 @@ class CallSession: if short_name: self._customer_name = short_name self._awaiting_customer_name = False - return self._personalized_greeting_template.format(name=short_name) + return self._personalized_greeting_text(short_name) + return self._name_retry_text_for_current_language() + + def _name_retry_text_for_current_language(self) -> str: + if self._session_language == "kk": + return KAZAKH_NAME_RETRY_TEXT return self._name_retry_text + def _personalized_greeting_text(self, name: str) -> str: + if self._session_language == "kk": + return KAZAKH_PERSONALIZED_GREETING_TEMPLATE.format(name=name) + return self._personalized_greeting_template.format(name=name) + def _build_llm_context(self) -> list[object]: context: list[object] = list(self._conversation) if self._customer_name: @@ -1719,7 +1810,7 @@ class CallSession: await playback_task def _start_initial_greeting(self) -> None: - if not self._initial_greeting_text or self._closed: + if not self._initial_greeting_segments or self._closed: return if self._greeting_task is not None and not self._greeting_task.done(): return @@ -1727,13 +1818,15 @@ class CallSession: self._play_initial_greeting( epoch=self.generation_epoch, greeting_text=self._initial_greeting_text, + greeting_segments=self._initial_greeting_segments, ), name=f"{self.session_id}-greeting-{self.generation_epoch}", ) LOGGER.info( - "realtime session %s initial greeting scheduled: epoch=%s chars=%s text=%r", + "realtime session %s initial greeting scheduled: epoch=%s segments=%s chars=%s text=%r", self.session_id, self.generation_epoch, + [(language, len(text)) for text, language in self._initial_greeting_segments], len(self._initial_greeting_text), _preview_text(self._initial_greeting_text), ) @@ -1743,10 +1836,8 @@ class CallSession: *, epoch: int, greeting_text: str, + greeting_segments: Iterable[tuple[str, str | None]], ) -> None: - async def one_shot_text_stream() -> AsyncGenerator[str, None]: - yield greeting_text - first_audio_seen = False started_monotonic = time.perf_counter() cached_greeting = self._filler_audio.pick("initial_greeting") if self._filler_audio is not None else None @@ -1761,22 +1852,37 @@ class CallSession: try: audio_chunk_count = 0 audio_byte_count = 0 - async for audio_chunk in self._tts.synthesize_stream(one_shot_text_stream()): - self._ensure_generation(epoch) - if not audio_chunk: - continue - audio_chunk_count += 1 - audio_byte_count += len(audio_chunk) - if not first_audio_seen: - first_audio_seen = True - self._log_latency( - "initial_greeting_ttfa", - started_monotonic, - epoch=epoch, - message="session start -> initial greeting first audio", - ) - self._set_state(SessionState.ASSISTANT_SPEAKING, reason="initial greeting started") - await self.transport.send_audio(audio_chunk) + for segment_text, language_code in greeting_segments: + async def one_shot_text_stream() -> AsyncGenerator[str, None]: + yield segment_text + + LOGGER.info( + "realtime session %s initial greeting segment start: epoch=%s language=%s chars=%s text=%r", + self.session_id, + epoch, + language_code, + len(segment_text), + _preview_text(segment_text), + ) + async for audio_chunk in self._tts.synthesize_stream( + one_shot_text_stream(), + language_code=language_code, + ): + self._ensure_generation(epoch) + if not audio_chunk: + continue + audio_chunk_count += 1 + audio_byte_count += len(audio_chunk) + if not first_audio_seen: + first_audio_seen = True + self._log_latency( + "initial_greeting_ttfa", + started_monotonic, + epoch=epoch, + message="session start -> initial greeting first audio", + ) + self._set_state(SessionState.ASSISTANT_SPEAKING, reason="initial greeting started") + await self.transport.send_audio(audio_chunk) await self.transport.flush_audio() self._ensure_generation(epoch) LOGGER.info( diff --git a/main.py b/main.py index 4d3af2e..8872b81 100644 --- a/main.py +++ b/main.py @@ -87,9 +87,43 @@ def _http_port() -> int: def _initial_greeting_text() -> str: raw = os.getenv("REALTIME_VOICE_INITIAL_GREETING_TEXT") - if raw is None: - return "Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться? Я работаю на основе искусственного интеллекта и постараюсь максимально внимательно разобраться с вашим вопросом. Расскажите, пожалуйста, чем могу помочь?" - return str(raw).strip() + if raw is not None and str(raw).strip(): + return str(raw).strip() + return " ".join(text for text, _ in _initial_greeting_segments()) + + +def _initial_greeting_segments() -> tuple[tuple[str, str | None], ...]: + raw = os.getenv("REALTIME_VOICE_INITIAL_GREETING_TEXT") + if raw is not None and str(raw).strip(): + language = str(os.getenv("REALTIME_VOICE_INITIAL_GREETING_LANGUAGE_CODE", "")).strip() or None + text = str(raw).strip() + return ((text, language),) + + kk_text = str( + os.getenv( + "REALTIME_VOICE_INITIAL_GREETING_KK_TEXT", + ( + "Сәлеметсіз бе! Менің атым Айнұр, мен ДиджиОпс компаниясының эй-ай ассистентімін. " + "Сізге қалай жүгінсем болады және қай тілде сөйлескен ыңғайлы: қазақша ма, орысша ма?" + ), + ) + ).strip() + ru_text = str( + os.getenv( + "REALTIME_VOICE_INITIAL_GREETING_RU_TEXT", + ( + "Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. " + "Как я могу к вам обращаться и на каком языке вам удобнее продолжить: на казахском или на русском?" + ), + ) + ).strip() + segments: list[tuple[str, str | None]] = [] + if kk_text: + segments.append((kk_text, "kk")) + if ru_text: + segments.append((ru_text, "ru")) + return tuple(segments) + def _llm_provider_name() -> str: return str(os.getenv("LLM_PROVIDER") or os.getenv("REALTIME_VOICE_LLM_PROVIDER") or "openai").strip().lower() @@ -167,17 +201,19 @@ class RealtimeVoiceService: LOGGER.exception("failed to preload filler audio clips") async def _preload_initial_greeting_audio(self) -> None: - greeting_text = _initial_greeting_text() - if not greeting_text: + greeting_segments = _initial_greeting_segments() + greeting_text = " ".join(text for text, _ in greeting_segments).strip() + if not greeting_segments or not greeting_text: LOGGER.info("initial greeting audio preload skipped: empty text") return try: LOGGER.info( - "initial greeting audio preload start: chars=%s text=%r", + "initial greeting audio preload start: segments=%s chars=%s text=%r", + [(language, len(text)) for text, language in greeting_segments], len(greeting_text), greeting_text[:120], ) - clip = await self._filler_audio.synthesize_text(self._tts, greeting_text) + clip = await self._filler_audio.synthesize_segments(self._tts, greeting_segments) if not clip: LOGGER.warning("initial greeting audio preload produced empty clip") return @@ -240,6 +276,7 @@ class RealtimeVoiceService: tts=self._tts, filler_audio=self._filler_audio, initial_greeting_text=_initial_greeting_text(), + initial_greeting_segments=_initial_greeting_segments(), ) @staticmethod