.

.env.example

@@ -33,10 +33,14 @@ OLLAMA_LLM_NUM_CTX=1024
 
 STT_PROVIDER=openai
 STT_FALLBACK_PROVIDER=elevenlabs
-STT_NAME_CAPTURE_LANGUAGE_CODE=kz
+# Empty means auto-detect the first answer, because the greeting asks for name and preferred language.
+STT_NAME_CAPTURE_LANGUAGE_CODE=
 STT_PROMPT=Это русская речь по телефону. Точно распознавай короткие ответы, имена, фамилии, слова благодарности и прощания. Особое внимание именам: Айнур, Айгерим, Алия, Данияр, Азамат, Арман, Мадина, Нурсултан, Руслан, Асель. Если человек представился одним словом, сохраняй его как имя. Номера телефонов записывай цифрами.
 TTS_PROVIDER=elevenlabs
 
+REALTIME_VOICE_INITIAL_GREETING_KK_TEXT=Сәлеметсіз бе! Менің атым Айнұр, мен ДиджиОпс компаниясының эй-ай ассистентімін. Сізге қалай жүгінсем болады және қай тілде сөйлескен ыңғайлы: қазақша ма, орысша ма?
+REALTIME_VOICE_INITIAL_GREETING_RU_TEXT=Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться и на каком языке вам удобнее продолжить: на казахском или на русском?
+
 ENABLE_AUDIO_DUMP=false
 AUDIO_DUMP_DIR=debug_audio
 
@@ -68,7 +72,7 @@ VAD_SILENCE_TIMEOUT_MS=420
 VAD_SPEECH_PAD_MS=180
 VAD_MIN_SPEECH_DURATION_MS=0
 VAD_USE_ONNX=false
-REALTIME_VOICE_INITIAL_GREETING_TEXT=Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться?
+REALTIME_VOICE_INITIAL_GREETING_TEXT=
 SEMANTIC_ENDPOINTING_HOLD_MS=600
 REALTIME_VOICE_FILLER_DELAY_MS=300
 REALTIME_VOICE_TTS_CHUNK_SOFT_MIN_CHARS=10

core/filler_audio.py

@@ -8,6 +8,7 @@ import os
 import random
 import wave
 from collections.abc import AsyncIterable
+from collections.abc import Iterable
 
 from realtime_voice_service.providers.base import BaseTTS
 
@@ -98,6 +99,18 @@ class FillerAudioLibrary:
     async def synthesize_text(self, tts: BaseTTS, text: str) -> bytes:
         return await self._synthesize_clip(tts, text)
 
+    async def synthesize_segments(
+        self,
+        tts: BaseTTS,
+        segments: Iterable[tuple[str, str | None]],
+    ) -> bytes:
+        clip = bytearray()
+        for text, language_code in segments:
+            segment = await self._synthesize_clip(tts, text, language_code=language_code)
+            if segment:
+                clip.extend(segment)
+        return bytes(clip)
+
     def _load_from_files(self) -> None:
         raw_config = str(os.getenv(self._config_env, "")).strip()
         if not raw_config:
@@ -149,12 +162,12 @@ class FillerAudioLibrary:
             if not self._clips.get(key):
                 LOGGER.warning("no filler clips available for key=%s", key)
 
-    async def _synthesize_clip(self, tts: BaseTTS, text: str) -> bytes:
+    async def _synthesize_clip(self, tts: BaseTTS, text: str, *, language_code: str | None = None) -> bytes:
         async def one_shot_text_stream() -> AsyncIterable[str]:
             yield text
 
         clip = bytearray()
-        async for audio_chunk in tts.synthesize_stream(one_shot_text_stream()):
+        async for audio_chunk in tts.synthesize_stream(one_shot_text_stream(), language_code=language_code):
             if audio_chunk:
                 clip.extend(audio_chunk)
         return bytes(clip)

core/session.py

@@ -247,12 +247,49 @@ def _preview_text(text: str, *, limit: int = 160) -> str:
 
 _KAZAKH_SPECIFIC_LETTERS = set("әғқңөұүһі")
 _RUSSIAN_SPECIFIC_LETTERS = set("ыэъё")
+_KAZAKH_LANGUAGE_MARKERS = frozenset({
+    "казахский",
+    "казахском",
+    "казахски",
+    "казакша",
+    "казак",
+    "қазақ",
+    "қазақша",
+})
+_RUSSIAN_LANGUAGE_MARKERS = frozenset({
+    "русский",
+    "русском",
+    "русски",
+    "орыс",
+    "орысша",
+})
+
+
+def _detect_preferred_session_language(text: str) -> str | None:
+    normalized = _voice_text_key(text)
+    if not normalized:
+        return None
+    words = set(normalized.split())
+    if words & _KAZAKH_LANGUAGE_MARKERS:
+        return "kk"
+    if words & _RUSSIAN_LANGUAGE_MARKERS:
+        return "ru"
+    if re.search(r"\bпо\s+русски\b", normalized):
+        return "ru"
+    if re.search(r"\bна\s+казахском\b", normalized):
+        return "kk"
+    if re.search(r"\bна\s+русском\b", normalized):
+        return "ru"
+    return None
 
 
 def _detect_session_language(text: str) -> str | None:
     normalized = str(text or "").strip().lower()
     if not normalized:
         return None
+    preferred_language = _detect_preferred_session_language(normalized)
+    if preferred_language:
+        return preferred_language
     has_kazakh = any(ch in _KAZAKH_SPECIFIC_LETTERS for ch in normalized)
     if has_kazakh:
         return "kk"
@@ -605,6 +642,8 @@ def _sanitize_voice_text(text: str) -> str:
 
 DEFAULT_NAME_RETRY_TEXT = "Подскажите, пожалуйста, как я могу к вам обращаться?"
 DEFAULT_PERSONALIZED_GREETING_TEMPLATE = '{name}, я работаю на основе искусственного интеллекта и постараюсь максимально внимательно разобраться с вашим вопросом. Чем я могу помочь?'
+KAZAKH_NAME_RETRY_TEXT = "Өтінемін, сізге қалай жүгінсем болады?"
+KAZAKH_PERSONALIZED_GREETING_TEMPLATE = "{name}, мен жасанды интеллект негізінде жұмыс істеймін және сұрағыңызды мұқият түсінуге тырысамын. Қалай көмектесе аламын?"
 _NAME_CAPTURE_CONFUSION_ALIASES = {
     "парень": "Арнур",
     "арнер": "Арнур",
@@ -693,7 +732,11 @@ def _normalize_name_candidate(text: str | None) -> str | None:
         "атым",
         "аты",
         "болады",
+        "язык",
+        "тіл",
     }
+    stop_words.update(_KAZAKH_LANGUAGE_MARKERS)
+    stop_words.update(_RUSSIAN_LANGUAGE_MARKERS)
     filtered = [word for word, lowered_word in zip(words, lowered) if lowered_word not in stop_words]
     if not filtered:
         return None
@@ -756,6 +799,8 @@ def _normalize_name_candidate(text: str | None) -> str | None:
         "сұрақ",
         "көмек",
     }
+    invalid_tokens.update(_KAZAKH_LANGUAGE_MARKERS)
+    invalid_tokens.update(_RUSSIAN_LANGUAGE_MARKERS)
     invalid_letter_tokens = {_voice_letters_key(token) for token in invalid_tokens}
     filler_letters = {"а", "э", "е", "ё", "у", "о", "ы", "м", "m", "h"}
     for word in filtered:
@@ -838,6 +883,8 @@ def _extract_name_candidate(text: str | None) -> tuple[str | None, bool]:
         "сұра",
         "көмек",
     }
+    cutoff_tokens.update(_KAZAKH_LANGUAGE_MARKERS)
+    cutoff_tokens.update(_RUSSIAN_LANGUAGE_MARKERS)
     for pattern in explicit_patterns:
         match = re.search(pattern, normalized, flags=re.IGNORECASE)
         if not match:
@@ -1008,6 +1055,7 @@ class CallSession:
         tts: BaseTTS | None = None,
         filler_audio: FillerAudioLibrary | None = None,
         initial_greeting_text: str | None = None,
+        initial_greeting_segments: Iterable[tuple[str, str | None]] | None = None,
     ) -> None:
         self.session_id = session_id
         self.transport = transport
@@ -1033,9 +1081,13 @@ class CallSession:
         self._sentence_queue: asyncio.Queue[str | None] | None = None
         self._closed = False
         self._filler_audio = filler_audio
-        self._initial_greeting_text = str(initial_greeting_text or "").strip()
+        self._initial_greeting_segments = self._normalize_initial_greeting_segments(
+            initial_greeting_segments,
+            fallback_text=initial_greeting_text,
+        )
+        self._initial_greeting_text = _combine_user_transcripts(text for text, _ in self._initial_greeting_segments)
         self._initial_greeting_discarded_bytes = 0
-        self._awaiting_customer_name = bool(self._initial_greeting_text)
+        self._awaiting_customer_name = bool(self._initial_greeting_segments)
         self._customer_name: str | None = None
         self._name_retry_text = DEFAULT_NAME_RETRY_TEXT
         self._personalized_greeting_template = DEFAULT_PERSONALIZED_GREETING_TEMPLATE
@@ -1057,6 +1109,26 @@ class CallSession:
             self._semantic_hold_silence_timeout_ms,
         )
 
+    @staticmethod
+    def _normalize_initial_greeting_segments(
+        segments: Iterable[tuple[str, str | None]] | None,
+        *,
+        fallback_text: str | None,
+    ) -> tuple[tuple[str, str | None], ...]:
+        normalized_segments: list[tuple[str, str | None]] = []
+        for text, language_code in segments or ():
+            normalized_text = str(text or "").strip()
+            if not normalized_text:
+                continue
+            normalized_language = str(language_code or "").strip() or None
+            normalized_segments.append((normalized_text, normalized_language))
+        if normalized_segments:
+            return tuple(normalized_segments)
+        normalized_fallback = str(fallback_text or "").strip()
+        if normalized_fallback:
+            return ((normalized_fallback, None),)
+        return ()
+
     @property
     def conversation(self) -> tuple[tuple[str, str], ...]:
         return tuple(self._conversation)
@@ -1641,13 +1713,22 @@ class CallSession:
         raw = (
             os.getenv("STT_NAME_CAPTURE_LANGUAGE_CODE", "").strip()
             or os.getenv("STT_NAME_CAPTURE_LANGUAGE", "").strip()
-            or "kz"
         )
         return raw or None
 
     def _build_name_collection_response(self, transcript: str) -> str | None:
         if not self._awaiting_customer_name:
             return None
+        if self._session_language is None:
+            detected_language = _detect_session_language(transcript)
+            if detected_language:
+                self._session_language = detected_language
+                LOGGER.info(
+                    "realtime session %s language locked during name collection: language=%s transcript=%r",
+                    self.session_id,
+                    detected_language,
+                    _preview_text(transcript),
+                )
         status, name_value = _voice_start_name_outcome(transcript)
         LOGGER.info(
             "realtime session %s name collection outcome: status=%s name=%r transcript=%r",
@@ -1661,9 +1742,19 @@ class CallSession:
             if short_name:
                 self._customer_name = short_name
                 self._awaiting_customer_name = False
-                return self._personalized_greeting_template.format(name=short_name)
+                return self._personalized_greeting_text(short_name)
+        return self._name_retry_text_for_current_language()
+
+    def _name_retry_text_for_current_language(self) -> str:
+        if self._session_language == "kk":
+            return KAZAKH_NAME_RETRY_TEXT
         return self._name_retry_text
 
+    def _personalized_greeting_text(self, name: str) -> str:
+        if self._session_language == "kk":
+            return KAZAKH_PERSONALIZED_GREETING_TEMPLATE.format(name=name)
+        return self._personalized_greeting_template.format(name=name)
+
     def _build_llm_context(self) -> list[object]:
         context: list[object] = list(self._conversation)
         if self._customer_name:
@@ -1719,7 +1810,7 @@ class CallSession:
                     await playback_task
 
     def _start_initial_greeting(self) -> None:
-        if not self._initial_greeting_text or self._closed:
+        if not self._initial_greeting_segments or self._closed:
             return
         if self._greeting_task is not None and not self._greeting_task.done():
             return
@@ -1727,13 +1818,15 @@ class CallSession:
             self._play_initial_greeting(
                 epoch=self.generation_epoch,
                 greeting_text=self._initial_greeting_text,
+                greeting_segments=self._initial_greeting_segments,
             ),
             name=f"{self.session_id}-greeting-{self.generation_epoch}",
         )
         LOGGER.info(
-            "realtime session %s initial greeting scheduled: epoch=%s chars=%s text=%r",
+            "realtime session %s initial greeting scheduled: epoch=%s segments=%s chars=%s text=%r",
             self.session_id,
             self.generation_epoch,
+            [(language, len(text)) for text, language in self._initial_greeting_segments],
             len(self._initial_greeting_text),
             _preview_text(self._initial_greeting_text),
         )
@@ -1743,10 +1836,8 @@ class CallSession:
         *,
         epoch: int,
         greeting_text: str,
+        greeting_segments: Iterable[tuple[str, str | None]],
     ) -> None:
-        async def one_shot_text_stream() -> AsyncGenerator[str, None]:
-            yield greeting_text
-
         first_audio_seen = False
         started_monotonic = time.perf_counter()
         cached_greeting = self._filler_audio.pick("initial_greeting") if self._filler_audio is not None else None
@@ -1761,22 +1852,37 @@ class CallSession:
         try:
             audio_chunk_count = 0
             audio_byte_count = 0
-            async for audio_chunk in self._tts.synthesize_stream(one_shot_text_stream()):
-                self._ensure_generation(epoch)
-                if not audio_chunk:
-                    continue
-                audio_chunk_count += 1
-                audio_byte_count += len(audio_chunk)
-                if not first_audio_seen:
-                    first_audio_seen = True
-                    self._log_latency(
-                        "initial_greeting_ttfa",
-                        started_monotonic,
-                        epoch=epoch,
-                        message="session start -> initial greeting first audio",
-                    )
-                    self._set_state(SessionState.ASSISTANT_SPEAKING, reason="initial greeting started")
-                await self.transport.send_audio(audio_chunk)
+            for segment_text, language_code in greeting_segments:
+                async def one_shot_text_stream() -> AsyncGenerator[str, None]:
+                    yield segment_text
+
+                LOGGER.info(
+                    "realtime session %s initial greeting segment start: epoch=%s language=%s chars=%s text=%r",
+                    self.session_id,
+                    epoch,
+                    language_code,
+                    len(segment_text),
+                    _preview_text(segment_text),
+                )
+                async for audio_chunk in self._tts.synthesize_stream(
+                    one_shot_text_stream(),
+                    language_code=language_code,
+                ):
+                    self._ensure_generation(epoch)
+                    if not audio_chunk:
+                        continue
+                    audio_chunk_count += 1
+                    audio_byte_count += len(audio_chunk)
+                    if not first_audio_seen:
+                        first_audio_seen = True
+                        self._log_latency(
+                            "initial_greeting_ttfa",
+                            started_monotonic,
+                            epoch=epoch,
+                            message="session start -> initial greeting first audio",
+                        )
+                        self._set_state(SessionState.ASSISTANT_SPEAKING, reason="initial greeting started")
+                    await self.transport.send_audio(audio_chunk)
             await self.transport.flush_audio()
             self._ensure_generation(epoch)
             LOGGER.info(

main.py

@@ -87,9 +87,43 @@ def _http_port() -> int:
 
 def _initial_greeting_text() -> str:
     raw = os.getenv("REALTIME_VOICE_INITIAL_GREETING_TEXT")
-    if raw is None:
-        return "Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. Как я могу к вам обращаться? Я работаю на основе искусственного интеллекта и постараюсь максимально внимательно разобраться с вашим вопросом. Расскажите, пожалуйста, чем могу помочь?"
-    return str(raw).strip()
+    if raw is not None and str(raw).strip():
+        return str(raw).strip()
+    return " ".join(text for text, _ in _initial_greeting_segments())
+
+
+def _initial_greeting_segments() -> tuple[tuple[str, str | None], ...]:
+    raw = os.getenv("REALTIME_VOICE_INITIAL_GREETING_TEXT")
+    if raw is not None and str(raw).strip():
+        language = str(os.getenv("REALTIME_VOICE_INITIAL_GREETING_LANGUAGE_CODE", "")).strip() or None
+        text = str(raw).strip()
+        return ((text, language),)
+
+    kk_text = str(
+        os.getenv(
+            "REALTIME_VOICE_INITIAL_GREETING_KK_TEXT",
+            (
+                "Сәлеметсіз бе! Менің атым Айнұр, мен ДиджиОпс компаниясының эй-ай ассистентімін. "
+                "Сізге қалай жүгінсем болады және қай тілде сөйлескен ыңғайлы: қазақша ма, орысша ма?"
+            ),
+        )
+    ).strip()
+    ru_text = str(
+        os.getenv(
+            "REALTIME_VOICE_INITIAL_GREETING_RU_TEXT",
+            (
+                "Здравствуйте! Меня зовут Айнур, я эй-ай-ассистент компании ДиджиОпс. "
+                "Как я могу к вам обращаться и на каком языке вам удобнее продолжить: на казахском или на русском?"
+            ),
+        )
+    ).strip()
+    segments: list[tuple[str, str | None]] = []
+    if kk_text:
+        segments.append((kk_text, "kk"))
+    if ru_text:
+        segments.append((ru_text, "ru"))
+    return tuple(segments)
+
 
 def _llm_provider_name() -> str:
     return str(os.getenv("LLM_PROVIDER") or os.getenv("REALTIME_VOICE_LLM_PROVIDER") or "openai").strip().lower()
@@ -167,17 +201,19 @@ class RealtimeVoiceService:
             LOGGER.exception("failed to preload filler audio clips")
 
     async def _preload_initial_greeting_audio(self) -> None:
-        greeting_text = _initial_greeting_text()
-        if not greeting_text:
+        greeting_segments = _initial_greeting_segments()
+        greeting_text = " ".join(text for text, _ in greeting_segments).strip()
+        if not greeting_segments or not greeting_text:
             LOGGER.info("initial greeting audio preload skipped: empty text")
             return
         try:
             LOGGER.info(
-                "initial greeting audio preload start: chars=%s text=%r",
+                "initial greeting audio preload start: segments=%s chars=%s text=%r",
+                [(language, len(text)) for text, language in greeting_segments],
                 len(greeting_text),
                 greeting_text[:120],
             )
-            clip = await self._filler_audio.synthesize_text(self._tts, greeting_text)
+            clip = await self._filler_audio.synthesize_segments(self._tts, greeting_segments)
             if not clip:
                 LOGGER.warning("initial greeting audio preload produced empty clip")
                 return
@@ -240,6 +276,7 @@ class RealtimeVoiceService:
             tts=self._tts,
             filler_audio=self._filler_audio,
             initial_greeting_text=_initial_greeting_text(),
+            initial_greeting_segments=_initial_greeting_segments(),
         )
 
     @staticmethod