diff --git a/.env.example b/.env.example index 3d53093..4c34233 100644 --- a/.env.example +++ b/.env.example @@ -2,12 +2,35 @@ REALTIME_VOICE_HTTP_HOST=0.0.0.0 REALTIME_VOICE_HTTP_PORT=8000 REALTIME_VOICE_AUDIOSOCKET_HOST=0.0.0.0 REALTIME_VOICE_AUDIOSOCKET_PORT=9092 +REALTIME_VOICE_SAMPLE_RATE_HZ=8000 + +LLM_PROVIDER=openai OPENAI_API_KEY= OPENAI_BASE_URL= OPENAI_LLM_MODEL=gpt-4o-mini OPENAI_LLM_SYSTEM_PROMPT=You are a concise voice assistant for a telecom call center. Answer clearly and briefly. OPENAI_TIMEOUT_SECONDS=30 +OPENAI_STT_MODEL=whisper-1 +OPENAI_STT_LANGUAGE=ru +OPENAI_STT_TIMEOUT_SECONDS=30 + +OLLAMA_BASE_URL=http://host.docker.internal:11434 +OLLAMA_LLM_MODEL=qwen2.5:1.5b +OLLAMA_TIMEOUT_SECONDS=20 +OLLAMA_LLM_TEMPERATURE=0.3 +OLLAMA_LLM_SYSTEM_PROMPT=Ты Айнур, голосовой ИИ-оператор. Всегда отвечай только на русском, коротко и по делу. +OLLAMA_LLM_MAX_CONTEXT_MESSAGES=4 +OLLAMA_LLM_NUM_PREDICT=64 +OLLAMA_LLM_NUM_CTX=1024 + +STT_PROVIDER=openai +STT_FALLBACK_PROVIDER=elevenlabs +STT_PROMPT=Это разговор на русском языке, но могут встречаться казахские имена: Айбын, Магжан, Бауыржан, Асель. +TTS_PROVIDER=elevenlabs + +ENABLE_AUDIO_DUMP=false +AUDIO_DUMP_DIR=debug_audio ELEVENLABS_API_KEY= ELEVENLABS_API_BASE=https://api.elevenlabs.io @@ -15,13 +38,23 @@ ELEVENLABS_TTS_VOICE_ID= ELEVENLABS_TTS_MODEL_ID=eleven_flash_v2_5 ELEVENLABS_TTS_LANGUAGE_CODE=ru ELEVENLABS_TTS_OUTPUT_FORMAT=pcm_16000 +ELEVENLABS_TTS_SPEED=1.0 ELEVENLABS_STT_MODEL_ID=scribe_v2 -ELEVENLABS_STT_LANGUAGE_CODE= +ELEVENLABS_STT_REALTIME_MODEL_ID=scribe_v2_realtime +ELEVENLABS_STT_LANGUAGE_CODE=ru +ELEVENLABS_STT_USE_REALTIME=true +ELEVENLABS_STT_ALLOW_BATCH_FALLBACK=true ELEVENLABS_TIMEOUT_SECONDS=30 +YANDEX_STT_API_KEY= +YANDEX_STT_IAM_TOKEN= +YANDEX_STT_FOLDER_ID= +YANDEX_STT_LANGUAGE=ru-RU +YANDEX_STT_TOPIC=general + VAD_THRESHOLD=0.5 VAD_NEGATIVE_THRESHOLD= -VAD_SILENCE_TIMEOUT_MS=1600 +VAD_SILENCE_TIMEOUT_MS=550 VAD_SPEECH_PAD_MS=64 VAD_MIN_SPEECH_DURATION_MS=0 VAD_USE_ONNX=false diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49a3f6f --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Local environment and secrets +.env +.env.* +!.env.example + +# Python cache +__pycache__/ +*.py[cod] + +# Runtime and debug artifacts +trace.log +debug_audio/ + +# Local tool artifacts +.codex + +# Local backups and snapshots +*.backup.* +*.bak-* +backups/ +staging/ diff --git a/core/audio_pacer.py b/core/audio_pacer.py new file mode 100644 index 0000000..eec5ac2 --- /dev/null +++ b/core/audio_pacer.py @@ -0,0 +1,42 @@ +from __future__ import annotations + + +class AudioPacer: + def __init__(self, *, frame_bytes: int) -> None: + self._frame_bytes = max(int(frame_bytes), 1) + self._buffer = bytearray() + + @property + def buffered_bytes(self) -> int: + return len(self._buffer) + + def push(self, audio_chunk: bytes) -> list[bytes]: + if not audio_chunk: + return [] + self._buffer.extend(audio_chunk) + return self._pop_frames() + + def flush(self, *, pad_final_frame: bool = True) -> list[bytes]: + frames = self._pop_frames() + if not self._buffer: + return frames + if pad_final_frame: + padded = bytes(self._buffer) + (b"\x00" * (self._frame_bytes - len(self._buffer))) + frames.append(padded) + else: + frames.append(bytes(self._buffer)) + self._buffer.clear() + return frames + + def clear(self) -> None: + self._buffer.clear() + + def reset(self) -> None: + self.clear() + + def _pop_frames(self) -> list[bytes]: + frames: list[bytes] = [] + while len(self._buffer) >= self._frame_bytes: + frames.append(bytes(self._buffer[: self._frame_bytes])) + del self._buffer[: self._frame_bytes] + return frames diff --git a/core/filler_audio.py b/core/filler_audio.py new file mode 100644 index 0000000..25e6569 --- /dev/null +++ b/core/filler_audio.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import asyncio +import audioop +import io +import logging +import os +import random +import wave +from collections.abc import AsyncIterable + +from realtime_voice_service.providers.base import BaseTTS + + +LOGGER = logging.getLogger("uvicorn.error") + +DEFAULT_FILLER_TEXTS: dict[str, tuple[str, ...]] = { + "serper": ( + "Дайте-ка я поищу.", + "Минуточку, проверяю данные.", + "Сейчас быстро посмотрю информацию.", + ), + "generic": ( + "Секундочку.", + "Проверяю данные.", + ), +} + + +class FillerAudioLibrary: + def __init__( + self, + *, + sample_rate_hz: int = 16000, + config_env: str = "REALTIME_VOICE_FILLER_AUDIO_FILES", + enabled_env: str = "REALTIME_VOICE_FILLER_AUDIO_ENABLED", + ) -> None: + self._sample_rate_hz = max(int(sample_rate_hz), 1) + self._config_env = config_env + self._enabled = self._read_bool_env(enabled_env, True) + self._clips: dict[str, list[bytes]] = {} + self._preload_lock = asyncio.Lock() + self._preloaded = False + self._random = random.Random() + + async def preload(self, *, tts: BaseTTS | None = None) -> None: + if not self._enabled or self._preloaded: + LOGGER.info( + "filler audio preload skipped: enabled=%s preloaded=%s clips=%s", + self._enabled, + self._preloaded, + {key: len(value) for key, value in self._clips.items()}, + ) + return + async with self._preload_lock: + if self._preloaded: + return + LOGGER.info("filler audio preload start: sample_rate=%s", self._sample_rate_hz) + self._load_from_files() + if tts is not None: + await self._synthesize_missing_fillers(tts) + self._preloaded = True + LOGGER.info( + "filler audio preload done: clips=%s", + {key: len(value) for key, value in self._clips.items()}, + ) + + def pick(self, key: str | None = None) -> bytes | None: + if not self._enabled: + return None + normalized_key = (key or "").strip().lower() or "generic" + candidates = self._clips.get(normalized_key) or self._clips.get("generic") or [] + if not candidates: + LOGGER.info("filler audio pick missed: key=%s available=%s", normalized_key, list(self._clips)) + return None + clip = self._random.choice(candidates) + LOGGER.info( + "filler audio picked: key=%s candidates=%s bytes=%s", + normalized_key, + len(candidates), + len(clip), + ) + return clip + + def add_clip(self, key: str, clip: bytes) -> None: + if not clip: + return + normalized_key = (key or "").strip().lower() or "generic" + self._clips.setdefault(normalized_key, []).append(clip) + LOGGER.info( + "filler audio clip added: key=%s total=%s bytes=%s", + normalized_key, + len(self._clips[normalized_key]), + len(clip), + ) + + async def synthesize_text(self, tts: BaseTTS, text: str) -> bytes: + return await self._synthesize_clip(tts, text) + + def _load_from_files(self) -> None: + raw_config = str(os.getenv(self._config_env, "")).strip() + if not raw_config: + return + for raw_entry in raw_config.split(","): + entry = raw_entry.strip() + if not entry: + continue + clip_key = "generic" + clip_path = entry + if "=" in entry: + maybe_key, maybe_path = entry.split("=", 1) + clip_key = maybe_key.strip().lower() or "generic" + clip_path = maybe_path.strip() + if not clip_path: + continue + try: + clip = self._load_clip_from_path(clip_path) + except Exception: + LOGGER.exception("failed to preload filler audio clip path=%s key=%s", clip_path, clip_key) + continue + self._clips.setdefault(clip_key, []).append(clip) + LOGGER.info( + "filler audio file loaded: key=%s path=%s bytes=%s", + clip_key, + clip_path, + len(clip), + ) + + async def _synthesize_missing_fillers(self, tts: BaseTTS) -> None: + for key, texts in DEFAULT_FILLER_TEXTS.items(): + if self._clips.get(key): + continue + for text in texts: + try: + LOGGER.info("filler audio synth start: key=%s text=%r", key, text) + clip = await self._synthesize_clip(tts, text) + except Exception: + LOGGER.exception("failed to synthesize filler clip key=%s", key) + break + if clip: + self._clips.setdefault(key, []).append(clip) + LOGGER.info( + "filler audio synth done: key=%s text=%r bytes=%s", + key, + text, + len(clip), + ) + if not self._clips.get(key): + LOGGER.warning("no filler clips available for key=%s", key) + + async def _synthesize_clip(self, tts: BaseTTS, text: str) -> bytes: + async def one_shot_text_stream() -> AsyncIterable[str]: + yield text + + clip = bytearray() + async for audio_chunk in tts.synthesize_stream(one_shot_text_stream()): + if audio_chunk: + clip.extend(audio_chunk) + return bytes(clip) + + def _load_clip_from_path(self, clip_path: str) -> bytes: + with open(clip_path, "rb") as handle: + payload = handle.read() + if clip_path.lower().endswith(".wav"): + return self._decode_wav(payload) + return payload + + def _decode_wav(self, payload: bytes) -> bytes: + with wave.open(io.BytesIO(payload), "rb") as wav_file: + pcm_bytes = wav_file.readframes(wav_file.getnframes()) + sample_width = wav_file.getsampwidth() + channels = wav_file.getnchannels() + sample_rate_hz = int(wav_file.getframerate() or self._sample_rate_hz) + if sample_width != 2: + raise ValueError("filler audio WAV must be PCM16") + if channels == 2: + pcm_bytes = audioop.tomono(pcm_bytes, sample_width, 0.5, 0.5) + if sample_rate_hz == self._sample_rate_hz: + return pcm_bytes + converted, _ = audioop.ratecv( + pcm_bytes, + 2, + 1, + sample_rate_hz, + self._sample_rate_hz, + None, + ) + return converted + + @staticmethod + def _read_bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return str(raw).strip().lower() in {"1", "true", "yes", "on"} diff --git a/core/session.py b/core/session.py index 032075e..541fe86 100644 --- a/core/session.py +++ b/core/session.py @@ -2,17 +2,602 @@ from __future__ import annotations import asyncio import contextlib +import datetime +import difflib +import io import logging +import os +import re import time +import wave +from collections.abc import AsyncGenerator from collections.abc import Iterable from enum import Enum +from realtime_voice_service.core.filler_audio import FillerAudioLibrary from realtime_voice_service.core.vad import BaseVAD, SileroVADDetector -from realtime_voice_service.providers.base import BaseLLM, BaseSTT, BaseTTS, MockLLM, MockSTT, MockTTS +from realtime_voice_service.providers.base import BaseLLM, BaseSTT, BaseSTTStream, BaseTTS, MockLLM, MockSTT, MockTTS from realtime_voice_service.transports.base import BaseMediaTransport LOGGER = logging.getLogger("uvicorn.error") +SEMANTIC_ENDPOINTING_HOLD_MS = 2000 +SEMANTIC_CONTINUATION_TOKENS = { + "а", + "в", + "во", + "да", + "до", + "и", + "или", + "к", + "ко", + "на", + "но", + "ну", + "о", + "об", + "от", + "по", + "при", + "с", + "со", + "так", + "то", + "у", + "что", + "чтобы", + "эээ", + "ммм", +} + + +def _preview_text(text: str, *, limit: int = 160) -> str: + normalized = " ".join(str(text or "").split()) + if len(normalized) <= limit: + return normalized + return f"{normalized[:limit]}..." + + +def _audio_duration_ms(audio_bytes: bytes, *, sample_rate_hz: int) -> int: + if not audio_bytes or sample_rate_hz <= 0: + return 0 + return int(((len(audio_bytes) // 2) / float(sample_rate_hz)) * 1000.0) + + +def _audio_byte_count_duration_ms(byte_count: int, *, sample_rate_hz: int) -> int: + if byte_count <= 0 or sample_rate_hz <= 0: + return 0 + return int(((byte_count // 2) / float(sample_rate_hz)) * 1000.0) + + +RU_DIGIT_WORDS = ("ноль", "один", "два", "три", "четыре", "пять", "шесть", "семь", "восемь", "девять") +RU_UNITS_MASCULINE = ("", "один", "два", "три", "четыре", "пять", "шесть", "семь", "восемь", "девять") +RU_UNITS_FEMININE = ("", "одна", "две", "три", "четыре", "пять", "шесть", "семь", "восемь", "девять") +RU_TEENS = ( + "десять", + "одиннадцать", + "двенадцать", + "тринадцать", + "четырнадцать", + "пятнадцать", + "шестнадцать", + "семнадцать", + "восемнадцать", + "девятнадцать", +) +RU_TENS = ("", "", "двадцать", "тридцать", "сорок", "пятьдесят", "шестьдесят", "семьдесят", "восемьдесят", "девяносто") +RU_HUNDREDS = ( + "", + "сто", + "двести", + "триста", + "четыреста", + "пятьсот", + "шестьсот", + "семьсот", + "восемьсот", + "девятьсот", +) + + +RU_DAY_ORDINALS = { + 1: "первое", + 2: "второе", + 3: "третье", + 4: "четвертое", + 5: "пятое", + 6: "шестое", + 7: "седьмое", + 8: "восьмое", + 9: "девятое", + 10: "десятое", + 11: "одиннадцатое", + 12: "двенадцатое", + 13: "тринадцатое", + 14: "четырнадцатое", + 15: "пятнадцатое", + 16: "шестнадцатое", + 17: "семнадцатое", + 18: "восемнадцатое", + 19: "девятнадцатое", + 20: "двадцатое", + 21: "двадцать первое", + 22: "двадцать второе", + 23: "двадцать третье", + 24: "двадцать четвертое", + 25: "двадцать пятое", + 26: "двадцать шестое", + 27: "двадцать седьмое", + 28: "двадцать восьмое", + 29: "двадцать девятое", + 30: "тридцатое", + 31: "тридцать первое", +} +RU_MONTHS_GENITIVE = { + 1: "января", + 2: "февраля", + 3: "марта", + 4: "апреля", + 5: "мая", + 6: "июня", + 7: "июля", + 8: "августа", + 9: "сентября", + 10: "октября", + 11: "ноября", + 12: "декабря", +} + + +def _ru_plural(value: int, one: str, few: str, many: str) -> str: + value = abs(value) % 100 + if 11 <= value <= 19: + return many + last_digit = value % 10 + if last_digit == 1: + return one + if 2 <= last_digit <= 4: + return few + return many + + +def _ru_under_1000(value: int, *, feminine: bool = False) -> list[str]: + value = max(min(int(value), 999), 0) + words: list[str] = [] + hundreds = value // 100 + if hundreds: + words.append(RU_HUNDREDS[hundreds]) + remainder = value % 100 + if 10 <= remainder <= 19: + words.append(RU_TEENS[remainder - 10]) + return words + tens = remainder // 10 + units = remainder % 10 + if tens: + words.append(RU_TENS[tens]) + if units: + words.append((RU_UNITS_FEMININE if feminine else RU_UNITS_MASCULINE)[units]) + return words + + +def _ru_int_words(value: int) -> str: + value = int(value) + if value == 0: + return RU_DIGIT_WORDS[0] + if value < 0: + return f"минус {_ru_int_words(abs(value))}" + + words: list[str] = [] + billions = value // 1_000_000_000 + if billions: + words.extend(_ru_under_1000(billions)) + words.append(_ru_plural(billions, "миллиард", "миллиарда", "миллиардов")) + value %= 1_000_000_000 + + millions = value // 1_000_000 + if millions: + words.extend(_ru_under_1000(millions)) + words.append(_ru_plural(millions, "миллион", "миллиона", "миллионов")) + value %= 1_000_000 + + thousands = value // 1000 + if thousands: + words.extend(_ru_under_1000(thousands, feminine=True)) + words.append(_ru_plural(thousands, "тысяча", "тысячи", "тысяч")) + value %= 1000 + + if value: + words.extend(_ru_under_1000(value)) + return " ".join(word for word in words if word) + + +def _ru_digit_sequence(raw: str) -> str: + return " ".join(RU_DIGIT_WORDS[int(char)] for char in raw if char.isdigit()) + + +def _ru_number_words(raw: str) -> str: + compact = str(raw or "").strip().replace(" ", "") + decimal_match = re.fullmatch(r"(\d+)[,.](\d+)", compact) + if decimal_match: + integer_part = _ru_number_words(decimal_match.group(1)) + fractional_part = _ru_digit_sequence(decimal_match.group(2)) + return f"{integer_part} целых {fractional_part}" + + digits = re.sub(r"\D+", "", compact) + if not digits: + return str(raw or "") + if digits.startswith("0") or len(digits) > 6: + return _ru_digit_sequence(digits) + return _ru_int_words(int(digits)) + + +def _ru_genitive_number_words(raw: str) -> str: + digits = re.sub(r"\D+", "", str(raw or "")) + if not digits: + return _ru_number_words(raw) + value = int(digits) + small_genitive = { + 1: "одного", + 2: "двух", + 3: "трех", + 4: "четырех", + 5: "пяти", + 6: "шести", + 7: "семи", + 8: "восьми", + 9: "девяти", + 10: "десяти", + } + return small_genitive.get(value, _ru_number_words(raw)) + + +def _ru_percent_phrase(raw: str) -> str: + words = _ru_number_words(raw) + compact = str(raw or "").strip().replace(" ", "") + if re.fullmatch(r"\d+[,.]\d+", compact): + return f"{words} процента" + digits = re.sub(r"\D+", "", compact) + if not digits or digits.startswith("0") or len(digits) > 6: + return f"{words} процентов" + value = int(digits) + return f"{words} {_ru_plural(value, 'процент', 'процента', 'процентов')}" + + +def _ru_date_words(day_raw: str, month_raw: str, year_raw: str) -> str: + day = int(day_raw) + month = int(month_raw) + year = int(year_raw) + day_words = RU_DAY_ORDINALS.get(day, _ru_number_words(str(day))) + month_words = RU_MONTHS_GENITIVE.get(month, _ru_number_words(str(month))) + return f"{day_words} {month_words} {_ru_number_words(str(year))}" + + +def _normalize_voice_numbers(text: str) -> str: + normalized = str(text or "") + normalized = re.sub( + r"(? str: + cleaned = [re.sub(r"\s+", " ", str(part or "")).strip() for part in parts] + cleaned = [part for part in cleaned if part] + if len(cleaned) <= 1: + return cleaned[0] if cleaned else "" + normalized_parts: list[str] = [] + for part in cleaned: + if normalized_parts and not normalized_parts[-1].endswith((".", "!", "?", "…")): + normalized_parts[-1] = f"{normalized_parts[-1]}." + normalized_parts.append(part) + return " ".join(normalized_parts) + + +def _sanitize_voice_text(text: str) -> str: + sanitized = str(text or "") + sanitized = re.sub(r"\[([^\]]+)\]\((?:https?://|www\.)[^)\s]+[^)]*\)", r"\1", sanitized) + sanitized = re.sub(r"(?:https?://|www\.)\S+", "", sanitized) + sanitized = sanitized.replace("[", "").replace("]", "").replace("(", "").replace(")", "") + sanitized = re.sub(r"\s+", " ", sanitized).strip() + sanitized = _normalize_voice_numbers(sanitized) + sanitized = re.sub(r"\s+", " ", sanitized).strip() + return sanitized + + +DEFAULT_NAME_RETRY_TEXT = "Подскажите, пожалуйста, как я могу к вам обращаться?" +DEFAULT_PERSONALIZED_GREETING_TEMPLATE = "{name}, чем я могу помочь?" + + +def _voice_text_key(text: str | None) -> str: + compact = re.sub(r"[^\w\s]+", " ", str(text or "").lower(), flags=re.UNICODE) + return re.sub(r"\s+", " ", compact).strip() + + +def _voice_letters_key(text: str | None) -> str: + return "".join(re.findall(r"[^\W\d_]+", str(text or "").lower(), flags=re.UNICODE)) + + +def _canonical_name(text: str) -> str: + words = [part for part in re.split(r"\s+", text.strip()) if part] + return " ".join(word[:1].upper() + word[1:].lower() if len(word) > 1 else word.upper() for word in words) + + +def _looks_like_supported_name_text(text: str) -> bool: + return bool( + re.fullmatch( + r"[A-Za-zА-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]+(?:[ -][A-Za-zА-Яа-яЁёӘәҒғҚқҢңӨөҰұҮүҺһІі]+){0,2}", + str(text or "").strip(), + ) + ) + + +def _normalize_name_candidate(text: str | None) -> str | None: + raw = str(text or "").strip(" \t\r\n,.;:!?\"'()[]{}") + if not raw or any(ch.isdigit() for ch in raw): + return None + words = re.findall(r"[^\W\d_]+(?:[-'][^\W\d_]+)*", raw, flags=re.UNICODE) + if not words or len(words) > 4: + return None + lowered = [_voice_text_key(word) for word in words] + stop_words = { + "меня", + "зовут", + "это", + "я", + "мое", + "моё", + "имя", + "менің", + "атым", + "аты", + "болады", + } + filtered = [word for word, lowered_word in zip(words, lowered) if lowered_word not in stop_words] + if not filtered: + return None + if len(filtered) == 2 and len(filtered[1]) == 1: + filtered = [filtered[0]] + elif len(filtered) == 2 and len(filtered[0]) == 1: + filtered = [filtered[1]] + invalid_tokens = { + "да", + "нет", + "алло", + "привет", + "здравствуйте", + "добрый", + "день", + "хочу", + "хотел", + "узнать", + "помощь", + "вопрос", + "тариф", + "статус", + "оператор", + "проблема", + "интернет", + "click", + "noise", + "beep", + "tap", + "tick", + "hum", + "um", + "uh", + "umm", + "угу", + "ага", + "эм", + "эмм", + "эммм", + "мм", + "ммм", + "мммм", + "м", + "ум", + "ээ", + "эээ", + "ээээ", + "аа", + "ааа", + "аааа", + "хм", + "хмм", + "хммм", + "звони", + "слушаю", + "слушаюсь", + "хорошо", + "подожди", + "керек", + "сұрақ", + "көмек", + } + invalid_letter_tokens = {_voice_letters_key(token) for token in invalid_tokens} + filler_letters = {"а", "э", "е", "ё", "у", "о", "ы", "м", "m", "h"} + for word in filtered: + token_key = _voice_text_key(word) + letters_key = _voice_letters_key(word) + if token_key in invalid_tokens or letters_key in invalid_letter_tokens: + return None + if len(letters_key) >= 2 and len(set(letters_key)) == 1 and letters_key[0] in filler_letters: + return None + if letters_key in {"хм", "хмм", "хммм"}: + return None + if len(filtered) > 2: + return None + if len(filtered) == 2: + similarity = difflib.SequenceMatcher( + None, + _voice_text_key(filtered[0]), + _voice_text_key(filtered[1]), + ).ratio() + if similarity >= 0.72: + return None + candidate = _canonical_name(" ".join(filtered)) + if not _looks_like_supported_name_text(candidate): + return None + return candidate + + +def _name_followup_needed(text: str) -> bool: + normalized = _voice_text_key(text) + request_markers = ( + "хотел", + "хочу", + "нужн", + "помог", + "вопрос", + "проблем", + "тариф", + "статус", + "оператор", + "адрес", + "график", + "интернет", + "керек", + "сұра", + "көмек", + ) + return any(marker in normalized for marker in request_markers) + + +def _extract_name_candidate(text: str | None) -> tuple[str | None, bool]: + raw = str(text or "").strip() + if not raw: + return None, False + if any(marker in raw for marker in ("[", "]", "<", ">")): + return None, False + normalized = _voice_text_key(raw) + explicit_patterns = ( + r"(?:меня\s+зовут|мо[её]\s+имя|my name is|i am|this is)\s+(.+)", + r"(?:менің\s+атым|аты[мң]?|mening atym)\s+(.+)", + ) + cutoff_tokens = { + "мне", + "надо", + "нужно", + "хочу", + "хотел", + "узнать", + "график", + "работы", + "адрес", + "филиал", + "город", + "тариф", + "статус", + "оператор", + "вопрос", + "проблема", + "интернет", + "керек", + "сұра", + "көмек", + } + for pattern in explicit_patterns: + match = re.search(pattern, normalized, flags=re.IGNORECASE) + if not match: + continue + tail = match.group(1).strip() + tail_words = re.findall(r"[^\W\d_]+(?:[-'][^\W\d_]+)*", tail, flags=re.UNICODE) + candidate_words: list[str] = [] + for word in tail_words: + if _voice_text_key(word) in cutoff_tokens: + break + candidate_words.append(word) + if len(candidate_words) >= 3: + break + candidate = _normalize_name_candidate(" ".join(candidate_words)) or _normalize_name_candidate(tail) + if candidate: + return candidate, False + + candidate = _normalize_name_candidate(raw) + if candidate: + lower_candidate = candidate.lower() + stopwords = { + "здравствуйте", + "привет", + "алло", + "да", + "нет", + "добрый", + "день", + "вопрос", + "интернет", + "у", + "меня", + } + candidate_words = set(lower_candidate.split()) + if candidate_words.issubset(stopwords) or len(lower_candidate) < 2: + return None, False + word_count = len(re.findall(r"[^\W\d_]+(?:[-'][^\W\d_]+)*", raw, flags=re.UNICODE)) + if len(candidate.split()) == 1 and len(candidate) < 3: + return None, False + if word_count <= 2 and not _name_followup_needed(raw): + return candidate, False + return candidate, True + + return None, False + + +def _voice_start_name_outcome(text: str | None) -> tuple[str, str | None]: + raw = str(text or "").strip() + if not raw: + return "name_not_obtained", None + candidate, needs_followup = _extract_name_candidate(raw) + if candidate and not needs_followup: + return "name_obtained", candidate + if candidate: + return "name_followup_required", candidate + if _name_followup_needed(raw): + return "name_followup_required", None + return "name_not_obtained", None + + +def _voice_short_name(name: str | None) -> str | None: + canonical = _normalize_name_candidate(name) + if not canonical: + raw = str(name or "").strip() + if not raw: + return None + canonical = _canonical_name(raw) + short_name = canonical.split(" ", 1)[0].strip() + return short_name or None class SessionState(str, Enum): @@ -27,10 +612,19 @@ class GenerationInterrupted(RuntimeError): class TextChunker: - _BOUNDARY_CHARS = ".!?\n" + _HARD_BOUNDARY_CHARS = ".!?\n" + _SOFT_BOUNDARY_CHARS = ",:" + _SOFT_BOUNDARY_TOKENS = ("\u2014",) - def __init__(self) -> None: + def __init__( + self, + *, + soft_min_chars: int = 18, + soft_min_words: int = 3, + ) -> None: self._buffer = "" + self._soft_min_chars = max(soft_min_chars, 1) + self._soft_min_words = max(soft_min_words, 1) def feed(self, fragment: str) -> list[str]: if not fragment: @@ -60,19 +654,28 @@ class TextChunker: def _pop_next_chunk(self) -> str | None: for index, char in enumerate(self._buffer): - if char not in self._BOUNDARY_CHARS: - continue - end_index = index + 1 - while end_index < len(self._buffer) and self._buffer[end_index] in self._BOUNDARY_CHARS: - end_index += 1 - while end_index < len(self._buffer) and self._buffer[end_index].isspace(): - end_index += 1 - chunk = self._buffer[:end_index].strip() - self._buffer = self._buffer[end_index:] - if chunk: - return chunk + if char in self._HARD_BOUNDARY_CHARS: + return self._consume_chunk(index + 1) + if char in self._SOFT_BOUNDARY_CHARS or char in self._SOFT_BOUNDARY_TOKENS: + candidate = self._buffer[: index + 1].strip() + if self._is_soft_chunk_ready(candidate): + return self._consume_chunk(index + 1) return None + def _consume_chunk(self, end_index: int) -> str | None: + while end_index < len(self._buffer) and self._buffer[end_index] in self._HARD_BOUNDARY_CHARS: + end_index += 1 + while end_index < len(self._buffer) and self._buffer[end_index].isspace(): + end_index += 1 + chunk = self._buffer[:end_index].strip() + self._buffer = self._buffer[end_index:] + return chunk or None + + def _is_soft_chunk_ready(self, candidate: str) -> bool: + if len(candidate) < self._soft_min_chars: + return False + return len(candidate.split()) >= self._soft_min_words + class CallSession: def __init__( @@ -84,6 +687,8 @@ class CallSession: stt: BaseSTT | None = None, llm: BaseLLM | None = None, tts: BaseTTS | None = None, + filler_audio: FillerAudioLibrary | None = None, + initial_greeting_text: str | None = None, ) -> None: self.session_id = session_id self.transport = transport @@ -96,9 +701,41 @@ class CallSession: self._llm = llm or MockLLM() self._tts = tts or MockTTS(sample_rate_hz=transport.sample_rate_hz) self._conversation: list[tuple[str, str]] = [] + self._pending_unanswered_texts: list[str] = [] + self._pending_unanswered_audio: list[bytes] = [] + self._active_turn_epoch: int | None = None + self._active_user_audio: bytes | None = None + self._active_user_transcript: str | None = None + self._active_answer_audio_started = False + self._active_unanswered_captured = False self._assistant_task: asyncio.Task[None] | None = None + self._filler_task: asyncio.Task[None] | None = None + self._greeting_task: asyncio.Task[None] | None = None self._sentence_queue: asyncio.Queue[str | None] | None = None self._closed = False + self._filler_audio = filler_audio + self._initial_greeting_text = str(initial_greeting_text or "").strip() + self._initial_greeting_discarded_bytes = 0 + self._awaiting_customer_name = bool(self._initial_greeting_text) + self._customer_name: str | None = None + self._name_retry_text = DEFAULT_NAME_RETRY_TEXT + self._personalized_greeting_template = DEFAULT_PERSONALIZED_GREETING_TEMPLATE + self._live_stt_stream: BaseSTTStream | None = None + self._latest_partial_transcript = "" + self._default_vad_silence_timeout_ms = getattr(self._vad, "default_speech_end_silence_ms", 550) + self._semantic_hold_silence_timeout_ms = max(self._default_vad_silence_timeout_ms, SEMANTIC_ENDPOINTING_HOLD_MS) + LOGGER.info( + "realtime session %s initialized: protocol=%s sample_rate=%s frame_ms=%s frame_bytes=%s " + "initial_greeting=%s default_vad_silence_ms=%s semantic_hold_ms=%s", + self.session_id, + self.transport.protocol, + self.transport.sample_rate_hz, + self.transport.frame_duration_ms, + self.transport.frame_bytes, + bool(self._initial_greeting_text), + self._default_vad_silence_timeout_ms, + self._semantic_hold_silence_timeout_ms, + ) @property def conversation(self) -> tuple[tuple[str, str], ...]: @@ -107,9 +744,12 @@ class CallSession: async def run(self) -> None: if self._assistant_task is not None: raise RuntimeError("CallSession.run() can only be called once per session") + LOGGER.info("realtime session %s run started", self.session_id) try: + self._start_initial_greeting() await self.media_loop() finally: + LOGGER.info("realtime session %s run stopping", self.session_id) await self.stop() async def media_loop(self) -> None: @@ -121,6 +761,10 @@ class CallSession: if not audio_chunk: continue + if self._initial_greeting_in_progress(): + self._initial_greeting_discarded_bytes += len(audio_chunk) + continue + vad_result = self._vad.feed(audio_chunk) if vad_result.is_speech and self.state != SessionState.USER_SPEAKING: @@ -131,26 +775,62 @@ class CallSession: reason=f"speech detected prob={vad_result.speech_probability:.3f}", ) + if vad_result.speech_started: + LOGGER.info( + "realtime session %s VAD speech_started: prob=%.3f chunk_bytes=%s current_timeout_ms=%s", + self.session_id, + vad_result.speech_probability, + len(audio_chunk), + getattr(self._vad, "current_speech_end_silence_ms", self._default_vad_silence_timeout_ms), + ) + await self._start_live_stt_stream() + elif self._live_stt_stream is not None: + await self._push_live_stt_audio(audio_chunk) + if vad_result.speech_ended and vad_result.utterance_audio: speech_end_monotonic = time.perf_counter() + LOGGER.info( + "realtime session %s VAD speech_ended: utterance_bytes=%s utterance_ms=%s " + "chunk_bytes=%s current_timeout_ms=%s", + self.session_id, + len(vad_result.utterance_audio), + _audio_duration_ms(vad_result.utterance_audio, sample_rate_hz=self.transport.sample_rate_hz), + len(audio_chunk), + getattr(self._vad, "current_speech_end_silence_ms", self._default_vad_silence_timeout_ms), + ) self._set_state(SessionState.ASSISTANT_THINKING, reason="speech end detected") + live_stt_stream = self._detach_live_stt_stream() self._start_assistant_turn( epoch=self.generation_epoch, + stt_stream=live_stt_stream, utterance_audio=vad_result.utterance_audio, speech_end_monotonic=speech_end_monotonic, ) + def _initial_greeting_in_progress(self) -> bool: + return self._greeting_task is not None and not self._greeting_task.done() + def interrupt(self, reason: str = "interrupt") -> int: self.generation_epoch += 1 self.interruptions.append(reason) + if self.state in {SessionState.ASSISTANT_THINKING, SessionState.ASSISTANT_SPEAKING}: + self._capture_unanswered_user_turn(reason=reason) self._clear_sentence_queue() + self.transport.clear_buffer() + if self._greeting_task is not None and not self._greeting_task.done(): + self._greeting_task.cancel() + if self._filler_task is not None and not self._filler_task.done(): + self._filler_task.cancel() if self._assistant_task is not None and not self._assistant_task.done(): + self._capture_unanswered_user_turn(reason="superseded by new assistant turn") self._assistant_task.cancel() LOGGER.info( - "realtime session %s interrupted: epoch=%s reason=%s", + "realtime session %s interrupted: epoch=%s reason=%s state=%s conversation_entries=%s", self.session_id, self.generation_epoch, reason, + self.state.value, + len(self._conversation), ) return self.generation_epoch @@ -158,8 +838,27 @@ class CallSession: if self._closed: return self._closed = True + LOGGER.info( + "realtime session %s stop requested: state=%s epoch=%s interruptions=%s conversation_entries=%s", + self.session_id, + self.state.value, + self.generation_epoch, + self.interruptions, + len(self._conversation), + ) self._vad.reset() + self._reset_semantic_endpointing() self._clear_sentence_queue() + self.transport.clear_buffer() + await self._cancel_live_stt_stream() + if self._greeting_task is not None and not self._greeting_task.done(): + self._greeting_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._greeting_task + if self._filler_task is not None and not self._filler_task.done(): + self._filler_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._filler_task if self._assistant_task is not None and not self._assistant_task.done(): self._assistant_task.cancel() with contextlib.suppress(asyncio.CancelledError): @@ -170,14 +869,25 @@ class CallSession: self, *, epoch: int, + stt_stream: BaseSTTStream | None, utterance_audio: bytes, speech_end_monotonic: float, ) -> None: if self._assistant_task is not None and not self._assistant_task.done(): + self._capture_unanswered_user_turn(reason="superseded by new assistant turn") self._assistant_task.cancel() + LOGGER.info( + "realtime session %s assistant turn scheduled: epoch=%s utterance_bytes=%s utterance_ms=%s live_stt=%s", + self.session_id, + epoch, + len(utterance_audio), + _audio_duration_ms(utterance_audio, sample_rate_hz=self.transport.sample_rate_hz), + stt_stream is not None, + ) self._assistant_task = asyncio.create_task( self._run_assistant_turn( epoch=epoch, + stt_stream=stt_stream, utterance_audio=utterance_audio, speech_end_monotonic=speech_end_monotonic, ), @@ -188,14 +898,30 @@ class CallSession: self, *, epoch: int, + stt_stream: BaseSTTStream | None, utterance_audio: bytes, speech_end_monotonic: float, ) -> None: sentence_queue: asyncio.Queue[str | None] | None = None playback_task: asyncio.Task[None] | None = None assistant_fragments: list[str] = [] + actually_spoken_chunks: list[str] = [] + assistant_audio_started = [False] + transcript = "" + filler_started = False + self._set_active_unanswered_turn(epoch=epoch, utterance_audio=utterance_audio) try: - transcript = await self._stt.transcribe(utterance_audio) + LOGGER.info( + "realtime session %s assistant turn started: epoch=%s utterance_bytes=%s live_stt=%s", + self.session_id, + epoch, + len(utterance_audio), + stt_stream is not None, + ) + transcript = await self._resolve_transcript( + stt_stream=stt_stream, + utterance_audio=utterance_audio, + ) self._ensure_generation(epoch) self._log_latency( "stt_latency", @@ -209,18 +935,91 @@ class CallSession: self._set_state(SessionState.LISTENING, reason="empty transcript") return + pending_unanswered = await self._consume_pending_unanswered_transcripts() + if pending_unanswered: + original_transcript = transcript + transcript = _combine_user_transcripts([*pending_unanswered, transcript]) + LOGGER.info( + "realtime session %s merged unanswered user turns: epoch=%s pending=%s " + "current=%r merged=%r", + self.session_id, + epoch, + len(pending_unanswered), + _preview_text(original_transcript), + _preview_text(transcript), + ) + self._active_user_transcript = transcript + + LOGGER.info( + "realtime session %s transcript accepted: epoch=%s chars=%s text=%r", + self.session_id, + epoch, + len(transcript), + _preview_text(transcript), + ) self._conversation.append(("user", transcript)) + LOGGER.info( + "realtime session %s conversation append user: entries=%s", + self.session_id, + len(self._conversation), + ) + name_collection_response = self._build_name_collection_response(transcript) + if name_collection_response is not None: + LOGGER.info( + "realtime session %s name collection response prepared: epoch=%s name=%r text=%r", + self.session_id, + epoch, + self._customer_name, + _preview_text(name_collection_response), + ) + await self._play_prepared_response( + epoch=epoch, + response_text=name_collection_response, + actually_spoken_chunks=actually_spoken_chunks, + ) + self._ensure_generation(epoch) + self._commit_assistant_context( + llm_generated_text=name_collection_response, + actually_spoken_chunks=actually_spoken_chunks, + interrupted=False, + ) + self._set_state(SessionState.LISTENING, reason="name collection completed") + return sentence_queue = asyncio.Queue() self._sentence_queue = sentence_queue playback_task = asyncio.create_task( - self._stream_tts_pipeline(epoch=epoch, sentence_queue=sentence_queue), + self._stream_tts_pipeline( + epoch=epoch, + sentence_queue=sentence_queue, + actually_spoken_chunks=actually_spoken_chunks, + answer_audio_started=assistant_audio_started, + ), name=f"{self.session_id}-playback-{epoch}", ) llm_started_monotonic = time.perf_counter() chunker = TextChunker() first_token_seen = False - async for token in self._llm.generate_stream(transcript, list(self._conversation)): + async for event in self._llm.generate_stream(transcript, self._build_llm_context()): self._ensure_generation(epoch) + if event.type == "tool_call_start": + LOGGER.info( + "realtime session %s llm tool_call_start: epoch=%s name=%s tool_call_id=%s filler_started=%s", + self.session_id, + epoch, + event.name, + event.tool_call_id, + filler_started, + ) + if not filler_started: + filler_started = True + self._start_filler_audio( + epoch=epoch, + tool_name=event.name, + started_monotonic=llm_started_monotonic, + ) + continue + + token = str(event.content or "") if not token: continue if not first_token_seen: @@ -239,6 +1038,13 @@ class CallSession: ) assistant_text = "".join(assistant_fragments).strip() + LOGGER.info( + "realtime session %s llm stream completed: epoch=%s generated_chars=%s spoken_chunks_so_far=%s", + self.session_id, + epoch, + len(assistant_text), + len(actually_spoken_chunks), + ) if not assistant_text: await self._finish_sentence_queue(sentence_queue) LOGGER.warning("realtime session %s llm produced no text", self.session_id) @@ -254,15 +1060,42 @@ class CallSession: if playback_task is not None: await playback_task self._ensure_generation(epoch) - self._conversation.append(("assistant", assistant_text)) + self._commit_assistant_context( + llm_generated_text=assistant_text, + actually_spoken_chunks=actually_spoken_chunks, + interrupted=False, + ) + LOGGER.info( + "realtime session %s assistant turn completed: epoch=%s generated_chars=%s spoken_chunks=%s", + self.session_id, + epoch, + len(assistant_text), + len(actually_spoken_chunks), + ) self._set_state(SessionState.LISTENING, reason="assistant turn completed") except GenerationInterrupted: + if assistant_audio_started[0] or self._active_answer_audio_started: + self._commit_assistant_context( + llm_generated_text="".join(assistant_fragments).strip(), + actually_spoken_chunks=actually_spoken_chunks, + interrupted=True, + ) + else: + self._capture_unanswered_user_turn(reason="stale generation before answer audio") LOGGER.info( "realtime session %s ignored stale generation %s", self.session_id, epoch, ) except asyncio.CancelledError: + if assistant_audio_started[0] or self._active_answer_audio_started: + self._commit_assistant_context( + llm_generated_text="".join(assistant_fragments).strip(), + actually_spoken_chunks=actually_spoken_chunks, + interrupted=not self._closed, + ) + elif not self._closed: + self._capture_unanswered_user_turn(reason="cancelled before answer audio") LOGGER.info( "realtime session %s cancelled generation %s", self.session_id, @@ -278,15 +1111,283 @@ class CallSession: if epoch == self.generation_epoch and not self._closed: self._set_state(SessionState.LISTENING, reason="assistant generation failed") finally: + if stt_stream is not None: + with contextlib.suppress(Exception): + await stt_stream.cancel() self._clear_sentence_queue(sentence_queue) + if self._filler_task is not None and not self._filler_task.done(): + self._filler_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._filler_task + self._filler_task = None if playback_task is not None and not playback_task.done(): playback_task.cancel() with contextlib.suppress(asyncio.CancelledError): await playback_task current_task = asyncio.current_task() if self._assistant_task is current_task: + self._clear_active_unanswered_turn() self._assistant_task = None + def _set_active_unanswered_turn(self, *, epoch: int, utterance_audio: bytes) -> None: + self._active_turn_epoch = epoch + self._active_user_audio = utterance_audio + self._active_user_transcript = None + self._active_answer_audio_started = False + self._active_unanswered_captured = False + + def _clear_active_unanswered_turn(self) -> None: + self._active_turn_epoch = None + self._active_user_audio = None + self._active_user_transcript = None + self._active_answer_audio_started = False + self._active_unanswered_captured = False + + def _capture_unanswered_user_turn(self, *, reason: str) -> None: + if self._closed or self._active_unanswered_captured or self._active_answer_audio_started: + return + + captured_kind = "" + text = str(self._active_user_transcript or "").strip() + if text: + if self._conversation and self._conversation[-1] == ("user", text): + self._conversation.pop() + LOGGER.info( + "realtime session %s removed unanswered user turn from context: entries=%s text=%r", + self.session_id, + len(self._conversation), + _preview_text(text), + ) + if not self._pending_unanswered_texts or self._pending_unanswered_texts[-1] != text: + self._pending_unanswered_texts.append(text) + captured_kind = "text" + elif self._active_user_audio: + self._pending_unanswered_audio.append(self._active_user_audio) + captured_kind = "audio" + else: + return + + self._active_unanswered_captured = True + self._active_user_audio = None + self._active_user_transcript = None + LOGGER.info( + "realtime session %s captured unanswered user turn: epoch=%s kind=%s reason=%s " + "pending_texts=%s pending_audio=%s", + self.session_id, + self._active_turn_epoch, + captured_kind, + reason, + len(self._pending_unanswered_texts), + len(self._pending_unanswered_audio), + ) + + async def _consume_pending_unanswered_transcripts(self) -> list[str]: + pending_texts = [text for text in self._pending_unanswered_texts if text.strip()] + pending_audio = list(self._pending_unanswered_audio) + self._pending_unanswered_texts = [] + self._pending_unanswered_audio = [] + + for audio_bytes in pending_audio: + if not audio_bytes: + continue + self._maybe_dump_audio(audio_bytes) + try: + LOGGER.info( + "realtime session %s resolving pending unanswered audio: bytes=%s ms=%s", + self.session_id, + len(audio_bytes), + _audio_duration_ms(audio_bytes, sample_rate_hz=self.transport.sample_rate_hz), + ) + pending_transcript = (await self._stt.transcribe(audio_bytes)).strip() + except Exception: + LOGGER.exception("realtime session %s failed to transcribe pending unanswered audio", self.session_id) + continue + if pending_transcript: + pending_texts.append(pending_transcript) + + if pending_texts: + LOGGER.info( + "realtime session %s pending unanswered transcripts ready: count=%s preview=%r", + self.session_id, + len(pending_texts), + _preview_text(_combine_user_transcripts(pending_texts)), + ) + return pending_texts + + def _build_name_collection_response(self, transcript: str) -> str | None: + if not self._awaiting_customer_name: + return None + status, name_value = _voice_start_name_outcome(transcript) + LOGGER.info( + "realtime session %s name collection outcome: status=%s name=%r transcript=%r", + self.session_id, + status, + name_value, + _preview_text(transcript), + ) + if status == "name_obtained" and name_value: + short_name = _voice_short_name(name_value) or str(name_value).strip() + if short_name: + self._customer_name = short_name + self._awaiting_customer_name = False + return self._personalized_greeting_template.format(name=short_name) + return self._name_retry_text + + def _build_llm_context(self) -> list[object]: + context: list[object] = list(self._conversation) + if self._customer_name: + context.append( + { + "role": "system", + "content": ( + f"Имя пользователя: {self._customer_name}. " + "Не повторяй имя в каждом ответе. Используй имя только когда это естественно и полезно: " + "после первого знакомства, при уточнении, важном подтверждении, извинении или прощании. " + "В обычных ответах не обращайся по имени." + ), + } + ) + return context + + async def _play_prepared_response( + self, + *, + epoch: int, + response_text: str, + actually_spoken_chunks: list[str], + ) -> None: + sentence_queue: asyncio.Queue[str | None] = asyncio.Queue() + self._sentence_queue = sentence_queue + playback_task = asyncio.create_task( + self._stream_tts_pipeline( + epoch=epoch, + sentence_queue=sentence_queue, + actually_spoken_chunks=actually_spoken_chunks, + ), + name=f"{self.session_id}-prepared-playback-{epoch}", + ) + try: + chunker = TextChunker() + await self._enqueue_chunks( + epoch=epoch, + sentence_queue=sentence_queue, + chunks=chunker.feed(response_text), + ) + await self._enqueue_chunks( + epoch=epoch, + sentence_queue=sentence_queue, + chunks=chunker.flush(), + ) + await self._finish_sentence_queue(sentence_queue) + await playback_task + finally: + self._clear_sentence_queue(sentence_queue) + if not playback_task.done(): + playback_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await playback_task + + def _start_initial_greeting(self) -> None: + if not self._initial_greeting_text or self._closed: + return + if self._greeting_task is not None and not self._greeting_task.done(): + return + self._greeting_task = asyncio.create_task( + self._play_initial_greeting( + epoch=self.generation_epoch, + greeting_text=self._initial_greeting_text, + ), + name=f"{self.session_id}-greeting-{self.generation_epoch}", + ) + LOGGER.info( + "realtime session %s initial greeting scheduled: epoch=%s chars=%s text=%r", + self.session_id, + self.generation_epoch, + len(self._initial_greeting_text), + _preview_text(self._initial_greeting_text), + ) + + async def _play_initial_greeting( + self, + *, + epoch: int, + greeting_text: str, + ) -> None: + async def one_shot_text_stream() -> AsyncGenerator[str, None]: + yield greeting_text + + first_audio_seen = False + started_monotonic = time.perf_counter() + cached_greeting = self._filler_audio.pick("initial_greeting") if self._filler_audio is not None else None + if cached_greeting: + await self._play_cached_initial_greeting( + epoch=epoch, + greeting_text=greeting_text, + pcm_audio=cached_greeting, + started_monotonic=started_monotonic, + ) + return + try: + audio_chunk_count = 0 + audio_byte_count = 0 + async for audio_chunk in self._tts.synthesize_stream(one_shot_text_stream()): + self._ensure_generation(epoch) + if not audio_chunk: + continue + audio_chunk_count += 1 + audio_byte_count += len(audio_chunk) + if not first_audio_seen: + first_audio_seen = True + self._log_latency( + "initial_greeting_ttfa", + started_monotonic, + epoch=epoch, + message="session start -> initial greeting first audio", + ) + self._set_state(SessionState.ASSISTANT_SPEAKING, reason="initial greeting started") + await self.transport.send_audio(audio_chunk) + await self.transport.flush_audio() + self._ensure_generation(epoch) + LOGGER.info( + "realtime session %s initial greeting audio completed: epoch=%s chunks=%s bytes=%s", + self.session_id, + epoch, + audio_chunk_count, + audio_byte_count, + ) + if self._initial_greeting_discarded_bytes: + LOGGER.info( + "realtime session %s ignored inbound audio during initial greeting: bytes=%s ms=%s", + self.session_id, + self._initial_greeting_discarded_bytes, + _audio_byte_count_duration_ms( + self._initial_greeting_discarded_bytes, + sample_rate_hz=self.transport.sample_rate_hz, + ), + ) + self._initial_greeting_discarded_bytes = 0 + if not self._conversation or self._conversation[-1] != ("assistant", greeting_text): + self._conversation.append(("assistant", greeting_text)) + LOGGER.info( + "realtime session %s conversation append initial greeting: entries=%s", + self.session_id, + len(self._conversation), + ) + if self.state == SessionState.ASSISTANT_SPEAKING and self._assistant_task is None: + self._set_state(SessionState.LISTENING, reason="initial greeting completed") + except GenerationInterrupted: + LOGGER.info("realtime session %s initial greeting interrupted", self.session_id) + except asyncio.CancelledError: + raise + except Exception: + LOGGER.exception("realtime session %s initial greeting failed", self.session_id) + if self.state == SessionState.ASSISTANT_SPEAKING and self._assistant_task is None: + self._set_state(SessionState.LISTENING, reason="initial greeting failed") + finally: + current_task = asyncio.current_task() + if self._greeting_task is current_task: + self._greeting_task = None + def _ensure_generation(self, epoch: int) -> None: if self._closed or epoch != self.generation_epoch: raise GenerationInterrupted(f"stale generation {epoch}") @@ -322,6 +1423,22 @@ class CallSession: normalized = chunk.strip() if not normalized: continue + normalized = _sanitize_voice_text(normalized) + if not normalized: + LOGGER.info( + "realtime session %s skipped non-voice tts chunk after sanitization: epoch=%s raw=%r", + self.session_id, + epoch, + _preview_text(chunk), + ) + continue + LOGGER.info( + "realtime session %s enqueue_tts_text: epoch=%s chars=%s preview=%r", + self.session_id, + epoch, + len(normalized), + _preview_text(normalized), + ) await sentence_queue.put(normalized) async def _finish_sentence_queue(self, sentence_queue: asyncio.Queue[str | None]) -> None: @@ -332,9 +1449,61 @@ class CallSession: *, epoch: int, sentence_queue: asyncio.Queue[str | None], + actually_spoken_chunks: list[str], + answer_audio_started: list[bool] | None = None, ) -> None: first_audio_seen = False - first_tts_started_monotonic: float | None = None + tts_started_monotonic: list[float | None] = [None] + try: + audio_chunk_count = 0 + audio_byte_count = 0 + async for audio_chunk in self._tts.synthesize_stream( + self._iter_tts_text_chunks( + epoch=epoch, + sentence_queue=sentence_queue, + started_holder=tts_started_monotonic, + actually_spoken_chunks=actually_spoken_chunks, + ) + ): + self._ensure_generation(epoch) + if not audio_chunk: + continue + audio_chunk_count += 1 + audio_byte_count += len(audio_chunk) + if not first_audio_seen: + first_audio_seen = True + if answer_audio_started is not None: + answer_audio_started[0] = True + self._active_answer_audio_started = True + self._log_latency( + "ttfa", + tts_started_monotonic[0] or time.perf_counter(), + epoch=epoch, + message="tts request -> first audio", + ) + self._set_state(SessionState.ASSISTANT_SPEAKING, reason="assistant playback started") + await self.transport.send_audio(audio_chunk) + await self.transport.flush_audio() + LOGGER.info( + "realtime session %s tts pipeline completed: epoch=%s audio_chunks=%s audio_bytes=%s spoken_chunks=%s", + self.session_id, + epoch, + audio_chunk_count, + audio_byte_count, + len(actually_spoken_chunks), + ) + finally: + if not self._closed: + self.transport.clear_buffer() + + async def _iter_tts_text_chunks( + self, + *, + epoch: int, + sentence_queue: asyncio.Queue[str | None], + started_holder: list[float | None], + actually_spoken_chunks: list[str], + ) -> AsyncGenerator[str, None]: while True: self._ensure_generation(epoch) sentence = await sentence_queue.get() @@ -343,34 +1512,118 @@ class CallSession: normalized = sentence.strip() if not normalized: continue - if first_tts_started_monotonic is None: - first_tts_started_monotonic = time.perf_counter() - async for audio_chunk in self._tts.synthesize_stream(normalized): - self._ensure_generation(epoch) - if not audio_chunk: - continue - if not first_audio_seen: - first_audio_seen = True - self._log_latency( - "ttfa", - first_tts_started_monotonic or time.perf_counter(), - epoch=epoch, - message="tts request -> first audio", - ) - self._set_state(SessionState.ASSISTANT_SPEAKING, reason="assistant playback started") - await self.transport.send_audio(audio_chunk) + await self._wait_for_filler_audio() + if started_holder[0] is None: + started_holder[0] = time.perf_counter() + actually_spoken_chunks.append(normalized) + LOGGER.info( + "realtime session %s tts_text_yield: epoch=%s spoken_index=%s chars=%s preview=%r", + self.session_id, + epoch, + len(actually_spoken_chunks), + len(normalized), + _preview_text(normalized), + ) + yield normalized + + async def _wait_for_filler_audio(self) -> None: + filler_task = self._filler_task + if filler_task is None or filler_task.done(): + return + with contextlib.suppress(asyncio.CancelledError): + await filler_task + + def _start_filler_audio( + self, + *, + epoch: int, + tool_name: str | None, + started_monotonic: float, + ) -> None: + if self._filler_task is not None and not self._filler_task.done(): + return + LOGGER.info( + "realtime session %s filler audio scheduled: epoch=%s tool=%s", + self.session_id, + epoch, + tool_name or "generic", + ) + self._filler_task = asyncio.create_task( + self._play_filler_audio( + epoch=epoch, + tool_name=tool_name, + started_monotonic=started_monotonic, + ), + name=f"{self.session_id}-filler-{epoch}", + ) + + async def _play_filler_audio( + self, + *, + epoch: int, + tool_name: str | None, + started_monotonic: float, + ) -> None: + if self._filler_audio is None: + LOGGER.info("realtime session %s filler audio skipped: no library", self.session_id) + return + filler_pcm = self._filler_audio.pick(tool_name) + if not filler_pcm: + LOGGER.info( + "realtime session %s filler audio skipped: no clip tool=%s", + self.session_id, + tool_name or "generic", + ) + return + first_audio_seen = False + sent_chunks = 0 + sent_bytes = 0 + for offset in range(0, len(filler_pcm), max(self.transport.frame_bytes * 4, self.transport.frame_bytes)): + self._ensure_generation(epoch) + audio_chunk = filler_pcm[offset : offset + max(self.transport.frame_bytes * 4, self.transport.frame_bytes)] + if not audio_chunk: + continue + sent_chunks += 1 + sent_bytes += len(audio_chunk) + if not first_audio_seen: + first_audio_seen = True + self._log_latency( + "filler_ttfa", + started_monotonic, + epoch=epoch, + message=f"tool call -> filler audio started ({tool_name or 'generic'})", + ) + self._set_state(SessionState.ASSISTANT_SPEAKING, reason="filler audio playback started") + await self.transport.send_audio(audio_chunk) + await self.transport.flush_audio() + LOGGER.info( + "realtime session %s filler audio completed: epoch=%s tool=%s chunks=%s bytes=%s", + self.session_id, + epoch, + tool_name or "generic", + sent_chunks, + sent_bytes, + ) def _clear_sentence_queue(self, sentence_queue: asyncio.Queue[str | None] | None = None) -> None: queue = sentence_queue if sentence_queue is not None else self._sentence_queue if queue is None: return + removed = 0 while True: try: queue.get_nowait() + removed += 1 except asyncio.QueueEmpty: break with contextlib.suppress(asyncio.QueueFull): queue.put_nowait(None) + LOGGER.info( + "realtime session %s sentence queue cleared: removed=%s owns_queue=%s", + self.session_id, + removed, + queue is self._sentence_queue, + ) if queue is self._sentence_queue: self._sentence_queue = None @@ -387,3 +1640,279 @@ class CallSession: state.value, suffix, ) + + async def _start_live_stt_stream(self) -> None: + await self._cancel_live_stt_stream() + self._reset_semantic_endpointing() + try: + LOGGER.info("realtime session %s live STT stream start requested", self.session_id) + live_stt_stream = await self._stt.start_stream(partial_callback=self._handle_partial_transcript) + except Exception: + LOGGER.exception("realtime session %s failed to start live STT stream", self.session_id) + return + if live_stt_stream is None: + LOGGER.info("realtime session %s live STT stream unavailable; will use batch STT", self.session_id) + return + self._live_stt_stream = live_stt_stream + seed_audio = self._vad.current_utterance_audio() + if seed_audio: + try: + LOGGER.info( + "realtime session %s live STT seed audio: bytes=%s ms=%s", + self.session_id, + len(seed_audio), + _audio_duration_ms(seed_audio, sample_rate_hz=self.transport.sample_rate_hz), + ) + await live_stt_stream.push_audio(seed_audio) + except Exception: + LOGGER.exception("realtime session %s failed to seed live STT stream", self.session_id) + await self._cancel_live_stt_stream() + + async def _push_live_stt_audio(self, audio_chunk: bytes) -> None: + live_stt_stream = self._live_stt_stream + if live_stt_stream is None or not audio_chunk: + return + try: + await live_stt_stream.push_audio(audio_chunk) + except Exception: + LOGGER.exception("realtime session %s live STT push failed", self.session_id) + await self._cancel_live_stt_stream() + + def _detach_live_stt_stream(self) -> BaseSTTStream | None: + live_stt_stream = self._live_stt_stream + self._live_stt_stream = None + self._latest_partial_transcript = "" + self._reset_semantic_endpointing() + LOGGER.info( + "realtime session %s live STT stream detached: present=%s", + self.session_id, + live_stt_stream is not None, + ) + return live_stt_stream + + async def _cancel_live_stt_stream(self) -> None: + live_stt_stream = self._detach_live_stt_stream() + if live_stt_stream is None: + return + LOGGER.info("realtime session %s live STT stream canceling", self.session_id) + with contextlib.suppress(Exception): + await live_stt_stream.cancel() + + async def _resolve_transcript( + self, + *, + stt_stream: BaseSTTStream | None, + utterance_audio: bytes, + ) -> str: + self._maybe_dump_audio(utterance_audio) + if stt_stream is not None: + try: + LOGGER.info("realtime session %s resolving transcript via live STT stream", self.session_id) + transcript = (await stt_stream.finish()).strip() + if transcript: + return transcript + except Exception: + LOGGER.exception("realtime session %s live STT stream failed; falling back to batch STT", self.session_id) + LOGGER.info( + "realtime session %s resolving transcript via batch STT: utterance_bytes=%s utterance_ms=%s", + self.session_id, + len(utterance_audio), + _audio_duration_ms(utterance_audio, sample_rate_hz=self.transport.sample_rate_hz), + ) + return await self._stt.transcribe(utterance_audio) + + def _maybe_dump_audio(self, audio_bytes: bytes) -> None: + if str(os.getenv("ENABLE_AUDIO_DUMP", "")).strip().lower() not in {"1", "true", "yes", "on"}: + return + if not audio_bytes: + return + + try: + dump_dir = os.getenv("AUDIO_DUMP_DIR", "debug_audio").strip() or "debug_audio" + os.makedirs(dump_dir, exist_ok=True) + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") + safe_session_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", self.session_id) + filename = os.path.join(dump_dir, f"utterance_{safe_session_id}_{timestamp}.wav") + dump_sample_rate_hz = 8000 + if self.transport.sample_rate_hz != dump_sample_rate_hz: + LOGGER.warning( + "realtime session %s audio dump writing raw bytes with 8000Hz header while transport sample_rate=%s", + self.session_id, + self.transport.sample_rate_hz, + ) + + with wave.open(filename, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(dump_sample_rate_hz) + wav_file.writeframes(audio_bytes) + + LOGGER.info( + "realtime session %s audio dumped: path=%s bytes=%s sample_rate=%s channels=1 sample_width=16bit", + self.session_id, + filename, + len(audio_bytes), + dump_sample_rate_hz, + ) + except Exception: + LOGGER.exception("realtime session %s failed to dump audio", self.session_id) + + async def _play_cached_initial_greeting( + self, + *, + epoch: int, + greeting_text: str, + pcm_audio: bytes, + started_monotonic: float, + ) -> None: + audio_chunk_count = 0 + audio_byte_count = 0 + step = max(self.transport.frame_bytes * 4, self.transport.frame_bytes) + first_audio_seen = False + try: + for offset in range(0, len(pcm_audio), step): + self._ensure_generation(epoch) + audio_chunk = pcm_audio[offset : offset + step] + if not audio_chunk: + continue + audio_chunk_count += 1 + audio_byte_count += len(audio_chunk) + if not first_audio_seen: + first_audio_seen = True + self._log_latency( + "initial_greeting_cached_ttfa", + started_monotonic, + epoch=epoch, + message="session start -> cached initial greeting first audio", + ) + self._set_state(SessionState.ASSISTANT_SPEAKING, reason="cached initial greeting started") + await self.transport.send_audio(audio_chunk) + await self.transport.flush_audio() + self._ensure_generation(epoch) + LOGGER.info( + "realtime session %s cached initial greeting completed: epoch=%s chunks=%s bytes=%s", + self.session_id, + epoch, + audio_chunk_count, + audio_byte_count, + ) + if self._initial_greeting_discarded_bytes: + LOGGER.info( + "realtime session %s ignored inbound audio during cached initial greeting: bytes=%s ms=%s", + self.session_id, + self._initial_greeting_discarded_bytes, + _audio_byte_count_duration_ms( + self._initial_greeting_discarded_bytes, + sample_rate_hz=self.transport.sample_rate_hz, + ), + ) + self._initial_greeting_discarded_bytes = 0 + if not self._conversation or self._conversation[-1] != ("assistant", greeting_text): + self._conversation.append(("assistant", greeting_text)) + LOGGER.info( + "realtime session %s conversation append cached initial greeting: entries=%s", + self.session_id, + len(self._conversation), + ) + if self.state == SessionState.ASSISTANT_SPEAKING and self._assistant_task is None: + self._set_state(SessionState.LISTENING, reason="cached initial greeting completed") + except asyncio.CancelledError: + raise + except Exception: + LOGGER.exception("realtime session %s cached initial greeting failed", self.session_id) + + async def _handle_partial_transcript(self, partial_text: str) -> None: + normalized = " ".join(str(partial_text or "").strip().lower().split()) + if not normalized or self._closed or self._live_stt_stream is None: + return + self._latest_partial_transcript = normalized + LOGGER.info( + "realtime session %s partial transcript received: chars=%s text=%r", + self.session_id, + len(normalized), + _preview_text(normalized), + ) + target_timeout_ms = ( + self._semantic_hold_silence_timeout_ms + if self._should_hold_endpointing(normalized) + else self._default_vad_silence_timeout_ms + ) + current_timeout_ms = getattr(self._vad, "current_speech_end_silence_ms", self._default_vad_silence_timeout_ms) + if target_timeout_ms == current_timeout_ms: + return + self._vad.set_speech_end_silence_ms(target_timeout_ms) + LOGGER.info( + "realtime session %s semantic endpointing timeout=%sms partial=%r", + self.session_id, + target_timeout_ms, + normalized[-80:], + ) + + def _reset_semantic_endpointing(self) -> None: + self._latest_partial_transcript = "" + self._vad.restore_default_speech_end_silence_ms() + + def _should_hold_endpointing(self, partial_text: str) -> bool: + normalized = partial_text.strip().lower() + if not normalized: + return False + if normalized.endswith(("...", "…", ",", ":", ";", "-", "—")): + return True + if len(normalized) < 12: + return True + words = normalized.split() + if len(words) < 3: + return True + last_word = words[-1].strip(".,!?;:()[]{}\"'") + if last_word in SEMANTIC_CONTINUATION_TOKENS: + return True + if len(last_word) <= 2 and len(words) < 5: + return True + if normalized.endswith((".", "!", "?")): + return False + return len(words) < 6 + + def _commit_assistant_context( + self, + *, + llm_generated_text: str, + actually_spoken_chunks: list[str], + interrupted: bool, + ) -> None: + if self._closed: + return + spoken_text = self._compose_assistant_history_text( + actually_spoken_chunks=actually_spoken_chunks, + fallback_text=llm_generated_text, + interrupted=interrupted, + ) + if not spoken_text: + return + self._conversation.append(("assistant", spoken_text)) + LOGGER.info( + "realtime session %s conversation append assistant: entries=%s interrupted=%s " + "generated_chars=%s spoken_chars=%s spoken_chunks=%s text=%r", + self.session_id, + len(self._conversation), + interrupted, + len(llm_generated_text), + len(spoken_text), + len(actually_spoken_chunks), + _preview_text(spoken_text), + ) + + @staticmethod + def _compose_assistant_history_text( + *, + actually_spoken_chunks: list[str], + fallback_text: str, + interrupted: bool, + ) -> str: + spoken_text = " ".join(chunk.strip() for chunk in actually_spoken_chunks if chunk and chunk.strip()).strip() + if not spoken_text and not interrupted: + spoken_text = fallback_text.strip() + if not spoken_text: + return "" + if interrupted: + return f"{spoken_text} [interrupted]" + return spoken_text diff --git a/core/vad.py b/core/vad.py index ea53fd9..ea2a5a5 100644 --- a/core/vad.py +++ b/core/vad.py @@ -9,6 +9,12 @@ from typing import Any, Callable LOGGER = logging.getLogger("uvicorn.error") +def _duration_ms_from_samples(samples: int, *, sample_rate_hz: int) -> int: + if samples <= 0 or sample_rate_hz <= 0: + return 0 + return int((samples / float(sample_rate_hz)) * 1000.0) + + @dataclass class VADFrameResult: is_speech: bool = False @@ -23,6 +29,18 @@ class BaseVAD(ABC): def feed(self, audio_chunk: bytes) -> VADFrameResult: raise NotImplementedError + @abstractmethod + def set_speech_end_silence_ms(self, value: int) -> None: + raise NotImplementedError + + @abstractmethod + def restore_default_speech_end_silence_ms(self) -> None: + raise NotImplementedError + + @abstractmethod + def current_utterance_audio(self) -> bytes: + raise NotImplementedError + @abstractmethod def flush(self) -> bytes | None: raise NotImplementedError @@ -55,18 +73,39 @@ class SileroVADDetector(BaseVAD): if negative_threshold is not None else max(self.threshold - 0.15, 0.01) ) - self.speech_end_silence_ms = max(speech_end_silence_ms, 32) + self._default_speech_end_silence_ms = max(speech_end_silence_ms, 32) + self.speech_end_silence_ms = self._default_speech_end_silence_ms self.speech_pad_ms = max(speech_pad_ms, 0) self.min_speech_duration_ms = max(min_speech_duration_ms, 0) self.window_samples = 512 if self.sample_rate_hz == 16000 else 256 self.window_bytes = self.window_samples * 2 - self._speech_end_silence_samples = int(self.sample_rate_hz * self.speech_end_silence_ms / 1000.0) self._speech_pad_bytes = int(self.sample_rate_hz * self.speech_pad_ms / 1000.0) * 2 self._min_speech_samples = int(self.sample_rate_hz * self.min_speech_duration_ms / 1000.0) self._model = model self._prediction_fn = prediction_fn self._use_onnx = use_onnx + self._set_speech_end_silence_ms(self.speech_end_silence_ms) self.reset() + LOGGER.info( + "Silero VAD config: sample_rate=%s threshold=%.3f negative_threshold=%.3f " + "silence_timeout_ms=%s speech_pad_ms=%s min_speech_ms=%s window_bytes=%s onnx=%s", + self.sample_rate_hz, + self.threshold, + self.negative_threshold, + self.speech_end_silence_ms, + self.speech_pad_ms, + self.min_speech_duration_ms, + self.window_bytes, + self._use_onnx, + ) + + @property + def default_speech_end_silence_ms(self) -> int: + return self._default_speech_end_silence_ms + + @property + def current_speech_end_silence_ms(self) -> int: + return self.speech_end_silence_ms def reset(self) -> None: self._window_buffer = bytearray() @@ -75,8 +114,19 @@ class SileroVADDetector(BaseVAD): self._triggered = False self._silence_samples = 0 self._speech_samples = 0 + self.restore_default_speech_end_silence_ms() if self._model is not None and hasattr(self._model, "reset_states"): self._model.reset_states() + LOGGER.info("Silero VAD reset: sample_rate=%s silence_timeout_ms=%s", self.sample_rate_hz, self.speech_end_silence_ms) + + def set_speech_end_silence_ms(self, value: int) -> None: + self._set_speech_end_silence_ms(value) + + def restore_default_speech_end_silence_ms(self) -> None: + self._set_speech_end_silence_ms(self._default_speech_end_silence_ms) + + def current_utterance_audio(self) -> bytes: + return bytes(self._utterance_audio) def feed(self, audio_chunk: bytes) -> VADFrameResult: result = VADFrameResult() @@ -101,6 +151,12 @@ class SileroVADDetector(BaseVAD): self._pre_speech_audio.clear() result.speech_started = True result.is_speech = True + LOGGER.info( + "Silero VAD speech_started: probability=%.3f pre_speech_bytes=%s window_bytes=%s", + speech_probability, + len(self._utterance_audio) - len(window), + len(window), + ) else: self._append_pre_speech_window(window) continue @@ -121,24 +177,41 @@ class SileroVADDetector(BaseVAD): if self._silence_samples < self._speech_end_silence_samples: continue - utterance_audio = bytes(self._utterance_audio) + utterance_audio = self._trim_trailing_silence( + bytes(self._utterance_audio), + trailing_silence_samples=self._silence_samples, + ) speech_samples = self._speech_samples self._reset_segment() result.is_speech = False - if speech_samples >= self._min_speech_samples: + if speech_samples >= self._min_speech_samples and utterance_audio: result.speech_ended = True result.utterance_audio = utterance_audio + LOGGER.info( + "Silero VAD speech_ended: speech_ms=%s trailing_silence_ms=%s utterance_bytes=%s", + _duration_ms_from_samples(speech_samples, sample_rate_hz=self.sample_rate_hz), + _duration_ms_from_samples(self._speech_end_silence_samples, sample_rate_hz=self.sample_rate_hz), + len(utterance_audio), + ) return result def flush(self) -> bytes | None: if not self._triggered or not self._utterance_audio: return None - utterance_audio = bytes(self._utterance_audio) + utterance_audio = self._trim_trailing_silence( + bytes(self._utterance_audio), + trailing_silence_samples=self._silence_samples, + ) speech_samples = self._speech_samples self._reset_segment() - if speech_samples < self._min_speech_samples: + if speech_samples < self._min_speech_samples or not utterance_audio: return None + LOGGER.info( + "Silero VAD flush utterance: speech_ms=%s utterance_bytes=%s", + _duration_ms_from_samples(speech_samples, sample_rate_hz=self.sample_rate_hz), + len(utterance_audio), + ) return utterance_audio def _reset_segment(self) -> None: @@ -157,6 +230,33 @@ class SileroVADDetector(BaseVAD): if overflow > 0: del self._pre_speech_audio[:overflow] + def _trim_trailing_silence(self, utterance_audio: bytes, *, trailing_silence_samples: int) -> bytes: + if not utterance_audio or trailing_silence_samples <= 0: + return utterance_audio + trim_bytes = min(trailing_silence_samples * 2, len(utterance_audio)) + if trim_bytes <= 0: + return utterance_audio + LOGGER.info( + "Silero VAD trim trailing silence: trim_bytes=%s trim_ms=%s before_bytes=%s after_bytes=%s", + trim_bytes, + _duration_ms_from_samples(trailing_silence_samples, sample_rate_hz=self.sample_rate_hz), + len(utterance_audio), + len(utterance_audio) - trim_bytes, + ) + return utterance_audio[:-trim_bytes] + + def _set_speech_end_silence_ms(self, value: int) -> None: + previous = getattr(self, "speech_end_silence_ms", None) + self.speech_end_silence_ms = max(int(value), 32) + self._speech_end_silence_samples = int(self.sample_rate_hz * self.speech_end_silence_ms / 1000.0) + if previous != self.speech_end_silence_ms: + LOGGER.info( + "Silero VAD silence timeout changed: previous_ms=%s current_ms=%s samples=%s", + previous, + self.speech_end_silence_ms, + self._speech_end_silence_samples, + ) + def _predict_speech_probability(self, window: bytes) -> float: if self._prediction_fn is not None: return max(0.0, min(float(self._prediction_fn(window)), 1.0)) diff --git a/docker-compose.yml b/docker-compose.yml index 49befc8..ebb370f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,4 +9,11 @@ services: ports: - "${REALTIME_VOICE_HTTP_PORT:-8000}:${REALTIME_VOICE_HTTP_PORT:-8000}" - "${REALTIME_VOICE_AUDIOSOCKET_PORT:-9092}:${REALTIME_VOICE_AUDIOSOCKET_PORT:-9092}" + - "${REALTIME_VOICE_AUDIOSOCKET_ALIAS_PORT:-9019}:${REALTIME_VOICE_AUDIOSOCKET_PORT:-9092}" + dns: + - 1.1.1.1 + - 8.8.8.8 + extra_hosts: + - "api.elevenlabs.io:34.8.184.191" + - "host.docker.internal:host-gateway" restart: unless-stopped diff --git a/main.py b/main.py index 6af5cf1..0419722 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import inspect import logging import os import uuid @@ -8,11 +9,10 @@ from contextlib import asynccontextmanager from fastapi import FastAPI, WebSocket +from realtime_voice_service.core.filler_audio import FillerAudioLibrary from realtime_voice_service.core.session import CallSession from realtime_voice_service.core.vad import SileroVADDetector -from realtime_voice_service.providers.llm import OpenAILLM -from realtime_voice_service.providers.stt import ElevenLabsSTT -from realtime_voice_service.providers.tts import ElevenLabsTTS +from realtime_voice_service.providers.factory import create_llm_provider, create_stt_provider, create_tts_provider from realtime_voice_service.transports.audiosocket import AudioSocketServer from realtime_voice_service.transports.base import BaseMediaTransport from realtime_voice_service.transports.websocket import WebSocketMediaTransport @@ -69,6 +69,11 @@ def _audiosocket_port() -> int: return max(_int_env("REALTIME_VOICE_AUDIOSOCKET_PORT", 9092), 1) +def _sample_rate_hz() -> int: + configured = max(_int_env("REALTIME_VOICE_SAMPLE_RATE_HZ", 8000), 1) + return configured if configured in {8000, 16000, 24000} else 8000 + + def _http_host() -> str: return str(os.getenv("REALTIME_VOICE_HTTP_HOST", "0.0.0.0") or "0.0.0.0").strip() @@ -77,38 +82,119 @@ def _http_port() -> int: return max(_int_env("REALTIME_VOICE_HTTP_PORT", 8000), 1) +def _initial_greeting_text() -> str: + raw = os.getenv("REALTIME_VOICE_INITIAL_GREETING_TEXT") + if raw is None: + return "Здравствуйте! Чем могу помочь?" + return str(raw).strip() + +def _llm_provider_name() -> str: + return str(os.getenv("LLM_PROVIDER") or os.getenv("REALTIME_VOICE_LLM_PROVIDER") or "openai").strip().lower() + + +def _llm_model_name() -> str: + provider = _llm_provider_name() + if provider in {"ollama", "local", "qwen"}: + return str(os.getenv("OLLAMA_LLM_MODEL", "qwen2.5:1.5b")).strip() or "qwen2.5:1.5b" + return str(os.getenv("OPENAI_LLM_MODEL", "gpt-4o-mini")).strip() or "gpt-4o-mini" + + + class RealtimeVoiceService: def __init__(self) -> None: - self._stt = ElevenLabsSTT() - self._llm = OpenAILLM() - self._tts = ElevenLabsTTS() + self._sample_rate_hz = _sample_rate_hz() + self._stt = create_stt_provider( + input_sample_rate_hz=self._sample_rate_hz, + target_sample_rate_hz=self._sample_rate_hz, + ) + self._llm = create_llm_provider() + self._tts = create_tts_provider( + output_format=f"pcm_{self._sample_rate_hz}", + target_sample_rate_hz=self._sample_rate_hz, + ) + self._filler_audio = FillerAudioLibrary(sample_rate_hz=self._sample_rate_hz) self._audiosocket_server = AudioSocketServer( host=_audiosocket_host(), port=_audiosocket_port(), + sample_rate_hz=self._sample_rate_hz, session_handler=self._run_transport_session, ) self._active_sessions: dict[str, CallSession] = {} self._session_lock = asyncio.Lock() + LOGGER.info( + "realtime voice service config: sample_rate=%s audiosocket=%s:%s http=%s:%s " + "llm_provider=%s llm_model=%s tools_enabled=%s serper_configured=%s stt_provider=%s stt_model=%s stt_realtime_model=%s " + "tts_voice_id=%s tts_model=%s tts_format=%s tts_speed=%s", + self._sample_rate_hz, + _audiosocket_host(), + _audiosocket_port(), + _http_host(), + _http_port(), + _llm_provider_name(), + _llm_model_name(), + os.getenv("OPENAI_LLM_ENABLE_TOOLS", "true"), + bool(os.getenv("SERPER_API_KEY")), + os.getenv("STT_PROVIDER") or os.getenv("REALTIME_VOICE_STT_PROVIDER") or os.getenv("AI_VOICE_ASR_PROVIDER", "elevenlabs"), + os.getenv("ELEVENLABS_STT_MODEL_ID", "scribe_v2"), + os.getenv("ELEVENLABS_STT_REALTIME_MODEL_ID", "scribe_v2_realtime"), + os.getenv("ELEVENLABS_TTS_VOICE_ID", ""), + os.getenv("ELEVENLABS_TTS_MODEL_ID", "eleven_turbo_v2_5"), + f"pcm_{self._sample_rate_hz}", + os.getenv("ELEVENLABS_TTS_SPEED", "1.0"), + ) @property def active_session_count(self) -> int: return len(self._active_sessions) async def start(self) -> None: + LOGGER.info("realtime voice service starting") await self._audiosocket_server.start() + try: + LOGGER.info("preloading filler audio library") + await self._filler_audio.preload(tts=self._tts) + await self._preload_initial_greeting_audio() + LOGGER.info("filler audio library preload completed") + except Exception: + LOGGER.exception("failed to preload filler audio clips") + + async def _preload_initial_greeting_audio(self) -> None: + greeting_text = _initial_greeting_text() + if not greeting_text: + LOGGER.info("initial greeting audio preload skipped: empty text") + return + try: + LOGGER.info( + "initial greeting audio preload start: chars=%s text=%r", + len(greeting_text), + greeting_text[:120], + ) + clip = await self._filler_audio.synthesize_text(self._tts, greeting_text) + if not clip: + LOGGER.warning("initial greeting audio preload produced empty clip") + return + self._filler_audio.add_clip("initial_greeting", clip) + LOGGER.info("initial greeting audio preload done: bytes=%s", len(clip)) + except Exception: + LOGGER.exception("failed to preload initial greeting audio") async def stop(self) -> None: + LOGGER.info("realtime voice service stopping active_sessions=%s", len(self._active_sessions)) await self._audiosocket_server.stop() sessions = list(self._active_sessions.values()) for session in sessions: await session.stop() self._active_sessions.clear() + await self._close_provider(self._stt) + await self._close_provider(self._llm) + await self._close_provider(self._tts) async def handle_websocket(self, websocket: WebSocket, *, client_id: str | None = None) -> None: await websocket.accept() transport = WebSocketMediaTransport( websocket=websocket, transport_id=client_id or str(uuid.uuid4()), + sample_rate_hz=self._sample_rate_hz, ) await self._run_transport_session(transport) @@ -116,9 +202,12 @@ class RealtimeVoiceService: session = self._build_session(transport) async with self._track_session(session): LOGGER.info( - "starting realtime session %s via %s", + "starting realtime session %s via %s sample_rate=%s frame_ms=%s frame_bytes=%s", session.session_id, transport.protocol, + transport.sample_rate_hz, + transport.frame_duration_ms, + transport.frame_bytes, ) await session.run() @@ -130,7 +219,7 @@ class RealtimeVoiceService: sample_rate_hz=transport.sample_rate_hz, threshold=_float_env("VAD_THRESHOLD", 0.5), negative_threshold=_optional_float_env("VAD_NEGATIVE_THRESHOLD"), - speech_end_silence_ms=_int_env("VAD_SILENCE_TIMEOUT_MS", 1600), + speech_end_silence_ms=_int_env("VAD_SILENCE_TIMEOUT_MS", 550), speech_pad_ms=_int_env("VAD_SPEECH_PAD_MS", 64), min_speech_duration_ms=_int_env("VAD_MIN_SPEECH_DURATION_MS", 0), use_onnx=_bool_env("VAD_USE_ONNX", False), @@ -138,17 +227,38 @@ class RealtimeVoiceService: stt=self._stt, llm=self._llm, tts=self._tts, + filler_audio=self._filler_audio, + initial_greeting_text=_initial_greeting_text(), ) + @staticmethod + async def _close_provider(provider: object) -> None: + close = getattr(provider, "close", None) + if close is None: + return + result = close() + if inspect.isawaitable(result): + await result + @asynccontextmanager async def _track_session(self, session: CallSession): async with self._session_lock: self._active_sessions[session.session_id] = session + LOGGER.info( + "session tracked: session=%s active_sessions=%s", + session.session_id, + len(self._active_sessions), + ) try: yield finally: async with self._session_lock: self._active_sessions.pop(session.session_id, None) + LOGGER.info( + "session untracked: session=%s active_sessions=%s", + session.session_id, + len(self._active_sessions), + ) service = RealtimeVoiceService() diff --git a/providers/__init__.py b/providers/__init__.py index 95a21fa..60bea5a 100644 --- a/providers/__init__.py +++ b/providers/__init__.py @@ -1,7 +1,9 @@ from realtime_voice_service.providers.base import BaseLLM, BaseSTT, BaseTTS, MockLLM, MockSTT, MockTTS from realtime_voice_service.providers.llm import OpenAILLM from realtime_voice_service.providers.stt import ElevenLabsSTT +from realtime_voice_service.providers.stt_openai import OpenAISTT from realtime_voice_service.providers.tts import ElevenLabsTTS +from realtime_voice_service.providers.factory import create_stt_provider, create_tts_provider __all__ = [ "BaseLLM", @@ -13,4 +15,7 @@ __all__ = [ "MockSTT", "MockTTS", "OpenAILLM", + "OpenAISTT", + "create_stt_provider", + "create_tts_provider", ] diff --git a/providers/base.py b/providers/base.py index 75c2f85..ef228cc 100644 --- a/providers/base.py +++ b/providers/base.py @@ -6,6 +6,27 @@ import re import struct from abc import ABC, abstractmethod from collections.abc import AsyncGenerator +from collections.abc import AsyncIterable +from collections.abc import Awaitable +from collections.abc import Callable +from dataclasses import dataclass + + +PartialTranscriptCallback = Callable[[str], Awaitable[None] | None] + + +class BaseSTTStream(ABC): + @abstractmethod + async def push_audio(self, audio_chunk: bytes) -> None: + raise NotImplementedError + + @abstractmethod + async def finish(self) -> str: + raise NotImplementedError + + @abstractmethod + async def cancel(self) -> None: + raise NotImplementedError class BaseSTT(ABC): @@ -13,25 +34,41 @@ class BaseSTT(ABC): async def transcribe(self, audio_bytes: bytes) -> str: raise NotImplementedError + async def start_stream( + self, + *, + partial_callback: PartialTranscriptCallback | None = None, + ) -> BaseSTTStream | None: + del partial_callback + return None + class BaseLLM(ABC): @abstractmethod - async def generate_stream(self, text: str, context: list) -> AsyncGenerator[str, None]: + async def generate_stream(self, text: str, context: list) -> AsyncGenerator[LLMStreamEvent, None]: raise NotImplementedError class BaseTTS(ABC): @abstractmethod - async def synthesize_stream(self, text: str) -> AsyncGenerator[bytes, None]: + async def synthesize_stream(self, text_stream: AsyncIterable[str]) -> AsyncGenerator[bytes, None]: raise NotImplementedError +@dataclass(slots=True) +class LLMStreamEvent: + type: str + content: str | None = None + name: str | None = None + tool_call_id: str | None = None + + class MockSTT(BaseSTT): def __init__( self, *, latency_ms: int = 40, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, scripted_transcripts: list[str] | None = None, ) -> None: self._latency_ms = max(latency_ms, 0) @@ -47,6 +84,13 @@ class MockSTT(BaseSTT): duration_ms = int(((len(audio_bytes) // 2) / float(self._sample_rate_hz)) * 1000.0) return f"mock user utterance {self._call_count} ({duration_ms} ms)" + async def start_stream( + self, + *, + partial_callback: PartialTranscriptCallback | None = None, + ) -> BaseSTTStream | None: + return _MockSTTStream(parent=self, partial_callback=partial_callback) + class MockLLM(BaseLLM): def __init__( @@ -58,7 +102,7 @@ class MockLLM(BaseLLM): self._token_delay_ms = max(token_delay_ms, 0) self._scripted_responses = list(scripted_responses or []) - async def generate_stream(self, text: str, context: list) -> AsyncGenerator[str, None]: + async def generate_stream(self, text: str, context: list) -> AsyncGenerator[LLMStreamEvent, None]: del context response_text = ( self._scripted_responses.pop(0) @@ -68,14 +112,14 @@ class MockLLM(BaseLLM): chunks = re.findall(r"\S+\s*", response_text) or [response_text] for chunk in chunks: await asyncio.sleep(self._token_delay_ms / 1000.0) - yield chunk + yield LLMStreamEvent(type="text", content=chunk) class MockTTS(BaseTTS): def __init__( self, *, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, chunk_duration_ms: int = 40, chunk_delay_ms: int = 15, tone_hz: float = 440.0, @@ -89,18 +133,60 @@ class MockTTS(BaseTTS): self._amplitude = amplitude self._milliseconds_per_word = max(milliseconds_per_word, 40) - async def synthesize_stream(self, text: str) -> AsyncGenerator[bytes, None]: - word_count = max(len(text.split()), 1) - total_ms = min(word_count * self._milliseconds_per_word, 2400) - total_samples = max(int(self._sample_rate_hz * (total_ms / 1000.0)), 1) - chunk_samples = max(int(self._sample_rate_hz * (self._chunk_duration_ms / 1000.0)), 1) + async def synthesize_stream(self, text_stream: AsyncIterable[str]) -> AsyncGenerator[bytes, None]: + async for text in text_stream: + normalized = text.strip() + if not normalized: + continue + word_count = max(len(normalized.split()), 1) + total_ms = min(word_count * self._milliseconds_per_word, 2400) + total_samples = max(int(self._sample_rate_hz * (total_ms / 1000.0)), 1) + chunk_samples = max(int(self._sample_rate_hz * (self._chunk_duration_ms / 1000.0)), 1) - for start in range(0, total_samples, chunk_samples): - end = min(start + chunk_samples, total_samples) - pcm = bytearray() - for index in range(start, end): - angle = 2.0 * math.pi * self._tone_hz * (index / float(self._sample_rate_hz)) - sample = int(self._amplitude * math.sin(angle)) - pcm.extend(struct.pack(" None: + self._parent = parent + self._partial_callback = partial_callback + self._audio_buffer = bytearray() + self._cancelled = False + self._partial_emitted = False + + async def push_audio(self, audio_chunk: bytes) -> None: + if self._cancelled or not audio_chunk: + return + self._audio_buffer.extend(audio_chunk) + if self._partial_callback is None or self._partial_emitted: + return + duration_ms = int(((len(self._audio_buffer) // 2) / float(self._parent._sample_rate_hz)) * 1000.0) + if duration_ms < 600: + return + self._partial_emitted = True + partial = f"mock partial utterance {self._parent._call_count + 1}" + maybe_awaitable = self._partial_callback(partial) + if maybe_awaitable is not None: + await maybe_awaitable + + async def finish(self) -> str: + if self._cancelled: + return "" + return await self._parent.transcribe(bytes(self._audio_buffer)) + + async def cancel(self) -> None: + self._cancelled = True + self._audio_buffer.clear() diff --git a/providers/factory.py b/providers/factory.py new file mode 100644 index 0000000..c968d84 --- /dev/null +++ b/providers/factory.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import logging +import os + +from realtime_voice_service.providers.base import BaseLLM, BaseSTT, BaseTTS +from realtime_voice_service.providers.llm import OllamaLLM, OpenAILLM +from realtime_voice_service.providers.stt import ElevenLabsSTT, FallbackSTT, YandexSpeechKitSTT +from realtime_voice_service.providers.stt_openai import OpenAISTT +from realtime_voice_service.providers.tts import ElevenLabsTTS + + +LOGGER = logging.getLogger("uvicorn.error") + + +def _first_env(names: tuple[str, ...], default: str = "") -> str: + for name in names: + value = os.getenv(name) + if value is not None and value.strip(): + return value.strip() + return default + + +def _stt_provider_name() -> str: + return _first_env( + ( + "STT_PROVIDER", + "REALTIME_VOICE_STT_PROVIDER", + "AI_VOICE_ASR_PROVIDER", + ), + "elevenlabs", + ).lower() + + +def _stt_fallback_provider_name() -> str: + return _first_env( + ( + "STT_FALLBACK_PROVIDER", + "REALTIME_VOICE_STT_FALLBACK_PROVIDER", + ) + ).lower() + + +def _llm_provider_name() -> str: + return _first_env(("LLM_PROVIDER", "REALTIME_VOICE_LLM_PROVIDER"), "openai").lower() + + +def _tts_provider_name() -> str: + return _first_env(("TTS_PROVIDER", "REALTIME_VOICE_TTS_PROVIDER"), "elevenlabs").lower() + + +def create_llm_provider() -> BaseLLM: + provider = _llm_provider_name() + if provider in {"openai", "openai_chat", "gpt"}: + LOGGER.info("LLM provider selected: provider=%s", provider) + return OpenAILLM() + if provider in {"ollama", "local", "qwen"}: + LOGGER.info("LLM provider selected: provider=%s", provider) + return OllamaLLM() + raise RuntimeError(f"Unsupported LLM provider: {provider}") + + +def _build_single_stt_provider( + provider: str, + *, + input_sample_rate_hz: int, + target_sample_rate_hz: int, +) -> BaseSTT: + normalized = provider.lower().strip() + if normalized in {"openai", "whisper", "openai_whisper"}: + # Whisper accepts an 8 kHz WAV container; keep the exact AudioSocket PCM, no resampling. + return OpenAISTT(input_sample_rate_hz=input_sample_rate_hz) + if normalized in {"elevenlabs", "eleven_labs", "scribe"}: + return ElevenLabsSTT( + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + if normalized in {"yandex", "yandex_speechkit", "speechkit"}: + return YandexSpeechKitSTT( + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + raise RuntimeError(f"Unsupported STT provider: {provider}") + + +def create_stt_provider( + *, + input_sample_rate_hz: int, + target_sample_rate_hz: int, +) -> BaseSTT: + provider = _stt_provider_name() + primary = _build_single_stt_provider( + provider, + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + fallback_provider = _stt_fallback_provider_name() + if not fallback_provider or fallback_provider == provider: + LOGGER.info("STT provider selected: provider=%s", provider) + return primary + + fallback = _build_single_stt_provider( + fallback_provider, + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + LOGGER.info("STT provider selected: provider=%s fallback=%s", provider, fallback_provider) + return FallbackSTT(primary=primary, fallback=fallback) + + +def create_tts_provider( + *, + target_sample_rate_hz: int, + output_format: str | None = None, +) -> BaseTTS: + provider = _tts_provider_name() + if provider in {"elevenlabs", "eleven_labs"}: + LOGGER.info("TTS provider selected: provider=%s", provider) + return ElevenLabsTTS( + output_format=output_format or f"pcm_{target_sample_rate_hz}", + target_sample_rate_hz=target_sample_rate_hz, + ) + raise RuntimeError(f"Unsupported TTS provider: {provider}") diff --git a/providers/llm.py b/providers/llm.py index e36a524..5517c12 100644 --- a/providers/llm.py +++ b/providers/llm.py @@ -1,10 +1,26 @@ from __future__ import annotations +import asyncio +import inspect +import json +import logging import os +import time from collections.abc import AsyncGenerator from typing import Any from realtime_voice_service.providers.base import BaseLLM +from realtime_voice_service.providers.base import LLMStreamEvent + + +LOGGER = logging.getLogger("uvicorn.error") + + +def _preview_text(text: str, *, limit: int = 160) -> str: + normalized = " ".join(str(text or "").split()) + if len(normalized) <= limit: + return normalized + return f"{normalized[:limit]}..." def _timeout_seconds() -> float: @@ -17,6 +33,33 @@ def _timeout_seconds() -> float: return 30.0 +def _max_context_messages() -> int: + raw = os.getenv("OPENAI_LLM_MAX_CONTEXT_MESSAGES") + if raw is None: + return 8 + try: + return max(int(raw.strip()), 0) + except ValueError: + return 8 + + +def _read_bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def _read_int_env(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + return int(str(raw).strip()) + except ValueError: + return default + + class OpenAILLM(BaseLLM): def __init__( self, @@ -28,6 +71,12 @@ class OpenAILLM(BaseLLM): timeout_seconds: float | None = None, temperature: float = 0.3, max_retries: int = 2, + max_context_messages: int | None = None, + reasoning_effort: str | None = None, + enable_tools: bool | None = None, + max_tool_roundtrips: int | None = None, + serper_api_key: str | None = None, + serper_api_base: str | None = None, ) -> None: self._api_key = str(api_key if api_key is not None else os.getenv("OPENAI_API_KEY", "")).strip() self._model = str(model or os.getenv("OPENAI_LLM_MODEL", "gpt-4o-mini")).strip() or "gpt-4o-mini" @@ -42,33 +91,225 @@ class OpenAILLM(BaseLLM): ).strip() self._timeout_seconds = max(float(timeout_seconds if timeout_seconds is not None else _timeout_seconds()), 1.0) self._temperature = max(min(float(temperature), 2.0), 0.0) + self._reasoning_effort = str( + reasoning_effort if reasoning_effort is not None else os.getenv("OPENAI_LLM_REASONING_EFFORT", "") + ).strip().lower() self._max_retries = max(int(max_retries), 0) + self._max_context_messages = max( + int(max_context_messages if max_context_messages is not None else _max_context_messages()), + 0, + ) + self._enable_tools = ( + _read_bool_env("OPENAI_LLM_ENABLE_TOOLS", True) + if enable_tools is None + else bool(enable_tools) + ) + self._max_tool_roundtrips = max( + int(max_tool_roundtrips if max_tool_roundtrips is not None else _read_int_env("OPENAI_LLM_MAX_TOOL_ROUNDTRIPS", 2)), + 0, + ) + self._serper_api_key = str( + serper_api_key if serper_api_key is not None else os.getenv("SERPER_API_KEY", "") + ).strip() + self._serper_api_base = ( + str(serper_api_base or os.getenv("SERPER_API_BASE", "https://google.serper.dev")).strip().rstrip("/") + or "https://google.serper.dev" + ) self._client: Any | None = None self._openai_module: Any | None = None + self._serper_session = None + self._serper_session_lock = asyncio.Lock() + LOGGER.info( + "OpenAI LLM config: model=%s base_url=%s timeout=%s max_context_messages=%s " + "reasoning_effort=%s tools_enabled=%s serper_configured=%s max_tool_roundtrips=%s", + self._model, + self._base_url or "default", + self._timeout_seconds, + self._max_context_messages, + self._reasoning_effort or "default", + self._enable_tools, + bool(self._serper_api_key), + self._max_tool_roundtrips, + ) - async def generate_stream(self, text: str, context: list) -> AsyncGenerator[str, None]: + async def generate_stream(self, text: str, context: list) -> AsyncGenerator[LLMStreamEvent, None]: if not self._api_key: raise RuntimeError("OPENAI_API_KEY is required for OpenAI LLM") - client = self._get_client() messages = self._build_messages(text, context) - try: - stream = await client.chat.completions.create( - model=self._model, + turn_started_monotonic = time.perf_counter() + LOGGER.info( + "OpenAI LLM turn start: model=%s input_chars=%s context_entries=%s messages=%s " + "tools_available=%s input_preview=%r", + self._model, + len(text), + len(context), + len(messages), + bool(self._build_tools()), + _preview_text(text), + ) + for round_index in range(self._max_tool_roundtrips + 1): + tool_buffers: dict[int, dict[str, str]] = {} + announced_tool_indexes: set[int] = set() + text_event_count = 0 + text_char_count = 0 + round_started_monotonic = time.perf_counter() + async for event in self._stream_completion( messages=messages, - temperature=self._temperature, - stream=True, + tool_buffers=tool_buffers, + announced_tool_indexes=announced_tool_indexes, + ): + if event.type == "text": + content = str(event.content or "") + text_event_count += 1 + text_char_count += len(content) + if text_event_count == 1 or text_event_count % 20 == 0: + LOGGER.info( + "OpenAI LLM text stream: round=%s events=%s chars=%s latest=%r", + round_index, + text_event_count, + text_char_count, + _preview_text(content, limit=80), + ) + elif event.type == "tool_call_start": + LOGGER.info( + "OpenAI LLM tool_call_start: round=%s name=%s tool_call_id=%s", + round_index, + event.name, + event.tool_call_id, + ) + yield event + LOGGER.info( + "OpenAI LLM stream round completed: round=%s text_events=%s text_chars=%s " + "tool_calls=%s latency_ms=%s", + round_index, + text_event_count, + text_char_count, + len(tool_buffers), + int((time.perf_counter() - round_started_monotonic) * 1000.0), ) + if not tool_buffers: + LOGGER.info( + "OpenAI LLM turn completed: total_latency_ms=%s", + int((time.perf_counter() - turn_started_monotonic) * 1000.0), + ) + return + + assistant_tool_calls = self._finalize_tool_calls(tool_buffers) + if not assistant_tool_calls: + LOGGER.warning("OpenAI LLM produced tool buffer without finalized tool calls") + return + messages.append( + { + "role": "assistant", + "content": None, + "tool_calls": assistant_tool_calls, + } + ) + tool_messages = await self._execute_tool_calls(assistant_tool_calls) + messages.extend(tool_messages) + LOGGER.warning( + "OpenAI LLM max tool roundtrips reached: max_tool_roundtrips=%s total_latency_ms=%s", + self._max_tool_roundtrips, + int((time.perf_counter() - turn_started_monotonic) * 1000.0), + ) + final_text_event_count = 0 + final_text_char_count = 0 + final_round_started_monotonic = time.perf_counter() + async for event in self._stream_completion( + messages=messages, + tool_buffers={}, + announced_tool_indexes=set(), + enable_tools=False, + ): + if event.type == "text": + content = str(event.content or "") + final_text_event_count += 1 + final_text_char_count += len(content) + if final_text_event_count == 1 or final_text_event_count % 20 == 0: + LOGGER.info( + "OpenAI LLM final no-tool stream: events=%s chars=%s latest=%r", + final_text_event_count, + final_text_char_count, + _preview_text(content, limit=80), + ) + yield event + LOGGER.info( + "OpenAI LLM final no-tool round completed: text_events=%s text_chars=%s latency_ms=%s total_latency_ms=%s", + final_text_event_count, + final_text_char_count, + int((time.perf_counter() - final_round_started_monotonic) * 1000.0), + int((time.perf_counter() - turn_started_monotonic) * 1000.0), + ) + + async def close(self) -> None: + client = self._client + self._client = None + if client is not None and hasattr(client, "close"): + result = client.close() + if inspect.isawaitable(result): + await result + session = self._serper_session + self._serper_session = None + if session is not None and not session.closed: + await session.close() + + async def _stream_completion( + self, + *, + messages: list[dict[str, Any]], + tool_buffers: dict[int, dict[str, str]], + announced_tool_indexes: set[int], + enable_tools: bool = True, + ) -> AsyncGenerator[LLMStreamEvent, None]: + client = self._get_client() + request_kwargs: dict[str, Any] = { + "model": self._model, + "messages": messages, + "stream": True, + } + if self._supports_custom_temperature(): + request_kwargs["temperature"] = self._temperature + tools = self._build_tools() if enable_tools else None + reasoning_effort = self._reasoning_effort + if tools and self._model.lower().startswith("gpt-5.5"): + reasoning_effort = "" + if reasoning_effort: + request_kwargs["reasoning_effort"] = reasoning_effort + if tools: + request_kwargs["tools"] = tools + request_kwargs["tool_choice"] = "auto" + LOGGER.info( + "OpenAI LLM stream request: model=%s messages=%s tools=%s temperature=%s reasoning_effort=%s enable_tools=%s", + self._model, + len(messages), + len(tools or []), + self._temperature if self._supports_custom_temperature() else "default", + reasoning_effort or "default", + enable_tools, + ) + + try: + stream = await client.chat.completions.create(**request_kwargs) async for chunk in stream: choices = getattr(chunk, "choices", None) or [] if not choices: continue - delta = getattr(choices[0], "delta", None) + choice = choices[0] + delta = getattr(choice, "delta", None) if delta is None: continue content = getattr(delta, "content", None) if content: - yield str(content) + yield LLMStreamEvent(type="text", content=str(content)) + tool_calls = getattr(delta, "tool_calls", None) or [] + for tool_delta in tool_calls: + for event in self._consume_tool_delta( + tool_delta=tool_delta, + tool_buffers=tool_buffers, + announced_tool_indexes=announced_tool_indexes, + ): + yield event except Exception as exc: # noqa: BLE001 openai_module = self._openai_module if openai_module is not None and isinstance(exc, getattr(openai_module, "APITimeoutError", ())): @@ -82,6 +323,198 @@ class OpenAILLM(BaseLLM): raise RuntimeError("OpenAI LLM connection failed") from exc raise RuntimeError("OpenAI LLM streaming failed") from exc + def _consume_tool_delta( + self, + *, + tool_delta: Any, + tool_buffers: dict[int, dict[str, str]], + announced_tool_indexes: set[int], + ) -> list[LLMStreamEvent]: + index = int(getattr(tool_delta, "index", 0) or 0) + state = tool_buffers.setdefault(index, {"id": "", "name": "", "arguments": ""}) + tool_id = getattr(tool_delta, "id", None) + if tool_id: + state["id"] = str(tool_id) + function = getattr(tool_delta, "function", None) + if function is not None: + function_name = getattr(function, "name", None) + if function_name: + state["name"] = str(function_name) + function_arguments = getattr(function, "arguments", None) + if function_arguments: + state["arguments"] += str(function_arguments) + if state["name"] and index not in announced_tool_indexes: + announced_tool_indexes.add(index) + return [ + LLMStreamEvent( + type="tool_call_start", + name=state["name"], + tool_call_id=state["id"] or f"tool-call-{index}", + ) + ] + return [] + + async def _execute_tool_calls(self, assistant_tool_calls: list[dict[str, Any]]) -> list[dict[str, str]]: + LOGGER.info("OpenAI LLM executing tool calls: count=%s", len(assistant_tool_calls)) + results = await asyncio.gather( + *(self._execute_tool_call(tool_call) for tool_call in assistant_tool_calls), + return_exceptions=False, + ) + tool_messages: list[dict[str, str]] = [] + for tool_call, tool_result in zip(assistant_tool_calls, results, strict=False): + tool_messages.append( + { + "role": "tool", + "tool_call_id": str(tool_call["id"]), + "content": tool_result, + } + ) + return tool_messages + + async def _execute_tool_call(self, tool_call: dict[str, Any]) -> str: + function = tool_call.get("function") or {} + name = str(function.get("name") or "").strip().lower() + raw_arguments = str(function.get("arguments") or "{}") + try: + arguments = json.loads(raw_arguments) + except json.JSONDecodeError: + arguments = {} + + started_monotonic = time.perf_counter() + LOGGER.info( + "OpenAI LLM tool execution start: name=%s tool_call_id=%s args=%s", + name, + tool_call.get("id"), + raw_arguments[:500], + ) + if name == "serper": + result = await self._run_serper_tool(arguments) + else: + result = f"Tool `{name}` is not supported by this runtime." + LOGGER.info( + "OpenAI LLM tool execution done: name=%s tool_call_id=%s latency_ms=%s result_chars=%s", + name, + tool_call.get("id"), + int((time.perf_counter() - started_monotonic) * 1000.0), + len(result), + ) + return result + + async def _run_serper_tool(self, arguments: dict[str, Any]) -> str: + if not self._serper_api_key: + return "Serper API is unavailable: SERPER_API_KEY is not configured." + query = str(arguments.get("query") or arguments.get("q") or "").strip() + if not query: + return "Serper API error: empty search query." + + try: + import aiohttp + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `aiohttp` package is required for Serper tool execution") from exc + + session = await self._get_serper_session() + request_payload = { + "q": query, + "gl": str(arguments.get("gl") or os.getenv("SERPER_SEARCH_GL", "kz")).strip(), + "hl": str(arguments.get("hl") or os.getenv("SERPER_SEARCH_HL", "ru")).strip(), + "num": max(int(arguments.get("num") or os.getenv("SERPER_SEARCH_NUM", 5)), 1), + } + started_monotonic = time.perf_counter() + LOGGER.info( + "Serper request start: query=%r gl=%s hl=%s num=%s", + _preview_text(query), + request_payload["gl"], + request_payload["hl"], + request_payload["num"], + ) + try: + async with session.post( + f"{self._serper_api_base}/search", + headers={ + "X-API-KEY": self._serper_api_key, + "Content-Type": "application/json", + }, + json=request_payload, + ) as response: + payload_text = await response.text() + LOGGER.info( + "Serper response: status=%s latency_ms=%s response_bytes=%s", + response.status, + int((time.perf_counter() - started_monotonic) * 1000.0), + len(payload_text), + ) + if response.status >= 400: + return f"Serper API returned HTTP {response.status}: {payload_text[:300]}" + except (TimeoutError, asyncio.TimeoutError): + return "Serper API timed out while searching." + except aiohttp.ClientError as exc: + return f"Serper API request failed: {exc}" + + try: + payload = json.loads(payload_text) + except json.JSONDecodeError: + return "Serper API returned invalid JSON." + summary = self._summarize_serper_payload(query=query, payload=payload) + LOGGER.info("Serper summary built: chars=%s preview=%r", len(summary), _preview_text(summary)) + return summary + + async def _get_serper_session(self): + try: + import aiohttp + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `aiohttp` package is required for Serper tool execution") from exc + + if self._serper_session is not None and not self._serper_session.closed: + return self._serper_session + async with self._serper_session_lock: + if self._serper_session is not None and not self._serper_session.closed: + return self._serper_session + timeout = aiohttp.ClientTimeout(total=self._timeout_seconds) + connector = aiohttp.TCPConnector(limit=16, ttl_dns_cache=300) + self._serper_session = aiohttp.ClientSession(timeout=timeout, connector=connector) + return self._serper_session + + def _build_tools(self) -> list[dict[str, Any]] | None: + if not self._enable_tools or not self._serper_api_key: + return None + return [ + { + "type": "function", + "function": { + "name": "serper", + "description": ( + "Search the public web for recent or external information when the user asks " + "about current facts, websites, company data, schedules, or anything requiring live search." + ), + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Precise search query to send to Serper.", + }, + "num": { + "type": "integer", + "description": "How many results to fetch, usually 3 to 5.", + "minimum": 1, + "maximum": 10, + }, + "hl": { + "type": "string", + "description": "UI language code, for example ru or en.", + }, + "gl": { + "type": "string", + "description": "Country code for result localization, for example kz or us.", + }, + }, + "required": ["query"], + "additionalProperties": False, + }, + }, + } + ] + def _get_client(self): if self._client is not None: return self._client @@ -107,11 +540,15 @@ class OpenAILLM(BaseLLM): ) return self._client - def _build_messages(self, text: str, context: list) -> list[dict[str, str]]: - messages: list[dict[str, str]] = [] + def _supports_custom_temperature(self) -> bool: + return not self._model.lower().startswith("gpt-5") + + def _build_messages(self, text: str, context: list) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] if self._system_prompt: messages.append({"role": "system", "content": self._system_prompt}) + context_messages: list[dict[str, Any]] = [] for entry in context: role: str | None = None content: str | None = None @@ -123,9 +560,280 @@ class OpenAILLM(BaseLLM): role = "assistant" if speaker == "assistant" else "user" content = str(entry[1]).strip() or None if role and content: - messages.append({"role": role, "content": content}) + context_messages.append({"role": role, "content": content}) + original_context_count = len(context_messages) + if self._max_context_messages > 0 and len(context_messages) > self._max_context_messages: + context_messages = context_messages[-self._max_context_messages :] + LOGGER.info( + "OpenAI LLM context trimmed: original=%s retained=%s max_context_messages=%s", + original_context_count, + len(context_messages), + self._max_context_messages, + ) + + messages.extend(context_messages) if text.strip(): if not messages or messages[-1].get("role") != "user" or messages[-1].get("content") != text: messages.append({"role": "user", "content": text}) return messages + + @staticmethod + def _finalize_tool_calls(tool_buffers: dict[int, dict[str, str]]) -> list[dict[str, Any]]: + tool_calls: list[dict[str, Any]] = [] + for index in sorted(tool_buffers): + state = tool_buffers[index] + name = str(state.get("name") or "").strip() + arguments = str(state.get("arguments") or "{}").strip() or "{}" + if not name: + continue + tool_calls.append( + { + "id": str(state.get("id") or f"tool-call-{index}"), + "type": "function", + "function": { + "name": name, + "arguments": arguments, + }, + } + ) + return tool_calls + + @staticmethod + def _summarize_serper_payload(*, query: str, payload: dict[str, Any]) -> str: + lines = [f"Search query: {query}"] + answer_box = payload.get("answerBox") + if isinstance(answer_box, dict): + answer_text = str(answer_box.get("answer") or answer_box.get("snippet") or "").strip() + if answer_text: + lines.append(f"Answer box: {answer_text}") + + knowledge_graph = payload.get("knowledgeGraph") + if isinstance(knowledge_graph, dict): + title = str(knowledge_graph.get("title") or "").strip() + description = str(knowledge_graph.get("description") or "").strip() + if title or description: + lines.append(f"Knowledge graph: {title} {description}".strip()) + + organic = payload.get("organic") + if isinstance(organic, list): + for index, item in enumerate(organic[:5], start=1): + if not isinstance(item, dict): + continue + title = str(item.get("title") or "").strip() + snippet = str(item.get("snippet") or "").strip() + link = str(item.get("link") or "").strip() + if title or snippet or link: + lines.append(f"{index}. {title} | {snippet} | {link}".strip()) + if len(lines) == 1: + lines.append("No useful search results were returned.") + return "\n".join(lines) + + + +class OllamaLLM(BaseLLM): + def __init__( + self, + *, + model: str | None = None, + base_url: str | None = None, + system_prompt: str | None = None, + timeout_seconds: float | None = None, + temperature: float | None = None, + max_context_messages: int | None = None, + ) -> None: + self._model = str(model or os.getenv("OLLAMA_LLM_MODEL", "qwen2.5:1.5b")).strip() or "qwen2.5:1.5b" + self._base_url = ( + str(base_url or os.getenv("OLLAMA_BASE_URL", "http://host.docker.internal:11434")).strip().rstrip("/") + or "http://host.docker.internal:11434" + ) + self._system_prompt = str( + system_prompt + if system_prompt is not None + else os.getenv( + "OLLAMA_LLM_SYSTEM_PROMPT", + os.getenv( + "OPENAI_LLM_SYSTEM_PROMPT", + "You are a concise voice assistant for a telecom call center. Answer clearly and briefly.", + ), + ) + ).strip() + self._timeout_seconds = max( + float(timeout_seconds if timeout_seconds is not None else self._read_float_env("OLLAMA_TIMEOUT_SECONDS", _timeout_seconds())), + 1.0, + ) + self._temperature = max( + min(float(temperature if temperature is not None else self._read_float_env("OLLAMA_LLM_TEMPERATURE", 0.3)), 2.0), + 0.0, + ) + self._max_context_messages = max( + int( + max_context_messages + if max_context_messages is not None + else self._read_int_env("OLLAMA_LLM_MAX_CONTEXT_MESSAGES", _max_context_messages()) + ), + 0, + ) + self._num_predict = max(self._read_int_env("OLLAMA_LLM_NUM_PREDICT", 64), 0) + self._num_ctx = max(self._read_int_env("OLLAMA_LLM_NUM_CTX", 1024), 0) + self._client: Any | None = None + LOGGER.info( + "Ollama LLM config: model=%s base_url=%s timeout=%s max_context_messages=%s " + "temperature=%s num_predict=%s num_ctx=%s", + self._model, + self._base_url, + self._timeout_seconds, + self._max_context_messages, + self._temperature, + self._num_predict or "default", + self._num_ctx or "default", + ) + + async def generate_stream(self, text: str, context: list) -> AsyncGenerator[LLMStreamEvent, None]: + messages = self._build_messages(text, context) + options: dict[str, Any] = {"temperature": self._temperature} + if self._num_predict > 0: + options["num_predict"] = self._num_predict + if self._num_ctx > 0: + options["num_ctx"] = self._num_ctx + payload: dict[str, Any] = { + "model": self._model, + "messages": messages, + "stream": True, + "options": options, + } + started_monotonic = time.perf_counter() + text_event_count = 0 + text_char_count = 0 + LOGGER.info( + "Ollama LLM turn start: model=%s input_chars=%s context_entries=%s messages=%s input_preview=%r", + self._model, + len(text), + len(context), + len(messages), + _preview_text(text), + ) + try: + client = self._get_client() + async with client.stream("POST", f"{self._base_url}/api/chat", json=payload) as response: + if response.status_code >= 400: + body = (await response.aread()).decode("utf-8", "replace") + raise RuntimeError(f"Ollama LLM returned HTTP {response.status_code}: {body[:300]}") + async for line in response.aiter_lines(): + if not line: + continue + try: + chunk = json.loads(line) + except json.JSONDecodeError: + LOGGER.warning("Ollama LLM ignored invalid stream line: %r", line[:200]) + continue + error = str(chunk.get("error") or "").strip() + if error: + raise RuntimeError(f"Ollama LLM error: {error}") + message = chunk.get("message") + content = "" + if isinstance(message, dict): + content = str(message.get("content") or "") + if content: + text_event_count += 1 + text_char_count += len(content) + if text_event_count == 1 or text_event_count % 20 == 0: + LOGGER.info( + "Ollama LLM text stream: events=%s chars=%s latest=%r", + text_event_count, + text_char_count, + _preview_text(content, limit=80), + ) + yield LLMStreamEvent(type="text", content=content) + if bool(chunk.get("done")): + break + except (TimeoutError, asyncio.TimeoutError) as exc: + raise RuntimeError("Ollama LLM request timed out") from exc + except RuntimeError: + raise + except Exception as exc: # noqa: BLE001 + raise RuntimeError("Ollama LLM streaming failed") from exc + finally: + LOGGER.info( + "Ollama LLM turn completed: text_events=%s text_chars=%s total_latency_ms=%s", + text_event_count, + text_char_count, + int((time.perf_counter() - started_monotonic) * 1000.0), + ) + + async def close(self) -> None: + client = self._client + self._client = None + if client is not None: + await client.aclose() + + def _get_client(self): + if self._client is not None: + return self._client + try: + import httpx + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `httpx` package is required for Ollama LLM") from exc + timeout = httpx.Timeout( + self._timeout_seconds, + connect=min(self._timeout_seconds, 3.0), + write=min(self._timeout_seconds, 10.0), + read=self._timeout_seconds, + ) + self._client = httpx.AsyncClient(timeout=timeout) + return self._client + + def _build_messages(self, text: str, context: list) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + if self._system_prompt: + messages.append({"role": "system", "content": self._system_prompt}) + + context_messages: list[dict[str, Any]] = [] + for entry in context: + role: str | None = None + content: str | None = None + if isinstance(entry, dict): + role = str(entry.get("role") or "").strip().lower() or None + content = str(entry.get("content") or "").strip() or None + elif isinstance(entry, (tuple, list)) and len(entry) >= 2: + speaker = str(entry[0]).strip().lower() + role = "assistant" if speaker == "assistant" else "user" + content = str(entry[1]).strip() or None + if role and content: + context_messages.append({"role": role, "content": content}) + + original_context_count = len(context_messages) + if self._max_context_messages > 0 and len(context_messages) > self._max_context_messages: + context_messages = context_messages[-self._max_context_messages :] + LOGGER.info( + "Ollama LLM context trimmed: original=%s retained=%s max_context_messages=%s", + original_context_count, + len(context_messages), + self._max_context_messages, + ) + + messages.extend(context_messages) + if text.strip(): + if not messages or messages[-1].get("role") != "user" or messages[-1].get("content") != text: + messages.append({"role": "user", "content": text}) + return messages + + @staticmethod + def _read_float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + return float(str(raw).strip()) + except ValueError: + return default + + @staticmethod + def _read_int_env(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + return int(str(raw).strip()) + except ValueError: + return default diff --git a/providers/stt.py b/providers/stt.py index 6e3aae9..02c3449 100644 --- a/providers/stt.py +++ b/providers/stt.py @@ -2,18 +2,109 @@ from __future__ import annotations import asyncio import audioop +import base64 +import contextlib import io import json +import logging import os +import time import wave +from urllib.parse import urlencode from realtime_voice_service.providers.base import BaseSTT +from realtime_voice_service.providers.base import BaseSTTStream +from realtime_voice_service.providers.base import PartialTranscriptCallback +from realtime_voice_service.providers.stt_openai import OpenAISTT + + +LOGGER = logging.getLogger("uvicorn.error") +_STREAM_COMMIT = object() +_STREAM_CANCEL = object() def _api_base() -> str: return (os.getenv("ELEVENLABS_API_BASE", "https://api.elevenlabs.io").strip() or "https://api.elevenlabs.io").rstrip("/") +def _stt_provider_name() -> str: + return ( + os.getenv("STT_PROVIDER", "").strip() + or os.getenv("REALTIME_VOICE_STT_PROVIDER", "").strip() + or os.getenv("AI_VOICE_ASR_PROVIDER", "").strip() + or "elevenlabs" + ).lower() + + +def _stt_fallback_provider_name() -> str: + return os.getenv("REALTIME_VOICE_STT_FALLBACK_PROVIDER", "").strip().lower() + + +def _yandex_api_base() -> str: + return ( + os.getenv("YANDEX_STT_API_BASE", "").strip() + or os.getenv("AI_VOICE_ASR_YANDEX_API_BASE", "").strip() + or "https://stt.api.cloud.yandex.net" + ).rstrip("/") + + +def _yandex_api_key() -> str: + return ( + os.getenv("YANDEX_STT_API_KEY", "").strip() + or os.getenv("AI_VOICE_ASR_YANDEX_API_KEY", "").strip() + or os.getenv("AI_VOICE_TTS_YANDEX_API_KEY", "").strip() + ) + + +def _yandex_iam_token() -> str: + return ( + os.getenv("YANDEX_STT_IAM_TOKEN", "").strip() + or os.getenv("AI_VOICE_ASR_YANDEX_IAM_TOKEN", "").strip() + or os.getenv("AI_VOICE_TTS_YANDEX_IAM_TOKEN", "").strip() + ) + + +def _yandex_folder_id() -> str: + return ( + os.getenv("YANDEX_STT_FOLDER_ID", "").strip() + or os.getenv("AI_VOICE_ASR_YANDEX_FOLDER_ID", "").strip() + or os.getenv("AI_VOICE_TTS_YANDEX_FOLDER_ID", "").strip() + ) + + +def _yandex_language() -> str: + raw = ( + os.getenv("YANDEX_STT_LANGUAGE", "").strip() + or os.getenv("AI_VOICE_ASR_YANDEX_LANGUAGE", "").strip() + or "ru-RU" + ) + normalized = raw.lower().replace("_", "-") + mapping = { + "ru": "ru-RU", + "rus": "ru-RU", + "ru-ru": "ru-RU", + "kk": "kk-KZ", + "kaz": "kk-KZ", + "kz": "kk-KZ", + "kk-kz": "kk-KZ", + "kz-kz": "kk-KZ", + } + return mapping.get(normalized, raw) + + +def _yandex_topic() -> str: + return os.getenv("YANDEX_STT_TOPIC", "").strip() or os.getenv("AI_VOICE_ASR_YANDEX_TOPIC", "general").strip() or "general" + + +def _ws_base(http_base: str) -> str: + normalized = http_base.rstrip("/") + if normalized.startswith("https://"): + return f"wss://{normalized[len('https://') :]}" + if normalized.startswith("http://"): + return f"ws://{normalized[len('http://') :]}" + return normalized + + def _timeout_seconds() -> float: raw = os.getenv("ELEVENLABS_TIMEOUT_SECONDS") if raw is None: @@ -24,6 +115,20 @@ def _timeout_seconds() -> float: return 30.0 +def _bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def _preview_text(text: str, *, limit: int = 120) -> str: + normalized = " ".join(str(text or "").split()) + if len(normalized) <= limit: + return normalized + return f"{normalized[:limit]}..." + + def _pcm16le_to_wav_bytes( pcm_bytes: bytes, *, @@ -59,6 +164,313 @@ def _resample_pcm16le( return converted +def _pcm_duration_ms(pcm_bytes: bytes, *, sample_rate_hz: int) -> int: + if not pcm_bytes or sample_rate_hz <= 0: + return 0 + sample_count = len(pcm_bytes) // 2 + return max(int((sample_count / float(sample_rate_hz)) * 1000.0), 0) + + +def _pad_pcm16le_to_duration( + pcm_bytes: bytes, + *, + sample_rate_hz: int, + min_duration_ms: int, +) -> bytes: + if not pcm_bytes: + return pcm_bytes + current_duration_ms = _pcm_duration_ms(pcm_bytes, sample_rate_hz=sample_rate_hz) + if current_duration_ms >= min_duration_ms: + return pcm_bytes + target_samples = max(int(sample_rate_hz * (min_duration_ms / 1000.0)), 1) + target_bytes = target_samples * 2 + if len(pcm_bytes) >= target_bytes: + return pcm_bytes + return pcm_bytes + (b"\x00" * (target_bytes - len(pcm_bytes))) + + +class ElevenLabsRealtimeSTTStream(BaseSTTStream): + def __init__( + self, + *, + api_key: str, + websocket_url: str, + sample_rate_hz: int, + timeout_seconds: float, + partial_callback: PartialTranscriptCallback | None = None, + ) -> None: + self._api_key = api_key + self._websocket_url = websocket_url + self._sample_rate_hz = sample_rate_hz + self._timeout_seconds = timeout_seconds + self._partial_callback = partial_callback + self._websocket = None + self._sender_task: asyncio.Task[None] | None = None + self._receiver_task: asyncio.Task[None] | None = None + self._outgoing_queue: asyncio.Queue[bytes | object] = asyncio.Queue() + self._final_transcript_future: asyncio.Future[str] = asyncio.get_running_loop().create_future() + self._close_lock = asyncio.Lock() + self._closed = False + self._commit_requested = False + self._audio_sent = False + self._latest_partial = "" + self._latest_committed = "" + self._connect_started_monotonic: float | None = None + self._sent_chunk_count = 0 + self._sent_byte_count = 0 + self._partial_count = 0 + self._committed_count = 0 + + async def connect(self) -> None: + try: + from websockets.legacy.client import connect as websocket_connect + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `websockets` package is required for ElevenLabs realtime STT") from exc + + self._connect_started_monotonic = time.perf_counter() + LOGGER.info( + "ElevenLabs realtime STT connecting: sample_rate=%s timeout=%s", + self._sample_rate_hz, + self._timeout_seconds, + ) + self._websocket = await websocket_connect( + self._websocket_url, + extra_headers=[("xi-api-key", self._api_key)], + open_timeout=min(self._timeout_seconds, 10.0), + close_timeout=1.0, + ping_interval=20.0, + ping_timeout=20.0, + max_size=None, + ) + try: + await self._await_session_started() + except Exception: + await self._close_websocket() + raise + LOGGER.info( + "ElevenLabs realtime STT connected: sample_rate=%s connect_ms=%s", + self._sample_rate_hz, + int((time.perf_counter() - self._connect_started_monotonic) * 1000.0), + ) + self._sender_task = asyncio.create_task(self._sender_loop(), name="elevenlabs-stt-sender") + self._receiver_task = asyncio.create_task(self._receiver_loop(), name="elevenlabs-stt-receiver") + + async def push_audio(self, audio_chunk: bytes) -> None: + if self._closed or not audio_chunk: + return + await self._outgoing_queue.put(audio_chunk) + + async def finish(self) -> str: + if self._closed: + return self._latest_committed or self._latest_partial + self._commit_requested = True + finished_started_monotonic = time.perf_counter() + LOGGER.info( + "ElevenLabs realtime STT finish requested: sent_chunks=%s sent_bytes=%s latest_partial=%r", + self._sent_chunk_count, + self._sent_byte_count, + _preview_text(self._latest_partial), + ) + await self._outgoing_queue.put(_STREAM_COMMIT) + try: + transcript = await asyncio.wait_for( + asyncio.shield(self._final_transcript_future), + timeout=self._timeout_seconds, + ) + LOGGER.info( + "ElevenLabs realtime STT final transcript: latency_ms=%s chars=%s transcript=%r", + int((time.perf_counter() - finished_started_monotonic) * 1000.0), + len(transcript), + _preview_text(transcript), + ) + return transcript + except asyncio.TimeoutError as exc: + transcript = self._latest_committed or self._latest_partial + if transcript: + LOGGER.warning("ElevenLabs realtime STT finish timed out; returning best-effort transcript") + return transcript + raise RuntimeError("ElevenLabs realtime STT finish timed out") from exc + finally: + await self._shutdown() + + async def cancel(self) -> None: + if self._closed: + return + LOGGER.info( + "ElevenLabs realtime STT cancel requested: sent_chunks=%s sent_bytes=%s partials=%s commits=%s", + self._sent_chunk_count, + self._sent_byte_count, + self._partial_count, + self._committed_count, + ) + await self._outgoing_queue.put(_STREAM_CANCEL) + await self._shutdown() + + async def _sender_loop(self) -> None: + pending_chunk: bytes | None = None + try: + while True: + item = await self._outgoing_queue.get() + if item is _STREAM_CANCEL: + return + if item is _STREAM_COMMIT: + if pending_chunk is not None: + await self._send_audio_chunk(pending_chunk, commit=True) + pending_chunk = None + elif not self._final_transcript_future.done(): + self._final_transcript_future.set_result(self._latest_committed or self._latest_partial) + return + audio_chunk = bytes(item) + if not audio_chunk: + continue + self._audio_sent = True + if pending_chunk is not None: + await self._send_audio_chunk(pending_chunk, commit=False) + pending_chunk = audio_chunk + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + if not self._final_transcript_future.done(): + self._final_transcript_future.set_exception(exc) + raise + + async def _receiver_loop(self) -> None: + try: + while True: + payload = await self._receive_payload() + message_type = str(payload.get("message_type") or "").strip().lower() + if message_type == "partial_transcript": + self._latest_partial = str(payload.get("text") or "").strip() + self._partial_count += 1 + LOGGER.info( + "ElevenLabs realtime STT partial: index=%s chars=%s text=%r", + self._partial_count, + len(self._latest_partial), + _preview_text(self._latest_partial), + ) + await self._emit_partial(self._latest_partial) + continue + if message_type in {"committed_transcript", "committed_transcript_with_timestamps"}: + self._latest_committed = str(payload.get("text") or "").strip() + self._committed_count += 1 + LOGGER.info( + "ElevenLabs realtime STT committed: index=%s chars=%s text=%r", + self._committed_count, + len(self._latest_committed), + _preview_text(self._latest_committed), + ) + if self._commit_requested and not self._final_transcript_future.done(): + self._final_transcript_future.set_result(self._latest_committed) + continue + if message_type == "session_started": + continue + if self._is_error_message(message_type): + raise RuntimeError(self._format_realtime_error(payload)) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 + if not self._final_transcript_future.done(): + transcript = self._latest_committed or self._latest_partial + if transcript and self._commit_requested: + self._final_transcript_future.set_result(transcript) + else: + self._final_transcript_future.set_exception(exc) + raise + + async def _send_audio_chunk(self, audio_chunk: bytes, *, commit: bool) -> None: + websocket = self._require_websocket() + payload = { + "message_type": "input_audio_chunk", + "audio_base_64": base64.b64encode(audio_chunk).decode("ascii"), + "sample_rate": self._sample_rate_hz, + "commit": commit, + } + await websocket.send(json.dumps(payload)) + self._sent_chunk_count += 1 + self._sent_byte_count += len(audio_chunk) + if commit or self._sent_chunk_count == 1 or self._sent_chunk_count % 10 == 0: + LOGGER.info( + "ElevenLabs realtime STT audio sent: chunks=%s bytes=%s last_chunk_bytes=%s commit=%s", + self._sent_chunk_count, + self._sent_byte_count, + len(audio_chunk), + commit, + ) + + async def _await_session_started(self) -> None: + while True: + payload = await self._receive_payload() + message_type = str(payload.get("message_type") or "").strip().lower() + if message_type == "session_started": + return + if self._is_error_message(message_type): + raise RuntimeError(self._format_realtime_error(payload)) + + async def _receive_payload(self) -> dict[str, object]: + websocket = self._require_websocket() + raw_message = await asyncio.wait_for(websocket.recv(), timeout=self._timeout_seconds) + if isinstance(raw_message, bytes): + raw_message = raw_message.decode("utf-8") + try: + payload = json.loads(raw_message) + except json.JSONDecodeError as exc: + raise RuntimeError("ElevenLabs realtime STT returned invalid JSON") from exc + if not isinstance(payload, dict): + raise RuntimeError("ElevenLabs realtime STT returned malformed payload") + return payload + + async def _emit_partial(self, text: str) -> None: + if not text or self._partial_callback is None: + return + maybe_awaitable = self._partial_callback(text) + if maybe_awaitable is not None: + await maybe_awaitable + + async def _shutdown(self) -> None: + async with self._close_lock: + if self._closed: + return + self._closed = True + tasks = [self._sender_task, self._receiver_task] + for task in tasks: + if task is not None and not task.done(): + task.cancel() + for task in tasks: + if task is not None: + try: + await task + except asyncio.CancelledError: + pass + except Exception: + LOGGER.debug("ignored realtime STT background task failure during shutdown", exc_info=True) + await self._close_websocket() + + async def _close_websocket(self) -> None: + websocket = self._websocket + self._websocket = None + if websocket is None: + return + with contextlib.suppress(Exception): + await websocket.close() + + def _require_websocket(self): + if self._websocket is None: + raise RuntimeError("ElevenLabs realtime STT websocket is not connected") + return self._websocket + + @staticmethod + def _is_error_message(message_type: str) -> bool: + return message_type in {"error", "auth_error", "input_error"} or message_type.endswith("_error") + + @staticmethod + def _format_realtime_error(payload: dict[str, object]) -> str: + message_type = str(payload.get("message_type") or "error") + detail = str(payload.get("message") or payload.get("detail") or payload.get("error") or "").strip() + if detail: + return f"ElevenLabs realtime STT {message_type}: {detail}" + return f"ElevenLabs realtime STT {message_type}" + + class ElevenLabsSTT(BaseSTT): def __init__( self, @@ -66,18 +478,55 @@ class ElevenLabsSTT(BaseSTT): api_key: str | None = None, api_base: str | None = None, model_id: str | None = None, - input_sample_rate_hz: int = 8000, + realtime_model_id: str | None = None, + input_sample_rate_hz: int = 16000, target_sample_rate_hz: int = 16000, timeout_seconds: float | None = None, language_code: str | None = None, + use_realtime: bool | None = None, + allow_batch_fallback: bool | None = None, + realtime_chunk_duration_ms: int = 120, ) -> None: self._api_key = str(api_key if api_key is not None else os.getenv("ELEVENLABS_API_KEY", "")).strip() self._api_base = str(api_base or _api_base()).strip().rstrip("/") + self._ws_base = _ws_base(self._api_base) self._model_id = str(model_id or os.getenv("ELEVENLABS_STT_MODEL_ID", "scribe_v2")).strip() or "scribe_v2" + self._realtime_model_id = ( + str(realtime_model_id or os.getenv("ELEVENLABS_STT_REALTIME_MODEL_ID", "scribe_v2_realtime")).strip() + or "scribe_v2_realtime" + ) self._input_sample_rate_hz = max(int(input_sample_rate_hz), 1) self._target_sample_rate_hz = max(int(target_sample_rate_hz), 1) self._timeout_seconds = max(float(timeout_seconds if timeout_seconds is not None else _timeout_seconds()), 1.0) self._language_code = str(language_code or os.getenv("ELEVENLABS_STT_LANGUAGE_CODE", "")).strip() or None + self._use_realtime = ( + bool(use_realtime) + if use_realtime is not None + else _bool_env("ELEVENLABS_STT_USE_REALTIME", True) + ) + self._allow_batch_fallback = ( + bool(allow_batch_fallback) + if allow_batch_fallback is not None + else _bool_env("ELEVENLABS_STT_ALLOW_BATCH_FALLBACK", True) + ) + self._realtime_chunk_duration_ms = max(int(realtime_chunk_duration_ms), 40) + self._batch_min_audio_ms = max(int(os.getenv("ELEVENLABS_STT_BATCH_MIN_AUDIO_MS", "800")), 200) + self._client_session = None + self._client_session_lock = asyncio.Lock() + LOGGER.info( + "ElevenLabs STT config: batch_model=%s realtime_model=%s input_sample_rate=%s " + "target_sample_rate=%s use_realtime=%s allow_batch_fallback=%s language=%s " + "realtime_chunk_ms=%s batch_min_audio_ms=%s", + self._model_id, + self._realtime_model_id, + self._input_sample_rate_hz, + self._target_sample_rate_hz, + self._use_realtime, + self._allow_batch_fallback, + self._language_code, + self._realtime_chunk_duration_ms, + self._batch_min_audio_ms, + ) async def transcribe(self, audio_bytes: bytes) -> str: if not audio_bytes: @@ -86,21 +535,231 @@ class ElevenLabsSTT(BaseSTT): raise RuntimeError("ELEVENLABS_API_KEY is required for ElevenLabs STT") pcm_bytes, sample_rate_hz = self._extract_pcm(audio_bytes) + LOGGER.info( + "ElevenLabs STT transcribe start: input_bytes=%s extracted_pcm_bytes=%s sample_rate=%s duration_ms=%s " + "use_realtime=%s", + len(audio_bytes), + len(pcm_bytes), + sample_rate_hz, + _pcm_duration_ms(pcm_bytes, sample_rate_hz=sample_rate_hz), + self._use_realtime, + ) if sample_rate_hz != self._target_sample_rate_hz: + before_rate_hz = sample_rate_hz + before_bytes = len(pcm_bytes) pcm_bytes = _resample_pcm16le( pcm_bytes, input_rate_hz=sample_rate_hz, output_rate_hz=self._target_sample_rate_hz, ) sample_rate_hz = self._target_sample_rate_hz + LOGGER.info( + "ElevenLabs STT resampled input: from_rate=%s to_rate=%s before_bytes=%s after_bytes=%s", + before_rate_hz, + sample_rate_hz, + before_bytes, + len(pcm_bytes), + ) - wav_bytes = _pcm16le_to_wav_bytes(pcm_bytes, sample_rate_hz=sample_rate_hz) + if self._use_realtime: + try: + transcript = await self._transcribe_realtime( + pcm_bytes=pcm_bytes, + sample_rate_hz=sample_rate_hz, + ) + LOGGER.info( + "ElevenLabs STT realtime transcript result: chars=%s transcript=%r", + len(transcript), + _preview_text(transcript), + ) + return transcript + except Exception: + if not self._allow_batch_fallback: + raise + LOGGER.exception("ElevenLabs STT realtime failed; falling back to batch STT") + transcript = await self._transcribe_batch( + pcm_bytes=pcm_bytes, + sample_rate_hz=sample_rate_hz, + ) + LOGGER.info( + "ElevenLabs STT batch transcript result: chars=%s transcript=%r", + len(transcript), + _preview_text(transcript), + ) + return transcript + + async def start_stream( + self, + *, + partial_callback: PartialTranscriptCallback | None = None, + ) -> BaseSTTStream | None: + if not self._use_realtime: + return None + if not self._api_key: + raise RuntimeError("ELEVENLABS_API_KEY is required for ElevenLabs STT") + + websocket_url = self._build_realtime_websocket_url(sample_rate_hz=self._target_sample_rate_hz) + LOGGER.info( + "ElevenLabs STT live stream starting: realtime_model=%s sample_rate=%s language=%s", + self._realtime_model_id, + self._target_sample_rate_hz, + self._language_code, + ) + stream = ElevenLabsRealtimeSTTStream( + api_key=self._api_key, + websocket_url=websocket_url, + sample_rate_hz=self._target_sample_rate_hz, + timeout_seconds=self._timeout_seconds, + partial_callback=partial_callback, + ) + await stream.connect() + LOGGER.info("ElevenLabs STT live stream started: sample_rate=%s", self._target_sample_rate_hz) + return stream + + async def close(self) -> None: + session = self._client_session + self._client_session = None + if session is not None and not session.closed: + await session.close() + + async def _transcribe_realtime( + self, + *, + pcm_bytes: bytes, + sample_rate_hz: int, + ) -> str: + try: + from websockets.exceptions import WebSocketException + from websockets.legacy.client import connect as websocket_connect + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `websockets` package is required for ElevenLabs realtime STT") from exc + + websocket_url = self._build_realtime_websocket_url(sample_rate_hz=sample_rate_hz) + chunk_bytes = max(int(sample_rate_hz * self._realtime_chunk_duration_ms / 1000.0) * 2, 320) + started_monotonic = time.perf_counter() + sent_chunks = 0 + sent_bytes = 0 + LOGGER.info( + "ElevenLabs STT realtime batch-style stream start: model=%s sample_rate=%s pcm_bytes=%s " + "duration_ms=%s chunk_bytes=%s", + self._realtime_model_id, + sample_rate_hz, + len(pcm_bytes), + _pcm_duration_ms(pcm_bytes, sample_rate_hz=sample_rate_hz), + chunk_bytes, + ) + + try: + async with websocket_connect( + websocket_url, + extra_headers=[("xi-api-key", self._api_key)], + open_timeout=min(self._timeout_seconds, 10.0), + close_timeout=1.0, + ping_interval=20.0, + ping_timeout=20.0, + max_size=None, + ) as websocket: + await self._await_realtime_message( + websocket, + allowed_message_types={"session_started"}, + ) + for offset in range(0, len(pcm_bytes), chunk_bytes): + audio_chunk = pcm_bytes[offset : offset + chunk_bytes] + if not audio_chunk: + continue + payload = { + "message_type": "input_audio_chunk", + "audio_base_64": base64.b64encode(audio_chunk).decode("ascii"), + "sample_rate": sample_rate_hz, + "commit": offset + chunk_bytes >= len(pcm_bytes), + } + await websocket.send(json.dumps(payload)) + sent_chunks += 1 + sent_bytes += len(audio_chunk) + if sent_chunks == 1 or payload["commit"] or sent_chunks % 10 == 0: + LOGGER.info( + "ElevenLabs STT realtime batch-style audio sent: chunks=%s bytes=%s commit=%s", + sent_chunks, + sent_bytes, + payload["commit"], + ) + + transcript = "" + partial_transcript = "" + while True: + message = await self._receive_realtime_payload(websocket) + message_type = str(message.get("message_type") or "").strip().lower() + if message_type in {"committed_transcript", "committed_transcript_with_timestamps"}: + transcript = str(message.get("text") or "").strip() + if transcript: + LOGGER.info( + "ElevenLabs STT realtime batch-style committed: latency_ms=%s chars=%s text=%r", + int((time.perf_counter() - started_monotonic) * 1000.0), + len(transcript), + _preview_text(transcript), + ) + return transcript + continue + if message_type == "partial_transcript": + partial_transcript = str(message.get("text") or "").strip() + LOGGER.info( + "ElevenLabs STT realtime batch-style partial: chars=%s text=%r", + len(partial_transcript), + _preview_text(partial_transcript), + ) + continue + if message_type == "session_started": + continue + if self._is_error_message(message_type): + raise RuntimeError(self._format_realtime_error(message)) + if not message_type: + continue + if partial_transcript: + LOGGER.info( + "ElevenLabs STT realtime batch-style returning partial: latency_ms=%s chars=%s", + int((time.perf_counter() - started_monotonic) * 1000.0), + len(partial_transcript), + ) + return partial_transcript + except asyncio.TimeoutError as exc: + raise RuntimeError("ElevenLabs realtime STT request timed out") from exc + except WebSocketException as exc: + raise RuntimeError("ElevenLabs realtime STT stream failed") from exc + except OSError as exc: + raise RuntimeError("ElevenLabs realtime STT connection failed") from exc + + return "" + + async def _transcribe_batch( + self, + *, + pcm_bytes: bytes, + sample_rate_hz: int, + ) -> str: + # TODO: Architectural Bottleneck: Рассмотреть замену STT на Deepgram WebSocket API для достижения true-streaming latency. try: import aiohttp except Exception as exc: # noqa: BLE001 raise RuntimeError("The `aiohttp` package is required for ElevenLabs STT") from exc + pcm_bytes = _pad_pcm16le_to_duration( + pcm_bytes, + sample_rate_hz=sample_rate_hz, + min_duration_ms=self._batch_min_audio_ms, + ) + wav_bytes = _pcm16le_to_wav_bytes(pcm_bytes, sample_rate_hz=sample_rate_hz) + started_monotonic = time.perf_counter() + LOGGER.info( + "ElevenLabs batch STT request start: model=%s sample_rate=%s pcm_bytes=%s wav_bytes=%s " + "duration_ms=%s language=%s", + self._model_id, + sample_rate_hz, + len(pcm_bytes), + len(wav_bytes), + _pcm_duration_ms(pcm_bytes, sample_rate_hz=sample_rate_hz), + self._language_code, + ) form = aiohttp.FormData() form.add_field("model_id", self._model_id) form.add_field("timestamps_granularity", "none") @@ -114,19 +773,27 @@ class ElevenLabsSTT(BaseSTT): content_type="audio/wav", ) - timeout = aiohttp.ClientTimeout(total=self._timeout_seconds) + session = await self._get_client_session() try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.post( - f"{self._api_base}/v1/speech-to-text", - headers={"xi-api-key": self._api_key}, - data=form, - ) as response: - payload_text = await response.text() - if response.status >= 400: - raise RuntimeError( - f"ElevenLabs STT returned HTTP {response.status}: {payload_text[:300]}" - ) + async with session.post( + f"{self._api_base}/v1/speech-to-text", + headers={"xi-api-key": self._api_key}, + data=form, + ) as response: + payload_text = await response.text() + LOGGER.info( + "ElevenLabs batch STT response: status=%s latency_ms=%s response_bytes=%s", + response.status, + int((time.perf_counter() - started_monotonic) * 1000.0), + len(payload_text), + ) + if response.status >= 400: + if response.status == 400 and "audio_too_short" in payload_text: + LOGGER.warning("ElevenLabs batch STT reported audio_too_short; ignoring utterance") + return "" + raise RuntimeError( + f"ElevenLabs STT returned HTTP {response.status}: {payload_text[:300]}" + ) except (TimeoutError, asyncio.TimeoutError) as exc: raise RuntimeError("ElevenLabs STT request timed out") from exc except aiohttp.ClientError as exc: @@ -138,6 +805,72 @@ class ElevenLabsSTT(BaseSTT): raise RuntimeError("ElevenLabs STT returned invalid JSON") from exc return str(payload.get("text") or payload.get("transcript") or "").strip() + async def _get_client_session(self): + try: + import aiohttp + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `aiohttp` package is required for ElevenLabs STT") from exc + + if self._client_session is not None and not self._client_session.closed: + return self._client_session + + async with self._client_session_lock: + if self._client_session is not None and not self._client_session.closed: + return self._client_session + timeout = aiohttp.ClientTimeout(total=self._timeout_seconds) + connector = aiohttp.TCPConnector(limit=32, ttl_dns_cache=300) + self._client_session = aiohttp.ClientSession(timeout=timeout, connector=connector) + return self._client_session + + async def _await_realtime_message( + self, + websocket, + *, + allowed_message_types: set[str], + ) -> dict[str, object]: + while True: + message = await self._receive_realtime_payload(websocket) + message_type = str(message.get("message_type") or "").strip().lower() + if message_type in allowed_message_types: + return message + if self._is_error_message(message_type): + raise RuntimeError(self._format_realtime_error(message)) + + async def _receive_realtime_payload(self, websocket) -> dict[str, object]: + raw_message = await asyncio.wait_for(websocket.recv(), timeout=self._timeout_seconds) + if isinstance(raw_message, bytes): + raw_message = raw_message.decode("utf-8") + try: + payload = json.loads(raw_message) + except json.JSONDecodeError as exc: + raise RuntimeError("ElevenLabs realtime STT returned invalid JSON") from exc + if not isinstance(payload, dict): + raise RuntimeError("ElevenLabs realtime STT returned malformed payload") + return payload + + @staticmethod + def _is_error_message(message_type: str) -> bool: + return message_type in {"error", "auth_error", "input_error"} or message_type.endswith("_error") + + @staticmethod + def _format_realtime_error(payload: dict[str, object]) -> str: + message_type = str(payload.get("message_type") or "error") + detail = str(payload.get("message") or payload.get("detail") or payload.get("error") or "").strip() + if detail: + return f"ElevenLabs realtime STT {message_type}: {detail}" + return f"ElevenLabs realtime STT {message_type}" + + def _build_realtime_websocket_url(self, *, sample_rate_hz: int) -> str: + query = { + "model_id": self._realtime_model_id, + "audio_format": f"pcm_{sample_rate_hz}", + "commit_strategy": "manual", + "include_timestamps": "false", + } + if self._language_code: + query["language_code"] = self._language_code + return f"{self._ws_base}/v1/speech-to-text/realtime?{urlencode(query)}" + def _extract_pcm(self, audio_bytes: bytes) -> tuple[bytes, int]: try: with wave.open(io.BytesIO(audio_bytes), "rb") as wav_file: @@ -152,3 +885,244 @@ class ElevenLabsSTT(BaseSTT): return pcm_bytes, sample_rate_hz except (wave.Error, EOFError): return audio_bytes, self._input_sample_rate_hz + + +class YandexSpeechKitSTT(BaseSTT): + name = "yandex" + + def __init__( + self, + *, + api_key: str | None = None, + iam_token: str | None = None, + folder_id: str | None = None, + api_base: str | None = None, + language: str | None = None, + topic: str | None = None, + input_sample_rate_hz: int = 8000, + target_sample_rate_hz: int = 8000, + timeout_seconds: float | None = None, + ) -> None: + self._api_key = str(api_key if api_key is not None else _yandex_api_key()).strip() + self._iam_token = str(iam_token if iam_token is not None else _yandex_iam_token()).strip() + self._folder_id = str(folder_id if folder_id is not None else _yandex_folder_id()).strip() + self._api_base = str(api_base or _yandex_api_base()).strip().rstrip("/") + self._language = str(language or _yandex_language()).strip() or "ru-RU" + self._topic = str(topic or _yandex_topic()).strip() or "general" + self._input_sample_rate_hz = max(int(input_sample_rate_hz), 1) + self._target_sample_rate_hz = max(int(target_sample_rate_hz), 1) + self._timeout_seconds = max(float(timeout_seconds if timeout_seconds is not None else _timeout_seconds()), 1.0) + self._client_session = None + self._client_session_lock = asyncio.Lock() + LOGGER.info( + "Yandex SpeechKit STT config: api_base=%s input_sample_rate=%s target_sample_rate=%s " + "language=%s topic=%s folder_configured=%s auth=%s", + self._api_base, + self._input_sample_rate_hz, + self._target_sample_rate_hz, + self._language, + self._topic, + bool(self._folder_id), + "iam" if self._iam_token else ("api-key" if self._api_key else "missing"), + ) + + async def transcribe(self, audio_bytes: bytes) -> str: + if not audio_bytes: + return "" + if not self._api_key and not self._iam_token: + raise RuntimeError("YANDEX_STT_API_KEY or YANDEX_STT_IAM_TOKEN is required for Yandex STT") + + pcm_bytes, sample_rate_hz = self._extract_pcm(audio_bytes) + LOGGER.info( + "Yandex SpeechKit STT transcribe start: input_bytes=%s extracted_pcm_bytes=%s sample_rate=%s duration_ms=%s", + len(audio_bytes), + len(pcm_bytes), + sample_rate_hz, + _pcm_duration_ms(pcm_bytes, sample_rate_hz=sample_rate_hz), + ) + if sample_rate_hz != self._target_sample_rate_hz: + before_rate_hz = sample_rate_hz + before_bytes = len(pcm_bytes) + pcm_bytes = _resample_pcm16le( + pcm_bytes, + input_rate_hz=sample_rate_hz, + output_rate_hz=self._target_sample_rate_hz, + ) + sample_rate_hz = self._target_sample_rate_hz + LOGGER.info( + "Yandex SpeechKit STT resampled input: from_rate=%s to_rate=%s before_bytes=%s after_bytes=%s", + before_rate_hz, + sample_rate_hz, + before_bytes, + len(pcm_bytes), + ) + + session = await self._get_client_session() + params = { + "lang": self._language, + "topic": self._topic, + "format": "lpcm", + "sampleRateHertz": str(sample_rate_hz), + } + if self._folder_id: + params["folderId"] = self._folder_id + headers = { + "Content-Type": f"audio/x-pcm;bit=16;rate={sample_rate_hz}", + "Authorization": f"Bearer {self._iam_token}" if self._iam_token else f"Api-Key {self._api_key}", + } + started_monotonic = time.perf_counter() + try: + async with session.post( + f"{self._api_base}/speech/v1/stt:recognize", + params=params, + headers=headers, + data=pcm_bytes, + ) as response: + payload_text = await response.text() + LOGGER.info( + "Yandex SpeechKit STT response: status=%s latency_ms=%s response_bytes=%s", + response.status, + int((time.perf_counter() - started_monotonic) * 1000.0), + len(payload_text), + ) + if response.status >= 400: + raise RuntimeError( + f"Yandex SpeechKit STT returned HTTP {response.status}: {payload_text[:300]}" + ) + except (TimeoutError, asyncio.TimeoutError) as exc: + raise RuntimeError("Yandex SpeechKit STT request timed out") from exc + except Exception as exc: + try: + import aiohttp + except Exception: # noqa: BLE001 + aiohttp = None + if aiohttp is not None and isinstance(exc, aiohttp.ClientError): + raise RuntimeError("Yandex SpeechKit STT request failed") from exc + raise + + try: + payload = json.loads(payload_text) + except json.JSONDecodeError as exc: + raise RuntimeError("Yandex SpeechKit STT returned invalid JSON") from exc + transcript = str(payload.get("result") or payload.get("text") or "").strip() + LOGGER.info( + "Yandex SpeechKit STT transcript result: chars=%s transcript=%r", + len(transcript), + _preview_text(transcript), + ) + return transcript + + async def close(self) -> None: + session = self._client_session + self._client_session = None + if session is not None and not session.closed: + await session.close() + + async def _get_client_session(self): + try: + import aiohttp + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `aiohttp` package is required for Yandex STT") from exc + + if self._client_session is not None and not self._client_session.closed: + return self._client_session + + async with self._client_session_lock: + if self._client_session is not None and not self._client_session.closed: + return self._client_session + timeout = aiohttp.ClientTimeout(total=self._timeout_seconds) + connector = aiohttp.TCPConnector(limit=16, ttl_dns_cache=300) + self._client_session = aiohttp.ClientSession(timeout=timeout, connector=connector) + return self._client_session + + def _extract_pcm(self, audio_bytes: bytes) -> tuple[bytes, int]: + try: + with wave.open(io.BytesIO(audio_bytes), "rb") as wav_file: + pcm_bytes = wav_file.readframes(wav_file.getnframes()) + sample_width = wav_file.getsampwidth() + channels = wav_file.getnchannels() + sample_rate_hz = int(wav_file.getframerate() or self._input_sample_rate_hz) + if sample_width != 2: + return audio_bytes, self._input_sample_rate_hz + if channels == 2: + pcm_bytes = audioop.tomono(pcm_bytes, sample_width, 0.5, 0.5) + return pcm_bytes, sample_rate_hz + except (wave.Error, EOFError): + return audio_bytes, self._input_sample_rate_hz + + +class FallbackSTT(BaseSTT): + def __init__(self, *, primary: BaseSTT, fallback: BaseSTT) -> None: + self._primary = primary + self._fallback = fallback + LOGGER.warning( + "STT fallback configured: primary=%s fallback=%s", + type(primary).__name__, + type(fallback).__name__, + ) + + async def transcribe(self, audio_bytes: bytes) -> str: + try: + return await self._primary.transcribe(audio_bytes) + except Exception: + LOGGER.exception( + "Primary STT provider failed; falling back: primary=%s fallback=%s", + type(self._primary).__name__, + type(self._fallback).__name__, + ) + return await self._fallback.transcribe(audio_bytes) + + async def close(self) -> None: + for provider in (self._primary, self._fallback): + close = getattr(provider, "close", None) + if close is None: + continue + result = close() + if asyncio.iscoroutine(result): + await result + + +def _build_single_stt_provider( + provider: str, + *, + input_sample_rate_hz: int, + target_sample_rate_hz: int, +) -> BaseSTT: + normalized = provider.lower() + if normalized in {"yandex", "yandex_speechkit", "speechkit"}: + return YandexSpeechKitSTT( + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + if normalized in {"elevenlabs", "eleven_labs", "scribe"}: + return ElevenLabsSTT( + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + if normalized in {"openai", "whisper"}: + return OpenAISTT( + input_sample_rate_hz=input_sample_rate_hz, + ) + raise RuntimeError(f"Unsupported STT provider: {provider}") + + +def build_stt_provider( + *, + input_sample_rate_hz: int, + target_sample_rate_hz: int, +) -> BaseSTT: + provider = _stt_provider_name() + primary = _build_single_stt_provider( + provider, + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + fallback_provider = _stt_fallback_provider_name() + if not fallback_provider or fallback_provider == provider: + return primary + fallback = _build_single_stt_provider( + fallback_provider, + input_sample_rate_hz=input_sample_rate_hz, + target_sample_rate_hz=target_sample_rate_hz, + ) + return FallbackSTT(primary=primary, fallback=fallback) diff --git a/providers/stt_openai.py b/providers/stt_openai.py new file mode 100644 index 0000000..76ec087 --- /dev/null +++ b/providers/stt_openai.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import asyncio +import io +import logging +import os +import time +import wave +from typing import Any + +from realtime_voice_service.providers.base import BaseSTT + + +LOGGER = logging.getLogger("uvicorn.error") + + +def _timeout_seconds() -> float: + raw = os.getenv("OPENAI_STT_TIMEOUT_SECONDS") or os.getenv("OPENAI_TIMEOUT_SECONDS") + if raw is None: + return 30.0 + try: + return max(float(raw.strip()), 1.0) + except ValueError: + return 30.0 + + +def _max_retries() -> int: + raw = os.getenv("OPENAI_STT_MAX_RETRIES") + if raw is None: + return 2 + try: + return max(int(raw.strip()), 0) + except ValueError: + return 2 + + +def _preview_text(text: str, *, limit: int = 120) -> str: + normalized = " ".join(str(text or "").split()) + if len(normalized) <= limit: + return normalized + return f"{normalized[:limit]}..." + + +def _pcm16le_to_wav_bytes( + pcm_bytes: bytes, + *, + sample_rate_hz: int, + channels: int = 1, + sample_width_bytes: int = 2, +) -> bytes: + handle = io.BytesIO() + with wave.open(handle, "wb") as wav_file: + wav_file.setnchannels(channels) + wav_file.setsampwidth(sample_width_bytes) + wav_file.setframerate(sample_rate_hz) + wav_file.writeframes(pcm_bytes) + return handle.getvalue() + + +class OpenAISTT(BaseSTT): + name = "openai" + + def __init__( + self, + *, + api_key: str | None = None, + model: str | None = None, + base_url: str | None = None, + input_sample_rate_hz: int = 8000, + prompt: str | None = None, + language: str | None = None, + timeout_seconds: float | None = None, + max_retries: int | None = None, + ) -> None: + self._api_key = str(api_key if api_key is not None else os.getenv("OPENAI_API_KEY", "")).strip() + self._model = str(model or os.getenv("OPENAI_STT_MODEL", "whisper-1")).strip() or "whisper-1" + self._base_url = str( + base_url if base_url is not None else (os.getenv("OPENAI_STT_BASE_URL") or os.getenv("OPENAI_BASE_URL") or "") + ).strip() or None + self._input_sample_rate_hz = max(int(input_sample_rate_hz), 1) + self._prompt = str(prompt if prompt is not None else os.getenv("STT_PROMPT", "")).strip() + self._language = str(language if language is not None else os.getenv("OPENAI_STT_LANGUAGE", "ru")).strip() or None + self._timeout_seconds = max(float(timeout_seconds if timeout_seconds is not None else _timeout_seconds()), 1.0) + self._max_retries = max(int(max_retries if max_retries is not None else _max_retries()), 0) + self._client: Any | None = None + self._openai_module: Any | None = None + LOGGER.info( + "OpenAI STT config: model=%s input_sample_rate=%s language=%s prompt_configured=%s base_url=%s timeout=%s", + self._model, + self._input_sample_rate_hz, + self._language, + bool(self._prompt), + self._base_url or "default", + self._timeout_seconds, + ) + + async def transcribe(self, audio_bytes: bytes) -> str: + if not audio_bytes: + return "" + if not self._api_key: + raise RuntimeError("OPENAI_API_KEY is required for OpenAI STT") + + wav_bytes = _pcm16le_to_wav_bytes(audio_bytes, sample_rate_hz=self._input_sample_rate_hz) + audio_file = io.BytesIO(wav_bytes) + audio_file.name = "utterance.wav" + + started_monotonic = time.perf_counter() + LOGGER.info( + "OpenAI STT request start: model=%s pcm_bytes=%s wav_bytes=%s sample_rate=%s language=%s prompt=%r", + self._model, + len(audio_bytes), + len(wav_bytes), + self._input_sample_rate_hz, + self._language, + _preview_text(self._prompt), + ) + + request: dict[str, object] = { + "file": audio_file, + "model": self._model, + "response_format": "json", + } + if self._prompt: + request["prompt"] = self._prompt + if self._language: + request["language"] = self._language + + try: + response = await self._get_client().audio.transcriptions.create(**request) + except (TimeoutError, asyncio.TimeoutError) as exc: + raise RuntimeError("OpenAI STT request timed out") from exc + except Exception as exc: # noqa: BLE001 + openai_module = self._openai_module + if openai_module is not None and isinstance(exc, getattr(openai_module, "APITimeoutError", ())): + raise RuntimeError("OpenAI STT request timed out") from exc + if openai_module is not None and isinstance(exc, getattr(openai_module, "RateLimitError", ())): + raise RuntimeError("OpenAI STT rate limit exceeded") from exc + if openai_module is not None and isinstance(exc, getattr(openai_module, "APIStatusError", ())): + status_code = getattr(exc, "status_code", "unknown") + raise RuntimeError(f"OpenAI STT returned HTTP {status_code}") from exc + if openai_module is not None and isinstance(exc, getattr(openai_module, "APIConnectionError", ())): + raise RuntimeError("OpenAI STT connection failed") from exc + raise RuntimeError("OpenAI STT request failed") from exc + + transcript_value = getattr(response, "text", None) + if transcript_value is None and isinstance(response, dict): + transcript_value = response.get("text") + transcript = str(transcript_value or "").strip() + LOGGER.info( + "OpenAI STT transcript result: latency_ms=%s chars=%s transcript=%r", + int((time.perf_counter() - started_monotonic) * 1000.0), + len(transcript), + _preview_text(transcript), + ) + return transcript + + def _get_client(self): + if self._client is not None: + return self._client + try: + import httpx + import openai + from openai import AsyncOpenAI + except Exception as exc: # noqa: BLE001 + raise RuntimeError("The `openai` package is required for OpenAI STT") from exc + + timeout = httpx.Timeout( + self._timeout_seconds, + connect=min(self._timeout_seconds, 5.0), + write=min(self._timeout_seconds, 15.0), + read=self._timeout_seconds, + ) + client_kwargs: dict[str, object] = { + "api_key": self._api_key, + "timeout": timeout, + "max_retries": self._max_retries, + } + if self._base_url: + client_kwargs["base_url"] = self._base_url + self._openai_module = openai + self._client = AsyncOpenAI(**client_kwargs) + return self._client + + async def close(self) -> None: + client = self._client + self._client = None + if client is not None: + await client.close() diff --git a/providers/tts.py b/providers/tts.py index 724d4bb..ca53ecb 100644 --- a/providers/tts.py +++ b/providers/tts.py @@ -2,16 +2,35 @@ from __future__ import annotations import asyncio import audioop +import base64 +import json +import logging import os +import time from collections.abc import AsyncGenerator +from collections.abc import AsyncIterable +from urllib.parse import quote +from urllib.parse import urlencode from realtime_voice_service.providers.base import BaseTTS +LOGGER = logging.getLogger("uvicorn.error") + + def _api_base() -> str: return (os.getenv("ELEVENLABS_API_BASE", "https://api.elevenlabs.io").strip() or "https://api.elevenlabs.io").rstrip("/") +def _ws_base(http_base: str) -> str: + normalized = http_base.rstrip("/") + if normalized.startswith("https://"): + return f"wss://{normalized[len('https://') :]}" + if normalized.startswith("http://"): + return f"ws://{normalized[len('http://') :]}" + return normalized + + def _timeout_seconds() -> float: raw = os.getenv("ELEVENLABS_TIMEOUT_SECONDS") if raw is None: @@ -22,78 +41,33 @@ def _timeout_seconds() -> float: return 30.0 -def _parse_output_format(output_format: str) -> tuple[str, int]: +def _parse_chunk_schedule(raw: str | None) -> list[int]: + if not raw: + return [80, 120, 160, 220] + values: list[int] = [] + for part in raw.split(","): + try: + values.append(max(int(part.strip()), 20)) + except ValueError: + continue + return values or [80, 120, 160, 220] + + +def _sample_rate_from_pcm_format(output_format: str) -> int | None: normalized = str(output_format or "").strip().lower() - if normalized == "ulaw_8000": - return "ulaw", 8000 - if normalized == "alaw_8000": - return "alaw", 8000 if not normalized.startswith("pcm_"): - raise ValueError("ElevenLabsTTS expects `pcm_*`, `ulaw_8000`, or `alaw_8000` output formats") - suffix = normalized.split("_", 1)[1] + return None try: - return "pcm16le", int(suffix) - except ValueError as exc: - raise ValueError(f"Unsupported ElevenLabs output format: {output_format}") from exc + return int(normalized.split("_", 1)[1]) + except (IndexError, ValueError): + return None -class _PCM16StreamAdapter: - def __init__(self, *, input_codec: str, input_rate_hz: int, output_rate_hz: int) -> None: - self._input_codec = input_codec - self._input_rate_hz = input_rate_hz - self._output_rate_hz = output_rate_hz - self._carry = b"" - self._state = None - - def process(self, chunk: bytes) -> bytes: - if not chunk: - return b"" - data = self._carry + chunk - sample_width_bytes = 2 if self._input_codec == "pcm16le" else 1 - usable_length = len(data) - (len(data) % sample_width_bytes) - self._carry = data[usable_length:] - if usable_length <= 0: - return b"" - pcm16 = self._decode_to_pcm16(data[:usable_length]) - if self._input_rate_hz == self._output_rate_hz: - return pcm16 - converted, self._state = audioop.ratecv( - pcm16, - 2, - 1, - self._input_rate_hz, - self._output_rate_hz, - self._state, - ) - return converted - - def flush(self) -> bytes: - if not self._carry: - return b"" - sample_width_bytes = 2 if self._input_codec == "pcm16le" else 1 - padded = self._carry + (b"\x00" * ((sample_width_bytes - len(self._carry)) % sample_width_bytes)) - self._carry = b"" - pcm16 = self._decode_to_pcm16(padded) - if self._input_rate_hz == self._output_rate_hz: - return pcm16 - converted, self._state = audioop.ratecv( - pcm16, - 2, - 1, - self._input_rate_hz, - self._output_rate_hz, - self._state, - ) - return converted - - def _decode_to_pcm16(self, chunk: bytes) -> bytes: - if self._input_codec == "pcm16le": - return chunk - if self._input_codec == "ulaw": - return audioop.ulaw2lin(chunk, 2) - if self._input_codec == "alaw": - return audioop.alaw2lin(chunk, 2) - raise ValueError(f"Unsupported input codec: {self._input_codec}") +def _preview_text(text: str, *, limit: int = 120) -> str: + normalized = " ".join(str(text or "").split()) + if len(normalized) <= limit: + return normalized + return f"{normalized[:limit]}..." class ElevenLabsTTS(BaseTTS): @@ -106,78 +80,357 @@ class ElevenLabsTTS(BaseTTS): model_id: str | None = None, language_code: str | None = None, output_format: str | None = None, - target_sample_rate_hz: int = 8000, + target_sample_rate_hz: int | None = None, + inactivity_timeout_seconds: int = 20, timeout_seconds: float | None = None, - stream_chunk_bytes: int = 4096, + auto_mode: bool | None = None, + chunk_length_schedule: list[int] | None = None, ) -> None: self._api_key = str(api_key if api_key is not None else os.getenv("ELEVENLABS_API_KEY", "")).strip() self._api_base = str(api_base or _api_base()).strip().rstrip("/") + self._ws_base = _ws_base(self._api_base) self._voice_id = str(voice_id or os.getenv("ELEVENLABS_TTS_VOICE_ID", "")).strip() - self._model_id = ( - str(model_id or os.getenv("ELEVENLABS_TTS_MODEL_ID", "eleven_flash_v2_5")).strip() - or "eleven_flash_v2_5" + self._requested_model_id = ( + str(model_id or os.getenv("ELEVENLABS_TTS_MODEL_ID", "eleven_turbo_v2_5")).strip() + or "eleven_turbo_v2_5" + ) + self._websocket_fallback_model_id = ( + str(os.getenv("ELEVENLABS_TTS_WS_FALLBACK_MODEL_ID", "eleven_turbo_v2_5")).strip() + or "eleven_turbo_v2_5" + ) + self._model_id = self._resolve_websocket_model_id( + requested_model_id=self._requested_model_id, + fallback_model_id=self._websocket_fallback_model_id, ) self._language_code = str(language_code or os.getenv("ELEVENLABS_TTS_LANGUAGE_CODE", "ru")).strip() or None - self._output_format = str(output_format or os.getenv("ELEVENLABS_TTS_OUTPUT_FORMAT", "pcm_16000")).strip().lower() or "pcm_16000" - self._source_codec, self._source_sample_rate_hz = _parse_output_format(self._output_format) - self._target_sample_rate_hz = max(int(target_sample_rate_hz), 1) + requested_output_format = ( + str(output_format or os.getenv("ELEVENLABS_TTS_OUTPUT_FORMAT", "pcm_16000")).strip().lower() + or "pcm_16000" + ) + self._target_sample_rate_hz = int( + target_sample_rate_hz + if target_sample_rate_hz is not None + else (_sample_rate_from_pcm_format(requested_output_format) or 16000) + ) + self._output_format = self._resolve_provider_output_format( + requested_output_format=requested_output_format, + target_sample_rate_hz=self._target_sample_rate_hz, + ) + self._provider_sample_rate_hz = _sample_rate_from_pcm_format(self._output_format) or self._target_sample_rate_hz self._timeout_seconds = max(float(timeout_seconds if timeout_seconds is not None else _timeout_seconds()), 1.0) - self._stream_chunk_bytes = max(int(stream_chunk_bytes), 256) + self._inactivity_timeout_seconds = max(int(inactivity_timeout_seconds), 5) + self._auto_mode = ( + str(os.getenv("ELEVENLABS_TTS_AUTO_MODE", "1")).strip().lower() in {"1", "true", "yes", "on"} + if auto_mode is None + else bool(auto_mode) + ) + self._chunk_length_schedule = list( + chunk_length_schedule + or _parse_chunk_schedule(os.getenv("ELEVENLABS_TTS_CHUNK_LENGTH_SCHEDULE")) + ) + self._voice_settings = { + "stability": self._read_float_env("ELEVENLABS_TTS_STABILITY", 0.35), + "similarity_boost": self._read_float_env("ELEVENLABS_TTS_SIMILARITY_BOOST", 0.75), + "speed": self._read_float_env("ELEVENLABS_TTS_SPEED", 1.0), + "use_speaker_boost": self._read_bool_env("ELEVENLABS_TTS_USE_SPEAKER_BOOST", False), + } + LOGGER.info( + "ElevenLabs TTS config: voice_id=%s requested_model=%s websocket_model=%s " + "provider_output_format=%s provider_sample_rate=%s target_sample_rate=%s " + "language=%s auto_mode=%s chunk_schedule=%s voice_settings=%s", + self._voice_id, + self._requested_model_id, + self._model_id, + self._output_format, + self._provider_sample_rate_hz, + self._target_sample_rate_hz, + self._language_code, + self._auto_mode, + self._chunk_length_schedule, + self._voice_settings, + ) - async def synthesize_stream(self, text: str) -> AsyncGenerator[bytes, None]: - if not text.strip(): - return + async def synthesize_stream(self, text_stream: AsyncIterable[str]) -> AsyncGenerator[bytes, None]: if not self._api_key: raise RuntimeError("ELEVENLABS_API_KEY is required for ElevenLabs TTS") if not self._voice_id: raise RuntimeError("ELEVENLABS_TTS_VOICE_ID is required for ElevenLabs TTS") try: - import aiohttp + from websockets.exceptions import WebSocketException + from websockets.legacy.client import connect as websocket_connect except Exception as exc: # noqa: BLE001 - raise RuntimeError("The `aiohttp` package is required for ElevenLabs TTS") from exc + raise RuntimeError("The `websockets` package is required for ElevenLabs TTS WebSocket streaming") from exc - payload = { - "text": text, + websocket_url = self._build_websocket_url() + started_monotonic = time.perf_counter() + audio_chunk_count = 0 + audio_byte_count = 0 + yielded_byte_count = 0 + resample_state = None + pcm_remainder = b"" + LOGGER.info( + "ElevenLabs TTS WebSocket connecting: voice_id=%s model=%s output_format=%s " + "provider_sample_rate=%s target_sample_rate=%s language=%s auto_mode=%s", + self._voice_id, + self._model_id, + self._output_format, + self._provider_sample_rate_hz, + self._target_sample_rate_hz, + self._language_code, + self._auto_mode, + ) + try: + async with websocket_connect( + websocket_url, + extra_headers=[("xi-api-key", self._api_key)], + open_timeout=min(self._timeout_seconds, 10.0), + close_timeout=1.0, + ping_interval=20.0, + ping_timeout=20.0, + max_size=None, + ) as websocket: + LOGGER.info( + "ElevenLabs TTS WebSocket connected: voice_id=%s model=%s connect_ms=%s", + self._voice_id, + self._model_id, + int((time.perf_counter() - started_monotonic) * 1000.0), + ) + await websocket.send(json.dumps(self._initial_payload())) + sender_task = asyncio.create_task( + self._send_text_chunks(websocket, text_stream), + name="elevenlabs-tts-sender", + ) + try: + while True: + raw_message = await asyncio.wait_for(websocket.recv(), timeout=self._timeout_seconds) + if isinstance(raw_message, bytes): + raw_message = raw_message.decode("utf-8") + payload = json.loads(raw_message) + if not isinstance(payload, dict): + continue + if payload.get("audio"): + audio_chunk = base64.b64decode(str(payload["audio"])) + audio_chunk_count += 1 + audio_byte_count += len(audio_chunk) + audio_chunk, resample_state, pcm_remainder = self._normalize_pcm_chunk( + audio_chunk, + resample_state=resample_state, + pcm_remainder=pcm_remainder, + ) + yielded_byte_count += len(audio_chunk) + if audio_chunk_count == 1: + LOGGER.info( + "ElevenLabs TTS first audio: voice_id=%s model=%s ttfa_ms=%s " + "provider_bytes=%s yielded_bytes=%s", + self._voice_id, + self._model_id, + int((time.perf_counter() - started_monotonic) * 1000.0), + audio_byte_count, + len(audio_chunk), + ) + elif audio_chunk_count % 10 == 0: + LOGGER.info( + "ElevenLabs TTS audio summary: voice_id=%s chunks=%s provider_bytes=%s yielded_bytes=%s", + self._voice_id, + audio_chunk_count, + audio_byte_count, + yielded_byte_count, + ) + if audio_chunk: + yield audio_chunk + if self._is_error_payload(payload): + LOGGER.error( + "ElevenLabs TTS error payload: voice_id=%s model=%s payload=%s", + self._voice_id, + self._model_id, + self._format_error_payload(payload), + ) + raise RuntimeError(self._format_error_payload(payload)) + if bool(payload.get("isFinal")) or bool(payload.get("is_final")): + LOGGER.info( + "ElevenLabs TTS final payload: voice_id=%s model=%s chunks=%s " + "provider_bytes=%s yielded_bytes=%s total_ms=%s", + self._voice_id, + self._model_id, + audio_chunk_count, + audio_byte_count, + yielded_byte_count, + int((time.perf_counter() - started_monotonic) * 1000.0), + ) + if sender_task.done(): + break + finally: + if not sender_task.done(): + sender_task.cancel() + try: + await sender_task + except asyncio.CancelledError: + pass + except asyncio.TimeoutError as exc: + raise RuntimeError("ElevenLabs TTS WebSocket request timed out") from exc + except WebSocketException as exc: + raise RuntimeError("ElevenLabs TTS WebSocket stream failed") from exc + except OSError as exc: + raise RuntimeError("ElevenLabs TTS WebSocket connection failed") from exc + + def _build_websocket_url(self) -> str: + query = { "model_id": self._model_id, + "output_format": self._output_format, + "inactivity_timeout": self._inactivity_timeout_seconds, + "auto_mode": str(self._auto_mode).lower(), + "sync_alignment": "false", "apply_text_normalization": "auto", } if self._language_code: - payload["language_code"] = self._language_code + query["language_code"] = self._language_code + encoded_voice_id = quote(self._voice_id, safe="") + return f"{self._ws_base}/v1/text-to-speech/{encoded_voice_id}/stream-input?{urlencode(query)}" - adapter = _PCM16StreamAdapter( - input_codec=self._source_codec, - input_rate_hz=self._source_sample_rate_hz, - output_rate_hz=self._target_sample_rate_hz, + def _initial_payload(self) -> dict[str, object]: + payload: dict[str, object] = { + "text": " ", + "xi_api_key": self._api_key, + "voice_settings": self._voice_settings, + } + if not self._auto_mode: + payload["generation_config"] = { + "chunk_length_schedule": self._chunk_length_schedule, + } + return payload + + async def _send_text_chunks(self, websocket, text_stream: AsyncIterable[str]) -> None: + pending_chunk: str | None = None + sent_count = 0 + sent_chars = 0 + async for raw_chunk in text_stream: + normalized = str(raw_chunk).strip() + if not normalized: + continue + normalized_chunk = self._normalize_stream_text(normalized) + if pending_chunk is not None: + await websocket.send(json.dumps({ + "text": pending_chunk, + "try_trigger_generation": True, + })) + sent_count += 1 + sent_chars += len(pending_chunk) + LOGGER.info( + "ElevenLabs TTS text chunk sent: index=%s chars=%s total_chars=%s preview=%r", + sent_count, + len(pending_chunk), + sent_chars, + _preview_text(pending_chunk), + ) + pending_chunk = normalized_chunk + + if pending_chunk is not None: + await websocket.send(json.dumps({ + "text": pending_chunk, + "try_trigger_generation": True, + "flush": True, + })) + sent_count += 1 + sent_chars += len(pending_chunk) + LOGGER.info( + "ElevenLabs TTS final text chunk sent: index=%s chars=%s total_chars=%s preview=%r", + sent_count, + len(pending_chunk), + sent_chars, + _preview_text(pending_chunk), + ) + + await websocket.send(json.dumps({"text": ""})) + LOGGER.info("ElevenLabs TTS text stream closed: chunks=%s chars=%s", sent_count, sent_chars) + + @staticmethod + def _normalize_stream_text(text: str) -> str: + if text.endswith((" ", "\n", "\t")): + return text + return f"{text} " + + @staticmethod + def _resolve_websocket_model_id(*, requested_model_id: str, fallback_model_id: str) -> str: + normalized_requested = requested_model_id.strip() or "eleven_turbo_v2_5" + if normalized_requested not in {"eleven_v3", "eleven_ttv_v3"}: + return normalized_requested + + normalized_fallback = fallback_model_id.strip() or "eleven_turbo_v2_5" + LOGGER.warning( + "ElevenLabs WebSocket TTS does not support model_id=%s; falling back to model_id=%s", + normalized_requested, + normalized_fallback, ) - timeout = aiohttp.ClientTimeout(total=self._timeout_seconds) - try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.post( - f"{self._api_base}/v1/text-to-speech/{self._voice_id}/stream", - params={"output_format": self._output_format}, - headers={ - "xi-api-key": self._api_key, - "Content-Type": "application/json", - }, - json=payload, - ) as response: - if response.status >= 400: - error_text = await response.text() - raise RuntimeError( - f"ElevenLabs TTS returned HTTP {response.status}: {error_text[:300]}" - ) - async for chunk in response.content.iter_chunked(self._stream_chunk_bytes): - if not chunk: - continue - converted = adapter.process(chunk) - if converted: - yield converted - except (TimeoutError, asyncio.TimeoutError) as exc: - raise RuntimeError("ElevenLabs TTS request timed out") from exc - except aiohttp.ClientError as exc: - raise RuntimeError("ElevenLabs TTS request failed") from exc + return normalized_fallback - tail = adapter.flush() - if tail: - yield tail + @staticmethod + def _resolve_provider_output_format(*, requested_output_format: str, target_sample_rate_hz: int) -> str: + normalized = requested_output_format.strip().lower() or "pcm_16000" + if normalized == "pcm_8000": + LOGGER.warning( + "ElevenLabs TTS does not provide reliable PCM16 8kHz streaming; requesting pcm_16000 " + "and resampling to %sHz locally", + target_sample_rate_hz, + ) + return "pcm_16000" + return normalized + + def _normalize_pcm_chunk( + self, + audio_chunk: bytes, + *, + resample_state, + pcm_remainder: bytes, + ) -> tuple[bytes, object, bytes]: + if not audio_chunk: + return b"", resample_state, pcm_remainder + if self._provider_sample_rate_hz == self._target_sample_rate_hz: + return audio_chunk, resample_state, pcm_remainder + + payload = pcm_remainder + audio_chunk + if len(payload) % 2: + pcm_remainder = payload[-1:] + payload = payload[:-1] + else: + pcm_remainder = b"" + if not payload: + return b"", resample_state, pcm_remainder + converted, resample_state = audioop.ratecv( + payload, + 2, + 1, + self._provider_sample_rate_hz, + self._target_sample_rate_hz, + resample_state, + ) + return converted, resample_state, pcm_remainder + + @staticmethod + def _is_error_payload(payload: dict[str, object]) -> bool: + message_type = str(payload.get("message_type") or payload.get("type") or "").strip().lower() + return message_type.endswith("error") or "error" in payload + + @staticmethod + def _format_error_payload(payload: dict[str, object]) -> str: + detail = str(payload.get("message") or payload.get("detail") or payload.get("error") or "").strip() + if detail: + return f"ElevenLabs TTS WebSocket error: {detail}" + return "ElevenLabs TTS WebSocket error" + + @staticmethod + def _read_bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + @staticmethod + def _read_float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + return float(str(raw).strip()) + except ValueError: + return default diff --git a/requirements.txt b/requirements.txt index ac19310..1dc919e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,9 @@ fastapi uvicorn[standard] websockets numpy -torch -torchaudio +--extra-index-url https://download.pytorch.org/whl/cpu +torch==2.3.1+cpu ; platform_system == "Linux" +torchaudio==2.3.1+cpu ; platform_system == "Linux" silero-vad onnxruntime openai diff --git a/transports/audiosocket.py b/transports/audiosocket.py index 59cdd1c..1fe2886 100644 --- a/transports/audiosocket.py +++ b/transports/audiosocket.py @@ -1,8 +1,11 @@ from __future__ import annotations import asyncio +import audioop import logging +import os import struct +import time import uuid from typing import Awaitable, Callable @@ -17,6 +20,16 @@ AUDIO_SOCKET_PACKET_DTMF = 0x03 AUDIO_SOCKET_PACKET_PCM16 = 0x10 +def _float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + return float(str(raw).strip()) + except ValueError: + return default + + def normalize_session_id(value: str | bytes) -> str: if isinstance(value, bytes): try: @@ -54,7 +67,7 @@ class AudioSocketTransport(BaseMediaTransport): transport_id: str, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, frame_duration_ms: int = 20, read_timeout_seconds: float = 30.0, ) -> None: @@ -67,6 +80,19 @@ class AudioSocketTransport(BaseMediaTransport): self._writer = writer self._read_timeout_seconds = max(read_timeout_seconds, 1.0) self._closed = False + self._rx_audio_packet_count = 0 + self._rx_audio_bytes = 0 + self._rx_ignored_packet_count = 0 + self._rx_rms_sum = 0 + self._rx_rms_count = 0 + self._rx_peak_abs = 0 + self._rx_low_level_packet_count = 0 + self._rx_silence_rms_threshold = max(int(os.getenv("REALTIME_VOICE_RX_SILENCE_RMS_THRESHOLD", "80")), 0) + self._rx_log_interval_seconds = max( + _float_env("REALTIME_VOICE_AUDIO_LOG_INTERVAL_SECONDS", 1.0), + 0.1, + ) + self._last_rx_summary_monotonic = time.perf_counter() @property def protocol(self) -> str: @@ -79,7 +105,7 @@ class AudioSocketTransport(BaseMediaTransport): writer: asyncio.StreamWriter, *, handshake_timeout_seconds: float = 5.0, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, frame_duration_ms: int = 20, ) -> AudioSocketTransport: packet_type, payload = await read_packet(reader, timeout_seconds=handshake_timeout_seconds) @@ -103,31 +129,141 @@ class AudioSocketTransport(BaseMediaTransport): self._reader, timeout_seconds=self._read_timeout_seconds, ) - except (asyncio.IncompleteReadError, asyncio.TimeoutError, ConnectionError): + except asyncio.TimeoutError: + LOGGER.info( + "AudioSocket receive timeout: session=%s timeout_seconds=%s rx_packets=%s rx_bytes=%s", + self.transport_id, + self._read_timeout_seconds, + self._rx_audio_packet_count, + self._rx_audio_bytes, + ) + return None + except asyncio.IncompleteReadError as exc: + LOGGER.info( + "AudioSocket peer closed stream: session=%s partial_bytes=%s expected_bytes=%s " + "rx_packets=%s rx_bytes=%s", + self.transport_id, + len(exc.partial or b""), + exc.expected, + self._rx_audio_packet_count, + self._rx_audio_bytes, + ) + return None + except ConnectionError: + LOGGER.info( + "AudioSocket connection error on receive: session=%s rx_packets=%s rx_bytes=%s", + self.transport_id, + self._rx_audio_packet_count, + self._rx_audio_bytes, + ) return None if packet_type == AUDIO_SOCKET_PACKET_HANGUP: + LOGGER.info( + "AudioSocket hangup packet: session=%s rx_packets=%s rx_bytes=%s", + self.transport_id, + self._rx_audio_packet_count, + self._rx_audio_bytes, + ) return None if packet_type == AUDIO_SOCKET_PACKET_PCM16: + self._rx_audio_packet_count += 1 + self._rx_audio_bytes += len(payload) + self._track_rx_audio_level(payload) + self._maybe_log_rx_summary() return payload - if packet_type in {AUDIO_SOCKET_PACKET_UUID, AUDIO_SOCKET_PACKET_DTMF}: + if packet_type == AUDIO_SOCKET_PACKET_DTMF: + self._rx_ignored_packet_count += 1 + LOGGER.info( + "AudioSocket DTMF packet ignored: session=%s payload=%r", + self.transport_id, + payload[:16], + ) continue + if packet_type == AUDIO_SOCKET_PACKET_UUID: + self._rx_ignored_packet_count += 1 + LOGGER.info("AudioSocket duplicate UUID packet ignored: session=%s", self.transport_id) + continue + self._rx_ignored_packet_count += 1 + LOGGER.warning( + "AudioSocket unknown packet ignored: session=%s packet_type=%s payload_bytes=%s", + self.transport_id, + packet_type, + len(payload), + ) return None - async def send_audio(self, audio_chunk: bytes) -> None: + async def _send_frame(self, frame: bytes) -> None: if self._closed or self._writer.is_closing(): return - self._writer.write(encode_audio_packet(audio_chunk)) + self._writer.write(encode_audio_packet(frame)) await self._writer.drain() async def close(self) -> None: if self._closed: return self._closed = True + LOGGER.info( + "AudioSocket transport closing: session=%s rx_packets=%s rx_bytes=%s ignored_packets=%s " + "avg_rms=%s peak_abs=%s low_level_packets=%s", + self.transport_id, + self._rx_audio_packet_count, + self._rx_audio_bytes, + self._rx_ignored_packet_count, + self._average_rx_rms(), + self._rx_peak_abs, + self._rx_low_level_packet_count, + ) if not self._writer.is_closing(): self._writer.close() await self._writer.wait_closed() + def _maybe_log_rx_summary(self, *, force: bool = False) -> None: + now = time.perf_counter() + if not force and (now - self._last_rx_summary_monotonic) < self._rx_log_interval_seconds: + return + self._last_rx_summary_monotonic = now + LOGGER.info( + "AudioSocket audio_rx_summary: session=%s sample_rate=%s frame_ms=%s frame_bytes=%s " + "rx_packets=%s rx_bytes=%s ignored_packets=%s avg_rms=%s peak_abs=%s " + "low_level_packets=%s low_level_ratio=%.3f", + self.transport_id, + self.sample_rate_hz, + self.frame_duration_ms, + self.frame_bytes, + self._rx_audio_packet_count, + self._rx_audio_bytes, + self._rx_ignored_packet_count, + self._average_rx_rms(), + self._rx_peak_abs, + self._rx_low_level_packet_count, + self._rx_low_level_ratio(), + ) + + def _track_rx_audio_level(self, payload: bytes) -> None: + if not payload: + return + try: + rms = int(audioop.rms(payload, 2)) + peak = int(audioop.max(payload, 2)) + except Exception: + return + self._rx_rms_sum += rms + self._rx_rms_count += 1 + self._rx_peak_abs = max(self._rx_peak_abs, peak) + if rms <= self._rx_silence_rms_threshold: + self._rx_low_level_packet_count += 1 + + def _average_rx_rms(self) -> int: + if self._rx_rms_count <= 0: + return 0 + return int(self._rx_rms_sum / self._rx_rms_count) + + def _rx_low_level_ratio(self) -> float: + if self._rx_rms_count <= 0: + return 0.0 + return self._rx_low_level_packet_count / float(self._rx_rms_count) + class AudioSocketServer: def __init__( @@ -137,7 +273,7 @@ class AudioSocketServer: port: int, session_handler: Callable[[AudioSocketTransport], Awaitable[None]], handshake_timeout_seconds: float = 5.0, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, frame_duration_ms: int = 20, ) -> None: self._host = host @@ -164,9 +300,12 @@ class AudioSocketServer: self._port, ) LOGGER.info( - "AudioSocket server listening on %s:%s", + "AudioSocket server listening on %s:%s sample_rate=%s frame_ms=%s frame_bytes=%s", self._host, self.bound_port, + self._sample_rate_hz, + self._frame_duration_ms, + int((self._sample_rate_hz * self._frame_duration_ms / 1000.0) * 2), ) async def stop(self) -> None: @@ -205,9 +344,12 @@ class AudioSocketServer: frame_duration_ms=self._frame_duration_ms, ) LOGGER.info( - "AudioSocket client accepted: session=%s peer=%s", + "AudioSocket client accepted: session=%s peer=%s sample_rate=%s frame_ms=%s frame_bytes=%s", transport.transport_id, peer, + transport.sample_rate_hz, + transport.frame_duration_ms, + transport.frame_bytes, ) await self._session_handler(transport) except asyncio.CancelledError: @@ -216,9 +358,11 @@ class AudioSocketServer: LOGGER.exception("AudioSocket connection failed from peer=%s", peer) finally: if transport is not None: + transport._maybe_log_rx_summary(force=True) await transport.close() elif not writer.is_closing(): writer.close() await writer.wait_closed() + LOGGER.info("AudioSocket connection finished: peer=%s session=%s", peer, transport.transport_id if transport else None) if current_task is not None: self._connection_tasks.discard(current_task) diff --git a/transports/base.py b/transports/base.py index 020271b..13de16c 100644 --- a/transports/base.py +++ b/transports/base.py @@ -1,19 +1,53 @@ from __future__ import annotations +import asyncio +import logging +import os +import time from abc import ABC, abstractmethod +from realtime_voice_service.core.audio_pacer import AudioPacer + + +LOGGER = logging.getLogger("uvicorn.error") + + +def _float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + return float(str(raw).strip()) + except ValueError: + return default + class BaseMediaTransport(ABC): def __init__( self, *, transport_id: str, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, frame_duration_ms: int = 20, ) -> None: self._transport_id = transport_id self._sample_rate_hz = sample_rate_hz self._frame_duration_ms = frame_duration_ms + self._audio_pacer = AudioPacer(frame_bytes=self.frame_bytes) + self._send_lock = asyncio.Lock() + self._next_frame_monotonic: float | None = None + self._send_generation = 0 + self._audio_log_interval_seconds = max( + _float_env("REALTIME_VOICE_AUDIO_LOG_INTERVAL_SECONDS", 1.0), + 0.1, + ) + self._last_tx_summary_monotonic = time.perf_counter() + self._tx_chunk_count = 0 + self._tx_input_bytes = 0 + self._tx_frame_count = 0 + self._tx_frame_bytes = 0 + self._tx_flush_count = 0 + self._tx_clear_count = 0 @property def transport_id(self) -> str: @@ -40,10 +74,131 @@ class BaseMediaTransport(ABC): async def receive_audio(self) -> bytes | None: raise NotImplementedError - @abstractmethod async def send_audio(self, audio_chunk: bytes) -> None: - raise NotImplementedError + if not audio_chunk: + return + async with self._send_lock: + generation = self._send_generation + frames = self._audio_pacer.push(audio_chunk) + self._tx_chunk_count += 1 + self._tx_input_bytes += len(audio_chunk) + self._tx_frame_count += len(frames) + self._tx_frame_bytes += sum(len(frame) for frame in frames) + self._maybe_log_tx_summary(generation=generation) + for frame in frames: + if generation != self._send_generation: + return + await self._pace_and_send_frame(frame, generation=generation) + + async def flush_audio(self, *, pad_final_frame: bool = True) -> None: + async with self._send_lock: + generation = self._send_generation + buffered_before = self._audio_pacer.buffered_bytes + frames = self._audio_pacer.flush(pad_final_frame=pad_final_frame) + self._tx_flush_count += 1 + self._tx_frame_count += len(frames) + self._tx_frame_bytes += sum(len(frame) for frame in frames) + LOGGER.info( + "transport %s flush_audio protocol=%s generation=%s frames=%s frame_bytes=%s " + "buffered_before=%s pad_final_frame=%s", + self.transport_id, + self.protocol, + generation, + len(frames), + sum(len(frame) for frame in frames), + buffered_before, + pad_final_frame, + ) + self._maybe_log_tx_summary(generation=generation, force=True) + for frame in frames: + if generation != self._send_generation: + return + await self._pace_and_send_frame(frame, generation=generation) + + def clear_buffer(self) -> None: + previous_generation = self._send_generation + buffered_before = self._audio_pacer.buffered_bytes + self._send_generation += 1 + self._tx_clear_count += 1 + self._audio_pacer.clear() + self._next_frame_monotonic = None + LOGGER.info( + "transport %s clear_buffer protocol=%s generation=%s->%s buffered_bytes=%s " + "tx_chunks=%s tx_frames=%s tx_bytes=%s clears=%s", + self.transport_id, + self.protocol, + previous_generation, + self._send_generation, + buffered_before, + self._tx_chunk_count, + self._tx_frame_count, + self._tx_frame_bytes, + self._tx_clear_count, + ) + + def discard_audio_buffer(self) -> None: + self.clear_buffer() @abstractmethod async def close(self) -> None: raise NotImplementedError + + @abstractmethod + async def _send_frame(self, frame: bytes) -> None: + raise NotImplementedError + + async def _pace_and_send_frame(self, frame: bytes, *, generation: int) -> None: + if generation != self._send_generation: + return + frame_duration_seconds = self.frame_duration_ms / 1000.0 + now = time.perf_counter() + if ( + self._next_frame_monotonic is None + or now > (self._next_frame_monotonic + (frame_duration_seconds * 4.0)) + ): + self._next_frame_monotonic = now + sleep_for = self._next_frame_monotonic - now + if sleep_for > 0: + await asyncio.sleep(sleep_for) + if generation != self._send_generation: + return + send_started_monotonic = time.perf_counter() + await self._send_frame(frame) + send_duration_ms = int((time.perf_counter() - send_started_monotonic) * 1000.0) + if send_duration_ms > (self.frame_duration_ms * 2): + LOGGER.warning( + "transport %s slow_frame_send protocol=%s generation=%s duration_ms=%s frame_bytes=%s", + self.transport_id, + self.protocol, + generation, + send_duration_ms, + len(frame), + ) + if generation != self._send_generation: + return + baseline = max(time.perf_counter(), self._next_frame_monotonic) + self._next_frame_monotonic = baseline + frame_duration_seconds + + def _maybe_log_tx_summary(self, *, generation: int, force: bool = False) -> None: + now = time.perf_counter() + if not force and (now - self._last_tx_summary_monotonic) < self._audio_log_interval_seconds: + return + self._last_tx_summary_monotonic = now + LOGGER.info( + "transport %s audio_tx_summary protocol=%s generation=%s sample_rate=%s frame_ms=%s " + "frame_bytes=%s chunks=%s input_bytes=%s frames=%s frame_payload_bytes=%s " + "pacer_buffered=%s flushes=%s clears=%s", + self.transport_id, + self.protocol, + generation, + self.sample_rate_hz, + self.frame_duration_ms, + self.frame_bytes, + self._tx_chunk_count, + self._tx_input_bytes, + self._tx_frame_count, + self._tx_frame_bytes, + self._audio_pacer.buffered_bytes, + self._tx_flush_count, + self._tx_clear_count, + ) diff --git a/transports/websocket.py b/transports/websocket.py index 7d0fac7..70a8772 100644 --- a/transports/websocket.py +++ b/transports/websocket.py @@ -15,7 +15,7 @@ class WebSocketMediaTransport(BaseMediaTransport): *, websocket: WebSocket, transport_id: str, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, frame_duration_ms: int = 20, ) -> None: super().__init__( @@ -52,10 +52,10 @@ class WebSocketMediaTransport(BaseMediaTransport): return decoded return None - async def send_audio(self, audio_chunk: bytes) -> None: + async def _send_frame(self, frame: bytes) -> None: if self._closed: return - await self._websocket.send_bytes(audio_chunk) + await self._websocket.send_bytes(frame) async def close(self) -> None: if self._closed: