feat(voice): switch voice ASR to Yandex SpeechKit
This commit is contained in:
Yera All
@@ -84,7 +84,7 @@ def _interaction_service_url() -> str:
|
||||
|
||||
|
||||
def _asr_provider_name() -> str:
|
||||
return os.getenv("AI_VOICE_ASR_PROVIDER", "openai").strip() or "openai"
|
||||
return os.getenv("AI_VOICE_ASR_PROVIDER", "yandex").strip() or "yandex"
|
||||
|
||||
|
||||
def _tts_provider_name() -> str:
|
||||
@@ -164,7 +164,7 @@ def _voice_v2_duplex_enabled() -> bool:
|
||||
|
||||
|
||||
def _voice_v2_streaming_asr_backend() -> str:
|
||||
return str(os.getenv("AI_VOICE_V2_STREAMING_ASR_BACKEND", "local_sidecar") or "local_sidecar").strip().lower()
|
||||
return str(os.getenv("AI_VOICE_V2_STREAMING_ASR_BACKEND", "yandex_speechkit") or "yandex_speechkit").strip().lower()
|
||||
|
||||
|
||||
def _voice_v2_prebaked_ack_enabled() -> bool:
|
||||
|
||||
@@ -1,7 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import wave
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
@@ -30,6 +35,99 @@ def _openai_asr_model() -> str:
|
||||
return os.getenv("AI_VOICE_ASR_MODEL", "gpt-4o-mini-transcribe").strip() or "gpt-4o-mini-transcribe"
|
||||
|
||||
|
||||
def _yandex_asr_api_base() -> str:
|
||||
explicit = os.getenv("AI_VOICE_ASR_YANDEX_API_BASE", "").strip()
|
||||
if explicit:
|
||||
return explicit.rstrip("/")
|
||||
tts_base = os.getenv("AI_VOICE_TTS_YANDEX_API_BASE", "https://tts.api.ml.yandexcloud.kz").strip().lower()
|
||||
if "yandexcloud.kz" in tts_base:
|
||||
return "https://stt.api.ml.yandexcloud.kz"
|
||||
return "https://stt.api.cloud.yandex.net"
|
||||
|
||||
|
||||
def _yandex_asr_api_key() -> str:
|
||||
return (
|
||||
os.getenv("AI_VOICE_ASR_YANDEX_API_KEY", "").strip()
|
||||
or os.getenv("AI_VOICE_TTS_YANDEX_API_KEY", "").strip()
|
||||
)
|
||||
|
||||
|
||||
def _yandex_asr_iam_token() -> str:
|
||||
return (
|
||||
os.getenv("AI_VOICE_ASR_YANDEX_IAM_TOKEN", "").strip()
|
||||
or os.getenv("AI_VOICE_TTS_YANDEX_IAM_TOKEN", "").strip()
|
||||
)
|
||||
|
||||
|
||||
def _yandex_asr_folder_id() -> str:
|
||||
return (
|
||||
os.getenv("AI_VOICE_ASR_YANDEX_FOLDER_ID", "").strip()
|
||||
or os.getenv("AI_VOICE_TTS_YANDEX_FOLDER_ID", "").strip()
|
||||
)
|
||||
|
||||
|
||||
def _yandex_asr_timeout_seconds() -> float:
|
||||
raw = os.getenv("AI_VOICE_ASR_YANDEX_TIMEOUT_SECONDS", "").strip()
|
||||
if not raw:
|
||||
return _timeout_seconds()
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
value = _timeout_seconds()
|
||||
return max(value, 3.0)
|
||||
|
||||
|
||||
def _yandex_asr_sample_rate_hz() -> int:
|
||||
raw = os.getenv("AI_VOICE_ASR_YANDEX_SAMPLE_RATE_HZ", "8000").strip()
|
||||
try:
|
||||
value = int(raw)
|
||||
except ValueError:
|
||||
value = 8000
|
||||
if value < 8000 or value > 48000:
|
||||
return 8000
|
||||
return value
|
||||
|
||||
|
||||
def _yandex_asr_topic() -> str:
|
||||
return os.getenv("AI_VOICE_ASR_YANDEX_TOPIC", "general").strip() or "general"
|
||||
|
||||
|
||||
def _yandex_asr_api_version(api_base: str) -> str:
|
||||
raw = os.getenv("AI_VOICE_ASR_YANDEX_API_VERSION", "auto").strip().lower() or "auto"
|
||||
if raw in {"v1", "v1_sync", "sync"}:
|
||||
return "v1"
|
||||
if raw in {"v3", "v3_async", "async"}:
|
||||
return "v3"
|
||||
if "yandexcloud.kz" in str(api_base or "").lower():
|
||||
return "v3"
|
||||
return "v1"
|
||||
|
||||
|
||||
def _yandex_asr_poll_interval_seconds() -> float:
|
||||
raw = os.getenv("AI_VOICE_ASR_YANDEX_POLL_INTERVAL_SECONDS", "0.25").strip()
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
value = 0.25
|
||||
return max(min(value, 2.0), 0.05)
|
||||
|
||||
|
||||
def _yandex_asr_default_language() -> str:
|
||||
return os.getenv("AI_VOICE_ASR_YANDEX_LANGUAGE", "ru-RU").strip() or "ru-RU"
|
||||
|
||||
|
||||
def _normalize_yandex_asr_language(language: str | None) -> str:
|
||||
raw = str(language or "").strip()
|
||||
if not raw:
|
||||
return _yandex_asr_default_language()
|
||||
lowered = raw.replace("_", "-").lower()
|
||||
if lowered in {"ru", "ru-ru"}:
|
||||
return "ru-RU"
|
||||
if lowered in {"kk", "kz", "kk-kz", "kz-kz"}:
|
||||
return "kk-KZ"
|
||||
return raw
|
||||
|
||||
|
||||
def _streaming_asr_api_base() -> str:
|
||||
return (
|
||||
os.getenv("AI_VOICE_V2_STREAMING_ASR_BASE_URL", "http://127.0.0.1:8021").strip()
|
||||
@@ -152,6 +250,214 @@ class OpenAIASRProvider(ASRProvider):
|
||||
return self.transcribe(audio_bytes, language_hint=language_hint)
|
||||
|
||||
|
||||
class YandexSpeechKitASRProvider(ASRProvider):
|
||||
name = "yandex"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
iam_token: str | None = None,
|
||||
folder_id: str | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
sample_rate_hz: int | None = None,
|
||||
topic: str | None = None,
|
||||
) -> None:
|
||||
self._api_base = str(api_base or _yandex_asr_api_base()).strip().rstrip("/")
|
||||
self._api_key = str(api_key if api_key is not None else _yandex_asr_api_key()).strip()
|
||||
self._iam_token = str(iam_token if iam_token is not None else _yandex_asr_iam_token()).strip()
|
||||
self._folder_id = str(folder_id if folder_id is not None else _yandex_asr_folder_id()).strip()
|
||||
self._timeout_seconds = max(
|
||||
float(timeout_seconds if timeout_seconds is not None else _yandex_asr_timeout_seconds()),
|
||||
3.0,
|
||||
)
|
||||
self._sample_rate_hz = int(sample_rate_hz if sample_rate_hz is not None else _yandex_asr_sample_rate_hz())
|
||||
self._topic = str(topic if topic is not None else _yandex_asr_topic()).strip()
|
||||
self._api_version = _yandex_asr_api_version(self._api_base)
|
||||
self._poll_interval_seconds = _yandex_asr_poll_interval_seconds()
|
||||
|
||||
@staticmethod
|
||||
def _lpcm_from_audio_bytes(audio_bytes: bytes, *, default_sample_rate_hz: int) -> tuple[bytes, int]:
|
||||
if not audio_bytes:
|
||||
return b"", default_sample_rate_hz
|
||||
try:
|
||||
with wave.open(io.BytesIO(audio_bytes), "rb") as wav_file:
|
||||
sample_width = wav_file.getsampwidth()
|
||||
channels = wav_file.getnchannels()
|
||||
sample_rate = wav_file.getframerate()
|
||||
if sample_width != 2 or channels != 1:
|
||||
return audio_bytes, default_sample_rate_hz
|
||||
return wav_file.readframes(wav_file.getnframes()), int(sample_rate or default_sample_rate_hz)
|
||||
except (wave.Error, EOFError):
|
||||
return audio_bytes, default_sample_rate_hz
|
||||
|
||||
def _headers(self, *, content_type: str = "application/octet-stream") -> dict[str, str]:
|
||||
if not self._api_key and not self._iam_token:
|
||||
raise RuntimeError(
|
||||
"AI_VOICE_ASR_YANDEX_API_KEY or AI_VOICE_ASR_YANDEX_IAM_TOKEN is required for Yandex ASR"
|
||||
)
|
||||
headers = {"Content-Type": content_type}
|
||||
if self._api_key:
|
||||
headers["Authorization"] = f"Api-Key {self._api_key}"
|
||||
else:
|
||||
headers["Authorization"] = f"Bearer {self._iam_token}"
|
||||
if self._folder_id and self._api_version == "v3":
|
||||
headers["x-folder-id"] = self._folder_id
|
||||
return headers
|
||||
|
||||
def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
|
||||
pcm_bytes, sample_rate_hz = self._lpcm_from_audio_bytes(
|
||||
audio_bytes,
|
||||
default_sample_rate_hz=self._sample_rate_hz,
|
||||
)
|
||||
if not pcm_bytes:
|
||||
return ASRTranscription(text="", language=language_hint, confidence=None)
|
||||
|
||||
language = _normalize_yandex_asr_language(language_hint)
|
||||
if self._api_version == "v3":
|
||||
return self._transcribe_v3_async(pcm_bytes, sample_rate_hz=sample_rate_hz, language=language)
|
||||
return self._transcribe_v1_sync(pcm_bytes, sample_rate_hz=sample_rate_hz, language=language)
|
||||
|
||||
def _transcribe_v1_sync(self, pcm_bytes: bytes, *, sample_rate_hz: int, language: str) -> ASRTranscription:
|
||||
params: dict[str, str] = {
|
||||
"lang": language,
|
||||
"format": "lpcm",
|
||||
"sampleRateHertz": str(sample_rate_hz),
|
||||
}
|
||||
if self._topic:
|
||||
params["topic"] = self._topic
|
||||
if self._folder_id:
|
||||
params["folderId"] = self._folder_id
|
||||
|
||||
with httpx.Client(timeout=self._timeout_seconds) as client:
|
||||
response = client.post(
|
||||
f"{self._api_base}/speech/v1/stt:recognize",
|
||||
headers=self._headers(),
|
||||
params=params,
|
||||
content=pcm_bytes,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
return ASRTranscription(
|
||||
text=str(payload.get("result") or "").strip(),
|
||||
language=language,
|
||||
confidence=None,
|
||||
)
|
||||
|
||||
def _transcribe_v3_async(self, pcm_bytes: bytes, *, sample_rate_hz: int, language: str) -> ASRTranscription:
|
||||
payload = {
|
||||
"content": base64.b64encode(pcm_bytes).decode("ascii"),
|
||||
"recognitionModel": {
|
||||
"model": self._topic or "general",
|
||||
"audioFormat": {
|
||||
"rawAudio": {
|
||||
"audioEncoding": "LINEAR16_PCM",
|
||||
"sampleRateHertz": str(sample_rate_hz),
|
||||
"audioChannelCount": "1",
|
||||
}
|
||||
},
|
||||
"textNormalization": {
|
||||
"textNormalization": "TEXT_NORMALIZATION_ENABLED",
|
||||
"profanityFilter": False,
|
||||
"literatureText": False,
|
||||
"phoneFormattingMode": "PHONE_FORMATTING_MODE_DISABLED",
|
||||
},
|
||||
"languageRestriction": {
|
||||
"restrictionType": "WHITELIST",
|
||||
"languageCode": [language],
|
||||
},
|
||||
"audioProcessingType": "FULL_DATA",
|
||||
},
|
||||
}
|
||||
headers = self._headers(content_type="application/json")
|
||||
deadline = time.monotonic() + self._timeout_seconds
|
||||
with httpx.Client(timeout=self._timeout_seconds) as client:
|
||||
response = client.post(
|
||||
f"{self._api_base}/stt/v3/recognizeFileAsync",
|
||||
headers=headers,
|
||||
json=payload,
|
||||
)
|
||||
response.raise_for_status()
|
||||
operation_payload = response.json()
|
||||
operation_id = str(operation_payload.get("id") or "").strip()
|
||||
if not operation_id:
|
||||
raise RuntimeError("Yandex ASR v3 response is missing operation id")
|
||||
|
||||
while not bool(operation_payload.get("done")):
|
||||
if time.monotonic() >= deadline:
|
||||
raise TimeoutError("Yandex ASR v3 recognition timed out")
|
||||
time.sleep(self._poll_interval_seconds)
|
||||
operation_response = client.get(
|
||||
f"{self._api_base}/operations/{operation_id}",
|
||||
headers=headers,
|
||||
)
|
||||
operation_response.raise_for_status()
|
||||
operation_payload = operation_response.json()
|
||||
|
||||
error_payload = operation_payload.get("error")
|
||||
if isinstance(error_payload, dict) and error_payload:
|
||||
raise RuntimeError(str(error_payload.get("message") or error_payload)[:500])
|
||||
|
||||
result_response = client.get(
|
||||
f"{self._api_base}/stt/v3/getRecognition",
|
||||
headers=headers,
|
||||
params={"operationId": operation_id},
|
||||
)
|
||||
result_response.raise_for_status()
|
||||
result_payload = result_response.json()
|
||||
|
||||
return ASRTranscription(
|
||||
text=self._extract_v3_text(result_payload),
|
||||
language=language,
|
||||
confidence=None,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _extract_v3_text(cls, payload: object) -> str:
|
||||
if isinstance(payload, list):
|
||||
texts = [cls._extract_v3_text(item) for item in payload]
|
||||
return " ".join(text for text in texts if text).strip()
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
|
||||
final_refinement = payload.get("finalRefinement")
|
||||
if isinstance(final_refinement, dict):
|
||||
normalized = final_refinement.get("normalizedText")
|
||||
text = cls._extract_alternatives_text(normalized)
|
||||
if text:
|
||||
return text
|
||||
|
||||
for key in ("final", "partial"):
|
||||
text = cls._extract_alternatives_text(payload.get(key))
|
||||
if text:
|
||||
return text
|
||||
|
||||
response = payload.get("response")
|
||||
if isinstance(response, (dict, list)):
|
||||
return cls._extract_v3_text(response)
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _extract_alternatives_text(payload: object) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
alternatives = payload.get("alternatives")
|
||||
if not isinstance(alternatives, list):
|
||||
return ""
|
||||
texts: list[str] = []
|
||||
for alternative in alternatives:
|
||||
if not isinstance(alternative, dict):
|
||||
continue
|
||||
text = str(alternative.get("text") or "").strip()
|
||||
if text:
|
||||
texts.append(text)
|
||||
return " ".join(texts).strip()
|
||||
|
||||
def transcribe_partial(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
|
||||
return self.transcribe(audio_bytes, language_hint=language_hint)
|
||||
|
||||
|
||||
class LocalSidecarStreamingASRProvider(StreamingASRProvider):
|
||||
name = "local-sidecar"
|
||||
supports_streaming = True
|
||||
@@ -245,10 +551,60 @@ class LocalSidecarStreamingASRProvider(StreamingASRProvider):
|
||||
return
|
||||
|
||||
|
||||
class YandexSpeechKitBufferedStreamingASRProvider(StreamingASRProvider):
|
||||
name = "yandex-speechkit-buffered"
|
||||
supports_streaming = True
|
||||
|
||||
def __init__(self, *, asr_provider: YandexSpeechKitASRProvider | None = None) -> None:
|
||||
self._asr_provider = asr_provider or YandexSpeechKitASRProvider()
|
||||
self._streams: dict[str, dict[str, object]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def open_stream(self, session_id: str, *, language_hint: str | None = None) -> str:
|
||||
stream_id = f"yasr_{uuid.uuid4().hex}"
|
||||
with self._lock:
|
||||
self._streams[stream_id] = {
|
||||
"session_id": str(session_id or "").strip(),
|
||||
"language_hint": str(language_hint or "").strip() or None,
|
||||
"pcm": bytearray(),
|
||||
}
|
||||
return stream_id
|
||||
|
||||
def push_pcm(self, stream_id: str, pcm_8k_chunk: bytes) -> None:
|
||||
if not pcm_8k_chunk:
|
||||
return
|
||||
with self._lock:
|
||||
stream = self._streams.get(stream_id)
|
||||
if stream is None:
|
||||
raise StreamingASRUnavailable("Yandex ASR stream is not active")
|
||||
pcm = stream.get("pcm")
|
||||
if isinstance(pcm, bytearray):
|
||||
pcm.extend(pcm_8k_chunk)
|
||||
|
||||
def poll_partial(self, stream_id: str) -> StreamingASRPartial | None:
|
||||
del stream_id
|
||||
return None
|
||||
|
||||
def finalize(self, stream_id: str) -> ASRTranscription:
|
||||
with self._lock:
|
||||
stream = self._streams.get(stream_id)
|
||||
if stream is None:
|
||||
raise StreamingASRUnavailable("Yandex ASR stream is not active")
|
||||
pcm = bytes(stream.get("pcm") or b"")
|
||||
language_hint = str(stream.get("language_hint") or "").strip() or None
|
||||
return self._asr_provider.transcribe(pcm, language_hint=language_hint)
|
||||
|
||||
def close_stream(self, stream_id: str) -> None:
|
||||
with self._lock:
|
||||
self._streams.pop(stream_id, None)
|
||||
|
||||
|
||||
def build_asr_provider(name: str) -> ASRProvider:
|
||||
normalized = str(name or "stub").strip().lower()
|
||||
if normalized == "openai":
|
||||
return OpenAIASRProvider()
|
||||
if normalized in {"yandex", "yandex_speechkit", "speechkit"}:
|
||||
return YandexSpeechKitASRProvider()
|
||||
return ASRProvider()
|
||||
|
||||
|
||||
@@ -256,4 +612,6 @@ def build_streaming_asr_provider(name: str) -> StreamingASRProvider:
|
||||
normalized = str(name or "disabled").strip().lower()
|
||||
if normalized in {"local_sidecar", "local-sidecar", "sidecar"}:
|
||||
return LocalSidecarStreamingASRProvider()
|
||||
if normalized in {"yandex", "yandex_speechkit", "yandex-speechkit", "speechkit"}:
|
||||
return YandexSpeechKitBufferedStreamingASRProvider()
|
||||
return StreamingASRProvider()
|
||||
|
||||
Reference in New Issue
Block a user