feat(voice): add elevenlabs realtime streaming asr
This commit is contained in:
Yera All
@@ -164,7 +164,7 @@ def _voice_v2_duplex_enabled() -> bool:
|
||||
|
||||
|
||||
def _voice_v2_streaming_asr_backend() -> str:
|
||||
return str(os.getenv("AI_VOICE_V2_STREAMING_ASR_BACKEND", "yandex_speechkit") or "yandex_speechkit").strip().lower()
|
||||
return str(os.getenv("AI_VOICE_V2_STREAMING_ASR_BACKEND", "elevenlabs_realtime") or "elevenlabs_realtime").strip().lower()
|
||||
|
||||
|
||||
def _voice_v2_prebaked_ack_enabled() -> bool:
|
||||
|
||||
@@ -3,12 +3,15 @@ from __future__ import annotations
|
||||
import audioop
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
import wave
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import httpx
|
||||
@@ -89,6 +92,73 @@ def _elevenlabs_asr_sample_rate_hz() -> int:
|
||||
return value
|
||||
|
||||
|
||||
def _elevenlabs_realtime_model_id() -> str:
|
||||
return (
|
||||
os.getenv("AI_VOICE_ASR_ELEVENLABS_REALTIME_MODEL_ID", "scribe_v2_realtime").strip()
|
||||
or "scribe_v2_realtime"
|
||||
)
|
||||
|
||||
|
||||
def _elevenlabs_realtime_audio_format() -> str:
|
||||
return (
|
||||
os.getenv("AI_VOICE_ASR_ELEVENLABS_REALTIME_AUDIO_FORMAT", "pcm_16000").strip().lower()
|
||||
or "pcm_16000"
|
||||
)
|
||||
|
||||
|
||||
def _elevenlabs_realtime_sample_rate_hz(audio_format: str | None = None) -> int:
|
||||
explicit = os.getenv("AI_VOICE_ASR_ELEVENLABS_REALTIME_SAMPLE_RATE_HZ", "").strip()
|
||||
if explicit:
|
||||
try:
|
||||
value = int(explicit)
|
||||
except ValueError:
|
||||
value = 16000
|
||||
return value if 8000 <= value <= 48000 else 16000
|
||||
|
||||
normalized = str(audio_format or _elevenlabs_realtime_audio_format()).strip().lower()
|
||||
if normalized.startswith("pcm_"):
|
||||
try:
|
||||
value = int(normalized.rsplit("_", 1)[-1])
|
||||
except ValueError:
|
||||
value = 16000
|
||||
return value if 8000 <= value <= 48000 else 16000
|
||||
return 16000
|
||||
|
||||
|
||||
def _elevenlabs_realtime_commit_strategy() -> str:
|
||||
raw = os.getenv("AI_VOICE_ASR_ELEVENLABS_REALTIME_COMMIT_STRATEGY", "manual").strip().lower()
|
||||
return raw if raw in {"manual", "vad"} else "manual"
|
||||
|
||||
|
||||
def _elevenlabs_realtime_finalize_timeout_seconds() -> float:
|
||||
raw = os.getenv("AI_VOICE_ASR_ELEVENLABS_REALTIME_FINALIZE_TIMEOUT_SECONDS", "").strip()
|
||||
if not raw:
|
||||
return max(_streaming_asr_timeout_seconds(), 1.0)
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
value = _streaming_asr_timeout_seconds()
|
||||
return max(value, 1.0)
|
||||
|
||||
|
||||
def _normalize_elevenlabs_realtime_language(language: str | None) -> str:
|
||||
raw = str(language or "").strip().lower().replace("_", "-")
|
||||
if not raw:
|
||||
raw = _elevenlabs_asr_default_language().strip().lower().replace("_", "-")
|
||||
mapping = {
|
||||
"rus": "ru",
|
||||
"ru-ru": "ru",
|
||||
"kaz": "kk",
|
||||
"kz": "kk",
|
||||
"kk-kz": "kk",
|
||||
"kz-kz": "kk",
|
||||
"eng": "en",
|
||||
"en-us": "en",
|
||||
"en-gb": "en",
|
||||
}
|
||||
return mapping.get(raw, raw or "ru")
|
||||
|
||||
|
||||
def _resolve_elevenlabs_asr_language(language_code: str | None, language_hint: str | None) -> str | None:
|
||||
normalized = str(language_code or "").strip().lower()
|
||||
if normalized in {"rus", "ru", "ru-ru"}:
|
||||
@@ -434,6 +504,330 @@ class ElevenLabsASRProvider(ASRProvider):
|
||||
return self.transcribe(audio_bytes, language_hint=language_hint)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ElevenLabsRealtimeStreamState:
|
||||
stream_id: str
|
||||
session_id: str
|
||||
language: str
|
||||
websocket: object
|
||||
updates_queue: queue.Queue[StreamingASRPartial]
|
||||
thread: threading.Thread | None = None
|
||||
latest_partial: StreamingASRPartial | None = None
|
||||
final_transcription: ASRTranscription | None = None
|
||||
error: BaseException | None = None
|
||||
close_requested: bool = False
|
||||
final_event: threading.Event = field(default_factory=threading.Event)
|
||||
lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
send_lock: threading.Lock = field(default_factory=threading.Lock)
|
||||
|
||||
|
||||
class ElevenLabsRealtimeStreamingASRProvider(StreamingASRProvider):
|
||||
name = "elevenlabs-realtime"
|
||||
supports_streaming = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
api_base: str | None = None,
|
||||
api_key: str | None = None,
|
||||
timeout_seconds: float | None = None,
|
||||
finalize_timeout_seconds: float | None = None,
|
||||
model_id: str | None = None,
|
||||
audio_format: str | None = None,
|
||||
commit_strategy: str | None = None,
|
||||
websocket_factory: Callable[..., object] | None = None,
|
||||
) -> None:
|
||||
self._api_base = str(api_base or _elevenlabs_asr_api_base()).strip().rstrip("/")
|
||||
self._api_key = str(api_key if api_key is not None else _elevenlabs_asr_api_key()).strip()
|
||||
self._timeout_seconds = max(
|
||||
float(timeout_seconds if timeout_seconds is not None else _streaming_asr_timeout_seconds()),
|
||||
0.25,
|
||||
)
|
||||
self._finalize_timeout_seconds = max(
|
||||
float(
|
||||
finalize_timeout_seconds
|
||||
if finalize_timeout_seconds is not None
|
||||
else _elevenlabs_realtime_finalize_timeout_seconds()
|
||||
),
|
||||
1.0,
|
||||
)
|
||||
self._model_id = str(model_id or _elevenlabs_realtime_model_id()).strip() or "scribe_v2_realtime"
|
||||
self._audio_format = str(audio_format or _elevenlabs_realtime_audio_format()).strip().lower() or "pcm_16000"
|
||||
self._sample_rate_hz = _elevenlabs_realtime_sample_rate_hz(self._audio_format)
|
||||
self._commit_strategy = str(commit_strategy or _elevenlabs_realtime_commit_strategy()).strip().lower()
|
||||
if self._commit_strategy not in {"manual", "vad"}:
|
||||
self._commit_strategy = "manual"
|
||||
self._websocket_factory = websocket_factory
|
||||
self._streams: dict[str, _ElevenLabsRealtimeStreamState] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _websocket_url(self, *, language: str) -> str:
|
||||
base = self._api_base
|
||||
if base.startswith("https://"):
|
||||
websocket_base = "wss://" + base[len("https://") :]
|
||||
elif base.startswith("http://"):
|
||||
websocket_base = "ws://" + base[len("http://") :]
|
||||
else:
|
||||
websocket_base = base
|
||||
params = {
|
||||
"model_id": self._model_id,
|
||||
"audio_format": self._audio_format,
|
||||
"language_code": language,
|
||||
"commit_strategy": self._commit_strategy,
|
||||
"include_timestamps": "false",
|
||||
"include_language_detection": "false",
|
||||
}
|
||||
return f"{websocket_base}/v1/speech-to-text/realtime?{urllib.parse.urlencode(params)}"
|
||||
|
||||
def _create_websocket(self, url: str) -> object:
|
||||
if not self._api_key:
|
||||
raise StreamingASRUnavailable(
|
||||
"AI_VOICE_ASR_ELEVENLABS_API_KEY or AI_VOICE_TTS_ELEVENLABS_API_KEY is required for ElevenLabs realtime ASR"
|
||||
)
|
||||
factory = self._websocket_factory
|
||||
if factory is None:
|
||||
try:
|
||||
import websocket # type: ignore[import-not-found]
|
||||
except ImportError as exc:
|
||||
raise StreamingASRUnavailable("websocket-client is required for ElevenLabs realtime ASR") from exc
|
||||
factory = websocket.create_connection
|
||||
try:
|
||||
return factory(
|
||||
url,
|
||||
header=[f"xi-api-key: {self._api_key}"],
|
||||
timeout=self._timeout_seconds,
|
||||
)
|
||||
except BaseException as exc:
|
||||
raise StreamingASRUnavailable(str(exc)[:500] or "ElevenLabs realtime ASR is unavailable") from exc
|
||||
|
||||
@staticmethod
|
||||
def _message_type(payload: object) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
return str(payload.get("message_type") or payload.get("type") or "").strip()
|
||||
|
||||
@staticmethod
|
||||
def _payload_text(payload: object) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return ""
|
||||
return str(payload.get("text") or payload.get("transcript") or "").strip()
|
||||
|
||||
@staticmethod
|
||||
def _payload_language(payload: object, fallback: str) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return fallback
|
||||
return str(payload.get("language_code") or payload.get("language") or fallback).strip() or fallback
|
||||
|
||||
@staticmethod
|
||||
def _payload_error_message(payload: object) -> str:
|
||||
if not isinstance(payload, dict):
|
||||
return "ElevenLabs realtime ASR error"
|
||||
for key in ("message", "error", "reason", "detail"):
|
||||
value = payload.get(key)
|
||||
if value:
|
||||
return str(value)[:500]
|
||||
return json.dumps(payload, ensure_ascii=True)[:500]
|
||||
|
||||
def _send_json(self, state: _ElevenLabsRealtimeStreamState, payload: dict[str, object]) -> None:
|
||||
with state.lock:
|
||||
if state.error is not None:
|
||||
raise StreamingASRUnavailable(str(state.error)[:500])
|
||||
if state.close_requested:
|
||||
raise StreamingASRUnavailable("ElevenLabs realtime ASR stream is closed")
|
||||
try:
|
||||
with state.send_lock:
|
||||
state.websocket.send(json.dumps(payload, separators=(",", ":"))) # type: ignore[attr-defined]
|
||||
except BaseException as exc:
|
||||
with state.lock:
|
||||
state.error = exc
|
||||
state.final_event.set()
|
||||
raise StreamingASRUnavailable(str(exc)[:500] or "ElevenLabs realtime ASR send failed") from exc
|
||||
|
||||
def _drain_updates(self, state: _ElevenLabsRealtimeStreamState) -> None:
|
||||
while True:
|
||||
try:
|
||||
update = state.updates_queue.get_nowait()
|
||||
except queue.Empty:
|
||||
return
|
||||
with state.lock:
|
||||
state.latest_partial = update
|
||||
if update.is_final:
|
||||
state.final_transcription = ASRTranscription(
|
||||
text=update.text,
|
||||
language=update.language or state.language,
|
||||
confidence=update.confidence,
|
||||
)
|
||||
state.final_event.set()
|
||||
|
||||
def _run_reader(self, state: _ElevenLabsRealtimeStreamState) -> None:
|
||||
try:
|
||||
while True:
|
||||
with state.lock:
|
||||
if state.close_requested:
|
||||
return
|
||||
raw_message = state.websocket.recv() # type: ignore[attr-defined]
|
||||
if raw_message is None:
|
||||
continue
|
||||
if isinstance(raw_message, bytes):
|
||||
raw_message = raw_message.decode("utf-8", errors="replace")
|
||||
try:
|
||||
payload = json.loads(str(raw_message))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
message_type = self._message_type(payload)
|
||||
if message_type == "partial_transcript":
|
||||
text = self._payload_text(payload)
|
||||
if text:
|
||||
state.updates_queue.put(
|
||||
StreamingASRPartial(
|
||||
text=text,
|
||||
language=self._payload_language(payload, state.language),
|
||||
confidence=None,
|
||||
is_final=False,
|
||||
is_stable=False,
|
||||
)
|
||||
)
|
||||
continue
|
||||
if message_type in {"committed_transcript", "committed_transcript_with_timestamps"}:
|
||||
text = self._payload_text(payload)
|
||||
language = self._payload_language(payload, state.language)
|
||||
if text:
|
||||
state.updates_queue.put(
|
||||
StreamingASRPartial(
|
||||
text=text,
|
||||
language=language,
|
||||
confidence=None,
|
||||
is_final=True,
|
||||
is_stable=True,
|
||||
)
|
||||
)
|
||||
else:
|
||||
with state.lock:
|
||||
state.final_event.set()
|
||||
continue
|
||||
if message_type.startswith("scribe") and "error" in message_type.lower():
|
||||
raise StreamingASRUnavailable(self._payload_error_message(payload))
|
||||
except BaseException as exc:
|
||||
with state.lock:
|
||||
if not state.close_requested:
|
||||
state.error = exc
|
||||
state.final_event.set()
|
||||
|
||||
def _state(self, stream_id: str) -> _ElevenLabsRealtimeStreamState:
|
||||
with self._lock:
|
||||
state = self._streams.get(stream_id)
|
||||
if state is None:
|
||||
raise StreamingASRUnavailable("ElevenLabs realtime ASR stream is not active")
|
||||
return state
|
||||
|
||||
def open_stream(self, session_id: str, *, language_hint: str | None = None) -> str:
|
||||
stream_id = f"elrt_{uuid.uuid4().hex}"
|
||||
language = _normalize_elevenlabs_realtime_language(language_hint)
|
||||
websocket = self._create_websocket(self._websocket_url(language=language))
|
||||
state = _ElevenLabsRealtimeStreamState(
|
||||
stream_id=stream_id,
|
||||
session_id=str(session_id or "").strip(),
|
||||
language=language,
|
||||
websocket=websocket,
|
||||
updates_queue=queue.Queue(),
|
||||
)
|
||||
thread = threading.Thread(
|
||||
target=self._run_reader,
|
||||
args=(state,),
|
||||
name=f"elevenlabs-realtime-asr-{stream_id[:16]}",
|
||||
daemon=True,
|
||||
)
|
||||
state.thread = thread
|
||||
with self._lock:
|
||||
self._streams[stream_id] = state
|
||||
thread.start()
|
||||
return stream_id
|
||||
|
||||
def push_pcm(self, stream_id: str, pcm_8k_chunk: bytes) -> None:
|
||||
if not pcm_8k_chunk:
|
||||
return
|
||||
state = self._state(stream_id)
|
||||
pcm_bytes = pcm_8k_chunk
|
||||
if self._sample_rate_hz != 8000:
|
||||
pcm_bytes = resample_pcm16le(
|
||||
pcm_8k_chunk,
|
||||
input_rate_hz=8000,
|
||||
output_rate_hz=self._sample_rate_hz,
|
||||
)
|
||||
self._send_json(
|
||||
state,
|
||||
{
|
||||
"message_type": "input_audio_chunk",
|
||||
"audio_base_64": base64.b64encode(pcm_bytes).decode("ascii"),
|
||||
"commit": False,
|
||||
"sample_rate": self._sample_rate_hz,
|
||||
},
|
||||
)
|
||||
|
||||
def poll_partial(self, stream_id: str) -> StreamingASRPartial | None:
|
||||
state = self._state(stream_id)
|
||||
self._drain_updates(state)
|
||||
with state.lock:
|
||||
if state.error is not None and state.latest_partial is None:
|
||||
raise StreamingASRUnavailable(str(state.error)[:500])
|
||||
return state.latest_partial
|
||||
|
||||
def finalize(self, stream_id: str) -> ASRTranscription:
|
||||
state = self._state(stream_id)
|
||||
silence = b"\x00\x00" * int(self._sample_rate_hz * 0.1)
|
||||
self._send_json(
|
||||
state,
|
||||
{
|
||||
"message_type": "input_audio_chunk",
|
||||
"audio_base_64": base64.b64encode(silence).decode("ascii"),
|
||||
"commit": True,
|
||||
"sample_rate": self._sample_rate_hz,
|
||||
},
|
||||
)
|
||||
deadline = time.monotonic() + self._finalize_timeout_seconds
|
||||
while True:
|
||||
self._drain_updates(state)
|
||||
with state.lock:
|
||||
if state.final_transcription is not None:
|
||||
return state.final_transcription
|
||||
latest_partial = state.latest_partial
|
||||
error = state.error
|
||||
event_is_set = state.final_event.is_set()
|
||||
if event_is_set or time.monotonic() >= deadline:
|
||||
if latest_partial is not None:
|
||||
return ASRTranscription(
|
||||
text=latest_partial.text,
|
||||
language=latest_partial.language or state.language,
|
||||
confidence=latest_partial.confidence,
|
||||
)
|
||||
if error is not None:
|
||||
raise StreamingASRUnavailable(str(error)[:500])
|
||||
if time.monotonic() >= deadline:
|
||||
raise StreamingASRUnavailable("ElevenLabs realtime ASR finalize timed out")
|
||||
return ASRTranscription(text="", language=state.language, confidence=None)
|
||||
state.final_event.wait(timeout=0.05)
|
||||
|
||||
def close_stream(self, stream_id: str) -> None:
|
||||
try:
|
||||
state = self._state(stream_id)
|
||||
except StreamingASRUnavailable:
|
||||
return
|
||||
with state.lock:
|
||||
state.close_requested = True
|
||||
state.final_event.set()
|
||||
close = getattr(state.websocket, "close", None)
|
||||
if callable(close):
|
||||
try:
|
||||
close()
|
||||
except BaseException:
|
||||
pass
|
||||
thread = state.thread
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=0.5)
|
||||
with self._lock:
|
||||
self._streams.pop(stream_id, None)
|
||||
|
||||
|
||||
class YandexSpeechKitASRProvider(ASRProvider):
|
||||
name = "yandex"
|
||||
|
||||
@@ -1144,6 +1538,15 @@ def build_asr_provider(name: str) -> ASRProvider:
|
||||
|
||||
def build_streaming_asr_provider(name: str) -> StreamingASRProvider:
|
||||
normalized = str(name or "disabled").strip().lower()
|
||||
if normalized in {
|
||||
"elevenlabs",
|
||||
"elevenlabs_realtime",
|
||||
"elevenlabs-realtime",
|
||||
"scribe_realtime",
|
||||
"scribe-v2-realtime",
|
||||
"scribe_v2_realtime",
|
||||
}:
|
||||
return ElevenLabsRealtimeStreamingASRProvider()
|
||||
if normalized in {"local_sidecar", "local-sidecar", "sidecar"}:
|
||||
return LocalSidecarStreamingASRProvider()
|
||||
if normalized in {"yandex", "yandex_speechkit", "yandex-speechkit", "speechkit", "yandex_grpc"}:
|
||||
|
||||
Reference in New Issue
Block a user