From d4d8555bd925a59a9de7a7c03a42b44238401749 Mon Sep 17 00:00:00 2001 From: Konturai DevOps Date: Fri, 1 May 2026 17:50:41 +0500 Subject: [PATCH] . --- .env.example | 2 +- core/__init__.py | 22 +++++++++++++--- core/session.py | 8 +----- core/vad.py | 2 +- main.py | 9 ++++++- providers/factory.py | 2 +- providers/stt.py | 4 +-- providers/stt_openai.py | 2 +- providers/tts.py | 9 ++++--- transports/audiosocket.py | 53 ++++++++++++++++++++++++++++++++++----- 10 files changed, 87 insertions(+), 26 deletions(-) diff --git a/.env.example b/.env.example index dea51db..0f38614 100644 --- a/.env.example +++ b/.env.example @@ -2,7 +2,7 @@ REALTIME_VOICE_HTTP_HOST=0.0.0.0 REALTIME_VOICE_HTTP_PORT=8000 REALTIME_VOICE_AUDIOSOCKET_HOST=0.0.0.0 REALTIME_VOICE_AUDIOSOCKET_PORT=9092 -REALTIME_VOICE_SAMPLE_RATE_HZ=8000 +REALTIME_VOICE_SAMPLE_RATE_HZ=16000 LLM_PROVIDER=openai diff --git a/core/__init__.py b/core/__init__.py index d08bc33..86fae20 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -1,4 +1,20 @@ -from realtime_voice_service.core.session import CallSession, SessionState -from realtime_voice_service.core.vad import BaseVAD, SileroVADDetector, VADFrameResult - __all__ = ["BaseVAD", "CallSession", "SessionState", "SileroVADDetector", "VADFrameResult"] + + +def __getattr__(name: str): + if name in {"CallSession", "SessionState"}: + from realtime_voice_service.core.session import CallSession, SessionState + + return { + "CallSession": CallSession, + "SessionState": SessionState, + }[name] + if name in {"BaseVAD", "SileroVADDetector", "VADFrameResult"}: + from realtime_voice_service.core.vad import BaseVAD, SileroVADDetector, VADFrameResult + + return { + "BaseVAD": BaseVAD, + "SileroVADDetector": SileroVADDetector, + "VADFrameResult": VADFrameResult, + }[name] + raise AttributeError(name) diff --git a/core/session.py b/core/session.py index df51367..fa3ecb7 100644 --- a/core/session.py +++ b/core/session.py @@ -2088,13 +2088,7 @@ class CallSession: timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S_%f") safe_session_id = re.sub(r"[^A-Za-z0-9_.-]+", "_", self.session_id) filename = os.path.join(dump_dir, f"utterance_{safe_session_id}_{timestamp}.wav") - dump_sample_rate_hz = 8000 - if self.transport.sample_rate_hz != dump_sample_rate_hz: - LOGGER.warning( - "realtime session %s audio dump writing raw bytes with 8000Hz header while transport sample_rate=%s", - self.session_id, - self.transport.sample_rate_hz, - ) + dump_sample_rate_hz = self.transport.sample_rate_hz with wave.open(filename, "wb") as wav_file: wav_file.setnchannels(1) diff --git a/core/vad.py b/core/vad.py index ea2a5a5..0ef0c37 100644 --- a/core/vad.py +++ b/core/vad.py @@ -54,7 +54,7 @@ class SileroVADDetector(BaseVAD): def __init__( self, *, - sample_rate_hz: int = 8000, + sample_rate_hz: int = 16000, threshold: float = 0.5, negative_threshold: float | None = None, speech_end_silence_ms: int = 1600, diff --git a/main.py b/main.py index 0c36686..be89ed8 100644 --- a/main.py +++ b/main.py @@ -72,7 +72,9 @@ def _audiosocket_port() -> int: def _sample_rate_hz() -> int: configured = max(_int_env("REALTIME_VOICE_SAMPLE_RATE_HZ", 16000), 1) - return configured if configured in {8000, 16000, 24000} else 16000 + if configured != 16000: + LOGGER.warning("REALTIME_VOICE_SAMPLE_RATE_HZ=%s is not allowed; using 16000 Hz", configured) + return 16000 def _http_host() -> str: @@ -148,6 +150,10 @@ class RealtimeVoiceService: def active_session_count(self) -> int: return len(self._active_sessions) + @property + def sample_rate_hz(self) -> int: + return self._sample_rate_hz + async def start(self) -> None: LOGGER.info("realtime voice service starting") await self._audiosocket_server.start() @@ -285,6 +291,7 @@ async def health() -> dict[str, object]: return { "status": "ok", "service": "realtime-voice-service", + "sample_rate_hz": service.sample_rate_hz, "active_sessions": service.active_session_count, "audiosocket_port": _audiosocket_port(), } diff --git a/providers/factory.py b/providers/factory.py index c968d84..546a491 100644 --- a/providers/factory.py +++ b/providers/factory.py @@ -68,7 +68,7 @@ def _build_single_stt_provider( ) -> BaseSTT: normalized = provider.lower().strip() if normalized in {"openai", "whisper", "openai_whisper"}: - # Whisper accepts an 8 kHz WAV container; keep the exact AudioSocket PCM, no resampling. + # Whisper accepts the configured WAV container; keep the exact AudioSocket PCM, no resampling. return OpenAISTT(input_sample_rate_hz=input_sample_rate_hz) if normalized in {"elevenlabs", "eleven_labs", "scribe"}: return ElevenLabsSTT( diff --git a/providers/stt.py b/providers/stt.py index 3caf690..fc78418 100644 --- a/providers/stt.py +++ b/providers/stt.py @@ -972,8 +972,8 @@ class YandexSpeechKitSTT(BaseSTT): api_base: str | None = None, language: str | None = None, topic: str | None = None, - input_sample_rate_hz: int = 8000, - target_sample_rate_hz: int = 8000, + input_sample_rate_hz: int = 16000, + target_sample_rate_hz: int = 16000, timeout_seconds: float | None = None, ) -> None: self._api_key = str(api_key if api_key is not None else _yandex_api_key()).strip() diff --git a/providers/stt_openai.py b/providers/stt_openai.py index 1ba8064..b7b3bf5 100644 --- a/providers/stt_openai.py +++ b/providers/stt_openai.py @@ -67,7 +67,7 @@ class OpenAISTT(BaseSTT): api_key: str | None = None, model: str | None = None, base_url: str | None = None, - input_sample_rate_hz: int = 8000, + input_sample_rate_hz: int = 16000, prompt: str | None = None, language: str | None = None, timeout_seconds: float | None = None, diff --git a/providers/tts.py b/providers/tts.py index e341b29..ebe75c7 100644 --- a/providers/tts.py +++ b/providers/tts.py @@ -119,11 +119,15 @@ class ElevenLabsTTS(BaseTTS): str(output_format or os.getenv("ELEVENLABS_TTS_OUTPUT_FORMAT", "pcm_16000")).strip().lower() or "pcm_16000" ) + requested_sample_rate_hz = _sample_rate_from_pcm_format(requested_output_format) self._target_sample_rate_hz = int( target_sample_rate_hz if target_sample_rate_hz is not None - else (_sample_rate_from_pcm_format(requested_output_format) or 16000) + else (requested_sample_rate_hz or 16000) ) + if self._target_sample_rate_hz == 8000: + LOGGER.warning("ElevenLabs TTS target sample rate 8000Hz is disabled; using 16000Hz") + self._target_sample_rate_hz = 16000 self._output_format = self._resolve_provider_output_format( requested_output_format=requested_output_format, target_sample_rate_hz=self._target_sample_rate_hz, @@ -381,8 +385,7 @@ class ElevenLabsTTS(BaseTTS): normalized = requested_output_format.strip().lower() or "pcm_16000" if normalized == "pcm_8000": LOGGER.warning( - "ElevenLabs TTS does not provide reliable PCM16 8kHz streaming; requesting pcm_16000 " - "and resampling to %sHz locally", + "ElevenLabs TTS PCM16 8kHz output is disabled; requesting pcm_16000 for %sHz target", target_sample_rate_hz, ) return "pcm_16000" diff --git a/transports/audiosocket.py b/transports/audiosocket.py index 1fe2886..1f0f7c2 100644 --- a/transports/audiosocket.py +++ b/transports/audiosocket.py @@ -17,7 +17,22 @@ LOGGER = logging.getLogger("uvicorn.error") AUDIO_SOCKET_PACKET_HANGUP = 0x00 AUDIO_SOCKET_PACKET_UUID = 0x01 AUDIO_SOCKET_PACKET_DTMF = 0x03 -AUDIO_SOCKET_PACKET_PCM16 = 0x10 +_AUDIO_SOCKET_PCM16_PACKET_TYPES_BY_SAMPLE_RATE_HZ: dict[int, int] = { + 8000: 0x10, + 12000: 0x11, + 16000: 0x12, + 24000: 0x13, + 32000: 0x14, + 44100: 0x15, + 48000: 0x16, + 96000: 0x17, + 192000: 0x18, +} +_AUDIO_SOCKET_SAMPLE_RATE_HZ_BY_PCM16_PACKET_TYPE: dict[int, int] = { + packet_type: sample_rate_hz + for sample_rate_hz, packet_type in _AUDIO_SOCKET_PCM16_PACKET_TYPES_BY_SAMPLE_RATE_HZ.items() +} +SUPPORTED_AUDIO_SOCKET_SAMPLE_RATES_HZ = frozenset(_AUDIO_SOCKET_PCM16_PACKET_TYPES_BY_SAMPLE_RATE_HZ) def _float_env(name: str, default: float) -> float: @@ -43,8 +58,20 @@ def encode_packet(packet_type: int, payload: bytes = b"") -> bytes: return struct.pack("!BH", packet_type & 0xFF, len(payload)) + payload -def encode_audio_packet(pcm_bytes: bytes) -> bytes: - return encode_packet(AUDIO_SOCKET_PACKET_PCM16, pcm_bytes) +def pcm16_packet_type_for_sample_rate(sample_rate_hz: int) -> int: + try: + return _AUDIO_SOCKET_PCM16_PACKET_TYPES_BY_SAMPLE_RATE_HZ[int(sample_rate_hz)] + except KeyError as exc: + supported = ", ".join(str(rate) for rate in sorted(SUPPORTED_AUDIO_SOCKET_SAMPLE_RATES_HZ)) + raise ValueError(f"unsupported AudioSocket PCM16 sample rate {sample_rate_hz}; supported: {supported}") from exc + + +def sample_rate_for_pcm16_packet_type(packet_type: int) -> int | None: + return _AUDIO_SOCKET_SAMPLE_RATE_HZ_BY_PCM16_PACKET_TYPE.get(packet_type) + + +def encode_audio_packet(pcm_bytes: bytes, *, sample_rate_hz: int = 16000) -> bytes: + return encode_packet(pcm16_packet_type_for_sample_rate(sample_rate_hz), pcm_bytes) async def read_packet( @@ -78,6 +105,7 @@ class AudioSocketTransport(BaseMediaTransport): ) self._reader = reader self._writer = writer + self._pcm_packet_type = pcm16_packet_type_for_sample_rate(self.sample_rate_hz) self._read_timeout_seconds = max(read_timeout_seconds, 1.0) self._closed = False self._rx_audio_packet_count = 0 @@ -166,7 +194,20 @@ class AudioSocketTransport(BaseMediaTransport): self._rx_audio_bytes, ) return None - if packet_type == AUDIO_SOCKET_PACKET_PCM16: + packet_sample_rate_hz = sample_rate_for_pcm16_packet_type(packet_type) + if packet_sample_rate_hz is not None: + if packet_sample_rate_hz != self.sample_rate_hz: + LOGGER.error( + "AudioSocket sample rate mismatch: session=%s expected_sample_rate=%s expected_packet_type=0x%02x " + "received_sample_rate=%s received_packet_type=0x%02x payload_bytes=%s", + self.transport_id, + self.sample_rate_hz, + self._pcm_packet_type, + packet_sample_rate_hz, + packet_type, + len(payload), + ) + return None self._rx_audio_packet_count += 1 self._rx_audio_bytes += len(payload) self._track_rx_audio_level(payload) @@ -186,7 +227,7 @@ class AudioSocketTransport(BaseMediaTransport): continue self._rx_ignored_packet_count += 1 LOGGER.warning( - "AudioSocket unknown packet ignored: session=%s packet_type=%s payload_bytes=%s", + "AudioSocket unknown packet ignored: session=%s packet_type=0x%02x payload_bytes=%s", self.transport_id, packet_type, len(payload), @@ -196,7 +237,7 @@ class AudioSocketTransport(BaseMediaTransport): async def _send_frame(self, frame: bytes) -> None: if self._closed or self._writer.is_closing(): return - self._writer.write(encode_audio_packet(frame)) + self._writer.write(encode_audio_packet(frame, sample_rate_hz=self.sample_rate_hz)) await self._writer.drain() async def close(self) -> None: