digiops/call-center
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(voice): decouple streaming asr from media loop

Browse Source
This commit is contained in:
Yera All
2026-04-16 23:11:24 +05:00
parent 03450cc308
commit d331981ac8
5 changed files with 271 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/ai_voice_runtime_service/app.py
+5
View File
@@ -123,6 +123,10 @@ def _turn_max_ms() -> int:
return max(_int_env("AI_VOICE_TURN_MAX_MS", 10000), _vad_frame_ms())
def _vad_rms_threshold() -> int:
return max(_int_env("AI_VOICE_VAD_RMS_THRESHOLD", 250), 1)
def _media_idle_timeout_seconds() -> float:
return max(_float_env("AI_VOICE_MEDIA_IDLE_TIMEOUT_SECONDS", 15.0), 5.0)
@@ -1227,6 +1231,7 @@ _MEDIA_RUNTIME = AudioSocketMediaRuntime(
min_speech_ms=_vad_min_speech_ms(),
trailing_silence_ms=_vad_trailing_silence_ms(),
max_turn_ms=_turn_max_ms(),
vad_rms_threshold=_vad_rms_threshold(),
asr_provider=_ASR_PROVIDER,
streaming_asr_provider=_STREAMING_ASR_PROVIDER,
tts_provider=_TTS_PROVIDER,
services/ai_voice_runtime_service/media_runtime.py
+157 -34
View File
@@ -95,9 +95,12 @@ class MediaActor:
utterance_generation: int = 0
finalized_utterance_generation: int = 0
partial_asr_task: asyncio.Task | None = None
streaming_asr_push_task: asyncio.Task | None = None
streaming_asr_push_queue: asyncio.Queue[bytes | None] | None = None
partial_asr_attempted: bool = False
asr_stream_id: str | None = None
asr_streaming_enabled: bool = False
asr_streaming_failed: bool = False
asr_poll_due_monotonic: float = 0.0
streaming_asr_backoff_until_monotonic: float = 0.0
barge_in_speech_ms: int = 0
@@ -126,6 +129,7 @@ class AudioSocketMediaRuntime:
min_speech_ms: int,
trailing_silence_ms: int,
max_turn_ms: int,
vad_rms_threshold: int = 250,
asr_provider: ASRProvider,
streaming_asr_provider: StreamingASRProvider | None = None,
tts_provider: TTSProvider,
@@ -155,6 +159,7 @@ class AudioSocketMediaRuntime:
self._min_speech_ms = max(min_speech_ms, self._frame_ms)
self._trailing_silence_ms = max(trailing_silence_ms, self._frame_ms)
self._max_turn_ms = max(max_turn_ms, self._frame_ms)
self._vad_rms_threshold = max(int(vad_rms_threshold), 1)
self._asr_provider = asr_provider
self._streaming_asr_provider = streaming_asr_provider or StreamingASRProvider()
self._tts_provider = tts_provider
@@ -184,6 +189,9 @@ class AudioSocketMediaRuntime:
self._stable_partial_hold_seconds = 0.40
self._barge_in_trigger_ms = 220
self._streaming_asr_reopen_backoff_seconds = 2.0
self._streaming_asr_push_queue_max_frames = 250
self._streaming_asr_push_batch_max_bytes = self._frame_bytes * 8
self._streaming_asr_push_drain_timeout_seconds = 1.5
self._thinking_continuation_grace_seconds = 0.45
self._thinking_continuation_max_bytes = int(1800 * 16)
@@ -377,6 +385,7 @@ class AudioSocketMediaRuntime:
actor.stable_partial_intent = None
actor.response_plan_id = None
actor.partial_asr_attempted = False
actor.asr_streaming_failed = False
actor.asr_poll_due_monotonic = 0.0
actor.speech_started_monotonic = time.monotonic()
actor.speech_ended_monotonic = 0.0
@@ -423,21 +432,134 @@ class AudioSocketMediaRuntime:
return
actor.asr_stream_id = stream_id
actor.asr_streaming_enabled = True
actor.asr_streaming_failed = False
actor.asr_poll_due_monotonic = 0.0
actor.streaming_asr_push_queue = asyncio.Queue(maxsize=self._streaming_asr_push_queue_max_frames)
actor.streaming_asr_push_task = asyncio.create_task(
self._streaming_asr_push_loop(actor, stream_id),
name=f"streaming-asr-push-{actor.registration.voice_session_id}",
)
def _mark_streaming_asr_backoff(self, actor: MediaActor) -> None:
actor.streaming_asr_backoff_until_monotonic = (
time.monotonic() + self._streaming_asr_reopen_backoff_seconds
)
async def _close_streaming_asr(self, actor: MediaActor) -> None:
async def _close_streaming_asr(self, actor: MediaActor, *, drain: bool = True) -> None:
stream_id = actor.asr_stream_id
actor.asr_stream_id = None
actor.asr_streaming_enabled = False
await self._stop_streaming_asr_push_loop(actor, drain=drain)
if not stream_id:
return
await asyncio.to_thread(self._streaming_asr_provider.close_stream, stream_id)
async def _streaming_asr_push_loop(self, actor: MediaActor, stream_id: str) -> None:
queue = actor.streaming_asr_push_queue
if queue is None:
return
while True:
item = await queue.get()
if item is None:
queue.task_done()
return
batch = bytearray(item)
task_done_count = 1
stop_after_batch = False
while len(batch) < self._streaming_asr_push_batch_max_bytes:
try:
extra = queue.get_nowait()
except asyncio.QueueEmpty:
break
if extra is None:
stop_after_batch = True
queue.task_done()
break
batch.extend(extra)
task_done_count += 1
try:
await asyncio.to_thread(
self._streaming_asr_provider.push_pcm,
stream_id,
bytes(batch),
)
with contextlib.suppress(StreamingASRUnavailable):
await self._poll_streaming_partial(actor)
except StreamingASRUnavailable as exc:
actor.asr_streaming_failed = True
actor.asr_streaming_enabled = False
self._mark_streaming_asr_backoff(actor)
logger.warning(
"audiosocket.streaming_asr_push_failed session_id=%s error=%s",
actor.registration.voice_session_id,
str(exc)[:500],
)
return
finally:
for _ in range(task_done_count):
queue.task_done()
if stop_after_batch:
return
async def _stop_streaming_asr_push_loop(self, actor: MediaActor, *, drain: bool) -> None:
task = actor.streaming_asr_push_task
queue = actor.streaming_asr_push_queue
actor.streaming_asr_push_task = None
actor.streaming_asr_push_queue = None
if task is None:
return
if task.done():
with contextlib.suppress(asyncio.CancelledError, Exception):
await task
return
if queue is not None:
if drain:
try:
await asyncio.wait_for(
queue.join(),
timeout=self._streaming_asr_push_drain_timeout_seconds,
)
except asyncio.TimeoutError:
actor.asr_streaming_failed = True
actor.asr_streaming_enabled = False
self._mark_streaming_asr_backoff(actor)
logger.warning(
"audiosocket.streaming_asr_push_drain_timeout session_id=%s queued_frames=%s",
actor.registration.voice_session_id,
queue.qsize(),
)
sentinel_enqueued = False
with contextlib.suppress(asyncio.QueueFull):
queue.put_nowait(None)
sentinel_enqueued = True
if not sentinel_enqueued:
task.cancel()
try:
await asyncio.wait_for(task, timeout=0.5)
except asyncio.TimeoutError:
task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):
await task
def _queue_streaming_asr_pcm(self, actor: MediaActor, pcm_frame: bytes) -> None:
queue = actor.streaming_asr_push_queue
if queue is None or actor.asr_streaming_failed:
return
try:
queue.put_nowait(pcm_frame)
except asyncio.QueueFull:
actor.asr_streaming_failed = True
actor.asr_streaming_enabled = False
self._mark_streaming_asr_backoff(actor)
logger.warning(
"audiosocket.streaming_asr_push_queue_full session_id=%s max_frames=%s",
actor.registration.voice_session_id,
self._streaming_asr_push_queue_max_frames,
)
asyncio.create_task(self._close_streaming_asr(actor, drain=False))
def _buffer_thinking_continuation(self, actor: MediaActor, pcm_frame: bytes, *, is_speech: bool, now: float) -> None:
if not is_speech and not actor.thinking_continuation_pcm:
return
@@ -500,6 +622,9 @@ class AudioSocketMediaRuntime:
if not actor.asr_streaming_enabled or not actor.asr_stream_id:
raise StreamingASRUnavailable("Streaming ASR stream is not active")
stream_id = actor.asr_stream_id
await self._stop_streaming_asr_push_loop(actor, drain=True)
if actor.asr_streaming_failed:
raise StreamingASRUnavailable("Streaming ASR push failed before finalize")
try:
return await asyncio.to_thread(self._streaming_asr_provider.finalize, stream_id)
finally:
@@ -891,6 +1016,7 @@ class AudioSocketMediaRuntime:
min_speech_ms=self._min_speech_ms,
trailing_silence_ms=self._trailing_silence_ms,
max_turn_ms=self._max_turn_ms,
rms_threshold=self._vad_rms_threshold,
),
frame_ms=self._frame_ms,
frame_bytes=self._frame_bytes,
@@ -965,7 +1091,7 @@ class AudioSocketMediaRuntime:
if actor.state not in {"listening", "speaking", "thinking"}:
return
is_speech = audioop.rms(pcm_frame, 2) >= 250
is_speech = audioop.rms(pcm_frame, 2) >= self._vad_rms_threshold
if actor.state == "thinking":
self._buffer_thinking_continuation(actor, pcm_frame, is_speech=is_speech, now=now)
return
@@ -988,43 +1114,25 @@ class AudioSocketMediaRuntime:
vad_result = actor.vad.feed(pcm_frame)
if vad_result.speech_started:
self._reset_live_turn_state(actor)
logger.info(
"audiosocket.speech_started session_id=%s",
actor.registration.voice_session_id,
)
await self._ensure_streaming_asr(actor)
stream_input_active = actor.input_active or actor.barge_in_pending
if actor.asr_streaming_enabled and actor.asr_stream_id and stream_input_active:
try:
await asyncio.to_thread(
self._streaming_asr_provider.push_pcm,
actor.asr_stream_id,
pcm_frame,
)
except StreamingASRUnavailable as exc:
logger.warning(
"audiosocket.streaming_asr_push_failed session_id=%s error=%s",
actor.registration.voice_session_id,
str(exc)[:500],
)
self._mark_streaming_asr_backoff(actor)
await self._close_streaming_asr(actor)
else:
try:
await self._poll_streaming_partial(actor)
except StreamingASRUnavailable as exc:
actor.asr_poll_due_monotonic = time.monotonic() + max(
self._partial_poll_interval_seconds,
0.60,
)
logger.warning(
"audiosocket.streaming_asr_partial_poll_failed session_id=%s error=%s",
actor.registration.voice_session_id,
str(exc)[:500],
)
self._mark_streaming_asr_backoff(actor)
await self._close_streaming_asr(actor)
self._queue_streaming_asr_pcm(actor, pcm_frame)
elif actor.registration.voice_v2_partial_asr:
self._maybe_schedule_partial_asr(actor)
if vad_result.utterance_pcm:
actor.input_active = False
actor.speech_ended_monotonic = time.monotonic()
logger.info(
"audiosocket.utterance_finalized session_id=%s duration_ms=%s barge_in=%s",
actor.registration.voice_session_id,
int(len(vad_result.utterance_pcm) / 16),
actor.barge_in_pending,
)
await actor.turn_queue.put((vad_result.utterance_pcm, actor.barge_in_pending))
actor.barge_in_pending = False
@@ -1101,8 +1209,23 @@ class AudioSocketMediaRuntime:
ack_source="immediate_turn_close",
)
if actor.asr_streaming_enabled:
transcription = await self._finalize_streaming_transcription(actor)
if actor.asr_streaming_enabled and not actor.asr_streaming_failed:
try:
transcription = await self._finalize_streaming_transcription(actor)
except StreamingASRUnavailable as exc:
logger.warning(
"audiosocket.streaming_asr_finalize_failed session_id=%s error=%s",
actor.registration.voice_session_id,
str(exc)[:500],
)
self._mark_streaming_asr_backoff(actor)
await self._close_streaming_asr(actor, drain=False)
wav_bytes = pcm16le_to_wav_bytes(pcm_bytes, sample_rate_hz=8000)
transcription = await asyncio.to_thread(
self._asr_provider.transcribe,
wav_bytes,
language_hint=actor.registration.language,
)
else:
wav_bytes = pcm16le_to_wav_bytes(pcm_bytes, sample_rate_hz=8000)
transcription = await asyncio.to_thread(
@@ -1455,7 +1578,7 @@ class AudioSocketMediaRuntime:
with contextlib.suppress(asyncio.CancelledError, Exception):
await actor.partial_asr_task
with contextlib.suppress(Exception):
await self._close_streaming_asr(actor)
await self._close_streaming_asr(actor, drain=False)
if actor.worker_task is not None:
actor.worker_task.cancel()
with contextlib.suppress(asyncio.CancelledError, Exception):