fix(voice): decouple streaming asr from media loop

This commit is contained in:
Yera All
2026-04-16 23:11:24 +05:00
parent 03450cc308
commit d331981ac8
5 changed files with 271 additions and 34 deletions
+107
View File
@@ -1504,6 +1504,10 @@ def test_media_runtime_streaming_timeout_enters_backoff_before_reopen():
speech_frame = (1000).to_bytes(2, "little", signed=True) * 160
await runtime._handle_pcm(actor, speech_frame)
await runtime._handle_pcm(actor, speech_frame)
for _ in range(20):
if not actor.asr_streaming_enabled:
break
await asyncio.sleep(0.01)
assert actor.asr_streaming_enabled is False
assert actor.streaming_asr_backoff_until_monotonic > time.monotonic()
await runtime._ensure_streaming_asr(actor)
@@ -1512,6 +1516,109 @@ def test_media_runtime_streaming_timeout_enters_backoff_before_reopen():
asyncio.run(_scenario())
def test_media_runtime_streaming_sidecar_push_does_not_block_vad_finalization():
class _SlowStreamingProvider(StreamingASRProvider):
name = "slow-streaming"
supports_streaming = True
def __init__(self) -> None:
self.push_count = 0
def open_stream(self, session_id: str, *, language_hint: str | None = None) -> str:
del session_id, language_hint
return "stream-1"
def push_pcm(self, stream_id: str, pcm_8k_chunk: bytes) -> None:
del stream_id, pcm_8k_chunk
self.push_count += 1
time.sleep(0.25)
registration = MediaRegistration(
voice_session_id="avs_media_runtime_nonblocking_asr",
call_id="call_media_runtime_nonblocking_asr",
interaction_id="int_media_runtime_nonblocking_asr",
ai_session_id="ais_media_runtime_nonblocking_asr",
language="ru",
media_uuid=str(uuid.uuid4()),
queue_code="voice_lab_ai",
queue_id="que_voice_lab_ai",
agent_profile="voice_support",
voice_v2_enabled=True,
voice_v2_ack_mode="immediate_short",
voice_v2_streaming_tts=True,
voice_v2_partial_asr=True,
voice_v2_duplex=True,
voice_v2_streaming_asr_backend="local_sidecar",
)
streaming_provider = _SlowStreamingProvider()
runtime = AudioSocketMediaRuntime(
enabled=True,
host="127.0.0.1",
port=0,
frame_ms=20,
idle_timeout_seconds=2.0,
registration_wait_timeout_seconds=0.5,
min_speech_ms=40,
trailing_silence_ms=40,
max_turn_ms=2000,
asr_provider=_StubASRProvider(),
streaming_asr_provider=streaming_provider,
tts_provider=_StubTTSProvider(),
load_registration_by_media_uuid=lambda value: None,
mark_media_connected=lambda session_id, value: None,
mark_media_ended=lambda session_id, reason: None,
touch_media_frame=lambda session_id: None,
set_state=lambda session_id, state, handoff_reason, metadata: None,
get_pending_greeting=lambda session_id: None,
mark_reply_delivered=lambda session_id, text, is_greeting: None,
plan_reply=lambda session_id, text, metadata, kind: None,
process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
language=language or "ru",
intent="clarification",
reply_text="reply",
confidence=0.8,
needs_handoff=False,
handoff_reason=None,
case_action="keep_open",
kb_refs=[],
summary_text="reply ready",
model="stub-voice",
latency_ms=1,
status="active",
),
request_handoff=lambda session_id, customer_request_text, decision: None,
handle_media_error=lambda session_id, message, metadata: None,
)
async def _scenario() -> float:
actor = MediaActor(
registration=registration,
reader=asyncio.StreamReader(),
writer=None, # type: ignore[arg-type]
vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
frame_ms=20,
frame_bytes=320,
state="listening",
)
speech_frame = (1000).to_bytes(2, "little", signed=True) * 160
silence_frame = b"\x00\x00" * 160
started = time.monotonic()
await runtime._handle_pcm(actor, speech_frame)
await runtime._handle_pcm(actor, speech_frame)
await runtime._handle_pcm(actor, silence_frame)
await runtime._handle_pcm(actor, silence_frame)
elapsed = time.monotonic() - started
pcm_bytes, _ = await asyncio.wait_for(actor.turn_queue.get(), timeout=0.1)
assert pcm_bytes
await runtime._close_streaming_asr(actor, drain=False)
return elapsed
elapsed = asyncio.run(_scenario())
assert elapsed < 0.15
assert streaming_provider.push_count >= 1
def test_media_runtime_merges_thinking_continuation_into_current_utterance():
runtime = AudioSocketMediaRuntime(
enabled=True,