feat: update ElevenLabs TTS model ID and add prebuffering for improved audio streaming
deploy / deploy (push) Successful in 31s
deploy / deploy (push) Successful in 31s
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import asyncio
|
||||
import contextlib
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
@@ -2899,3 +2900,91 @@ def test_media_runtime_no_speech_watchdog_reprompts_on_silence_timeout():
|
||||
assert spoken, "watchdog should reprompt after prolonged silence in listening state"
|
||||
|
||||
asyncio.run(_scenario())
|
||||
|
||||
|
||||
def test_media_runtime_streaming_tts_prebuffers_without_dropping_audio():
|
||||
class _ChunkedTTSProvider(TTSProvider):
|
||||
name = "chunked-tts"
|
||||
|
||||
def synthesize(self, text, *, language=None, style_hints=None):
|
||||
raise AssertionError("streaming session should use synthesize_chunks")
|
||||
|
||||
def synthesize_chunks(self, text, *, language=None, style_hints=None):
|
||||
del language, style_hints
|
||||
# 12 small 16kHz chunks (5ms each), each well under the prebuffer
|
||||
# target, so both the accumulation path and the trailing flush
|
||||
# (for whatever is still buffered once the stream ends) get exercised.
|
||||
tone = (500).to_bytes(2, "little", signed=True) * 80
|
||||
for _ in range(12):
|
||||
yield TTSSynthesis(text=text, audio_bytes=tone, sample_rate_hz=16000)
|
||||
|
||||
class _FakeWriter:
|
||||
def __init__(self) -> None:
|
||||
self.packets: list[bytes] = []
|
||||
|
||||
def write(self, data: bytes) -> None:
|
||||
self.packets.append(data)
|
||||
|
||||
async def drain(self) -> None:
|
||||
return None
|
||||
|
||||
registration = MediaRegistration(
|
||||
voice_session_id="avs_media_runtime_prebuffer",
|
||||
call_id="call_media_runtime_prebuffer",
|
||||
interaction_id="int_media_runtime_prebuffer",
|
||||
ai_session_id="ais_media_runtime_prebuffer",
|
||||
language="ru",
|
||||
media_uuid=str(uuid.uuid4()),
|
||||
voice_v2_streaming_tts=True,
|
||||
)
|
||||
runtime = AudioSocketMediaRuntime(
|
||||
enabled=True,
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
frame_ms=20,
|
||||
idle_timeout_seconds=2.0,
|
||||
registration_wait_timeout_seconds=0.5,
|
||||
min_speech_ms=40,
|
||||
trailing_silence_ms=40,
|
||||
max_turn_ms=400,
|
||||
asr_provider=_StubASRProvider(),
|
||||
tts_provider=_ChunkedTTSProvider(),
|
||||
load_registration_by_media_uuid=lambda value: None,
|
||||
mark_media_connected=lambda session_id, value: None,
|
||||
mark_media_ended=lambda session_id, reason: None,
|
||||
touch_media_frame=lambda session_id: None,
|
||||
set_state=lambda session_id, state, handoff_reason, metadata: None,
|
||||
get_pending_greeting=lambda session_id: None,
|
||||
mark_reply_delivered=lambda session_id, text, is_greeting: None,
|
||||
plan_reply=lambda session_id, text, metadata, kind: None,
|
||||
process_turn=lambda session_id, transcript_text, language, barge_in, metadata: None,
|
||||
request_handoff=lambda session_id, customer_request_text, decision: None,
|
||||
handle_media_error=lambda session_id, message, metadata: None,
|
||||
)
|
||||
runtime._tts_stream_prebuffer_ms = 40
|
||||
|
||||
writer = _FakeWriter()
|
||||
|
||||
async def _scenario() -> None:
|
||||
actor = MediaActor(
|
||||
registration=registration,
|
||||
reader=asyncio.StreamReader(),
|
||||
writer=writer, # type: ignore[arg-type]
|
||||
vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=400),
|
||||
frame_ms=20,
|
||||
frame_bytes=320,
|
||||
)
|
||||
await runtime._speak_text(actor, "тест", is_greeting=False, reply_phase="main")
|
||||
|
||||
asyncio.run(_scenario())
|
||||
|
||||
assert len(writer.packets) > 1, "audio should be paced out frame by frame, not as one blob"
|
||||
payload_bytes = 0
|
||||
for packet in writer.packets:
|
||||
packet_type, payload_length = struct.unpack("!BH", packet[:3])
|
||||
assert packet_type == AUDIO_SOCKET_PACKET_PCM16
|
||||
assert len(packet) == 3 + payload_length == 3 + 320, "every frame must be frame_bytes, zero-padded if short"
|
||||
payload_bytes += payload_length
|
||||
# 12 chunks * 160 bytes @16kHz downsample 2:1 -> 960 bytes @8kHz of real
|
||||
# audio; frame padding on flush boundaries can only add silence, never drop it.
|
||||
assert payload_bytes >= 960
|
||||
|
||||
Reference in New Issue
Block a user