digiops/call-center
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(voice): make fast ack more audible

Browse Source
This commit is contained in:
Yera All
2026-04-11 02:27:49 +05:00
parent 05dc31855d
commit 9d0b76deca
2 changed files with 204 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/ai_voice_runtime_service/media_runtime.py
+11 -1
View File
@@ -81,6 +81,7 @@ class MediaActor:
partial_asr_task: asyncio.Task | None = None
partial_asr_attempted: bool = False
last_ack_text: str | None = None
last_ack_completed_monotonic: float = 0.0
class AudioSocketMediaRuntime:
@@ -139,7 +140,8 @@ class AudioSocketMediaRuntime:
self._actors: dict[str, MediaActor] = {}
self._v2_ack_wait_seconds = 0.18
self._partial_asr_min_ms = 650
self._immediate_ack_min_ms = 450
self._immediate_ack_min_ms = 280
self._v2_ack_post_gap_seconds = 0.12
@staticmethod
def _normalize_intent_text(text: str) -> str:
@@ -378,6 +380,7 @@ class AudioSocketMediaRuntime:
},
)
await self._speak_text(actor, ack_text, is_greeting=False, style_hints=style_hints)
actor.last_ack_completed_monotonic = time.monotonic()
if not actor.closed:
await self._set_actor_state(actor, "thinking")
@@ -708,6 +711,13 @@ class AudioSocketMediaRuntime:
await self._set_actor_state(actor, "handoff_requested", decision.handoff_reason)
handoff_task = self._start_handoff_request(actor, transcript_text, decision)
if decision.reply_text:
if self._should_use_voice_v2(actor.registration) and actor.early_ack_started:
remaining_gap = self._v2_ack_post_gap_seconds - max(
0.0,
time.monotonic() - actor.last_ack_completed_monotonic,
)
if remaining_gap > 0:
await asyncio.sleep(remaining_gap)
if self._should_use_voice_v2(actor.registration):
await self._plan_reply_segment(
actor,
tests/test_ai_voice_media_runtime.py
+193
View File
@@ -846,6 +846,199 @@ def test_media_runtime_voice_v2_emits_generic_ack_before_full_asr_without_partia
assert speak_events[0][1] < timings["full_finished"]
def test_media_runtime_voice_v2_emits_ack_for_short_utterance_after_reduced_threshold():
timings: dict[str, float] = {}
speak_events: list[tuple[str, float]] = []
class _FastASRProvider(ASRProvider):
name = "fast-asr"
def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
assert audio_bytes
timings["full_started"] = time.monotonic()
time.sleep(0.12)
timings["full_finished"] = time.monotonic()
return ASRTranscription(text="hello there", language=language_hint or "ru", confidence=0.9)
runtime = AudioSocketMediaRuntime(
enabled=True,
host="127.0.0.1",
port=0,
frame_ms=20,
idle_timeout_seconds=2.0,
registration_wait_timeout_seconds=0.5,
min_speech_ms=40,
trailing_silence_ms=40,
max_turn_ms=2000,
asr_provider=_FastASRProvider(),
tts_provider=_StubTTSProvider(),
load_registration_by_media_uuid=lambda value: None,
mark_media_connected=lambda session_id, value: None,
mark_media_ended=lambda session_id, reason: None,
touch_media_frame=lambda session_id: None,
set_state=lambda session_id, state, handoff_reason, metadata: None,
get_pending_greeting=lambda session_id: None,
mark_reply_delivered=lambda session_id, text, is_greeting: None,
plan_reply=lambda session_id, text, metadata, kind: None,
process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
language=language or "ru",
intent="clarification",
reply_text="Подскажите подробнее, пожалуйста.",
confidence=0.9,
needs_handoff=False,
handoff_reason=None,
case_action="keep_open",
kb_refs=[],
summary_text="reply ready",
model="stub-voice",
latency_ms=1,
status="active",
),
request_handoff=lambda session_id, customer_request_text, decision: None,
handle_media_error=lambda session_id, message, metadata: None,
)
async def _fake_speak_text(
current_actor,
text: str,
*,
is_greeting: bool,
style_hints: dict[str, object] | None = None,
) -> None:
del current_actor, is_greeting, style_hints
speak_events.append((text, time.monotonic()))
await asyncio.sleep(0)
runtime._speak_text = _fake_speak_text # type: ignore[method-assign]
pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
async def _scenario() -> None:
actor = MediaActor(
registration=MediaRegistration(
voice_session_id="avs_media_runtime_v2_short_ack",
call_id="call_media_runtime_v2_short_ack",
interaction_id="int_media_runtime_v2_short_ack",
ai_session_id="ais_media_runtime_v2_short_ack",
language="ru",
media_uuid=str(uuid.uuid4()),
queue_code="voice_lab_ai",
queue_id="que_voice_lab_ai",
agent_profile="voice_support",
voice_v2_enabled=True,
voice_v2_ack_mode="immediate_short",
voice_v2_streaming_tts=True,
voice_v2_partial_asr=False,
),
reader=asyncio.StreamReader(),
writer=None, # type: ignore[arg-type]
vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
frame_ms=20,
frame_bytes=320,
)
await runtime._process_utterance(actor, pcm_frame * 20, False)
asyncio.run(_scenario())
assert speak_events
assert speak_events[0][0] == runtime._ack_text("ru", "generic")
assert speak_events[0][1] < timings["full_finished"]
def test_media_runtime_voice_v2_inserts_small_gap_between_ack_and_main_reply():
speak_events: list[tuple[str, float]] = []
class _InstantASRProvider(ASRProvider):
name = "instant-asr"
def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
assert audio_bytes
return ASRTranscription(text="hello there", language=language_hint or "ru", confidence=0.9)
runtime = AudioSocketMediaRuntime(
enabled=True,
host="127.0.0.1",
port=0,
frame_ms=20,
idle_timeout_seconds=2.0,
registration_wait_timeout_seconds=0.5,
min_speech_ms=40,
trailing_silence_ms=40,
max_turn_ms=2000,
asr_provider=_InstantASRProvider(),
tts_provider=_StubTTSProvider(),
load_registration_by_media_uuid=lambda value: None,
mark_media_connected=lambda session_id, value: None,
mark_media_ended=lambda session_id, reason: None,
touch_media_frame=lambda session_id: None,
set_state=lambda session_id, state, handoff_reason, metadata: None,
get_pending_greeting=lambda session_id: None,
mark_reply_delivered=lambda session_id, text, is_greeting: None,
plan_reply=lambda session_id, text, metadata, kind: None,
process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
language=language or "ru",
intent="clarification",
reply_text="Подскажите подробнее, пожалуйста.",
confidence=0.9,
needs_handoff=False,
handoff_reason=None,
case_action="keep_open",
kb_refs=[],
summary_text="reply ready",
model="stub-voice",
latency_ms=1,
status="active",
),
request_handoff=lambda session_id, customer_request_text, decision: None,
handle_media_error=lambda session_id, message, metadata: None,
)
async def _fake_speak_text(
current_actor,
text: str,
*,
is_greeting: bool,
style_hints: dict[str, object] | None = None,
) -> None:
del current_actor, is_greeting, style_hints
speak_events.append((text, time.monotonic()))
await asyncio.sleep(0)
runtime._speak_text = _fake_speak_text # type: ignore[method-assign]
pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
async def _scenario() -> None:
actor = MediaActor(
registration=MediaRegistration(
voice_session_id="avs_media_runtime_v2_gap",
call_id="call_media_runtime_v2_gap",
interaction_id="int_media_runtime_v2_gap",
ai_session_id="ais_media_runtime_v2_gap",
language="ru",
media_uuid=str(uuid.uuid4()),
queue_code="voice_lab_ai",
queue_id="que_voice_lab_ai",
agent_profile="voice_support",
voice_v2_enabled=True,
voice_v2_ack_mode="immediate_short",
voice_v2_streaming_tts=True,
voice_v2_partial_asr=False,
),
reader=asyncio.StreamReader(),
writer=None, # type: ignore[arg-type]
vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
frame_ms=20,
frame_bytes=320,
)
await runtime._process_utterance(actor, pcm_frame * 20, False)
asyncio.run(_scenario())
assert len(speak_events) == 2
assert speak_events[0][0] == runtime._ack_text("ru", "generic")
assert speak_events[1][0] == "Подскажите подробнее, пожалуйста."
assert (speak_events[1][1] - speak_events[0][1]) >= (runtime._v2_ack_post_gap_seconds - 0.02)
def test_media_runtime_voice_v2_emotive_ack_uses_ru_variants_and_style_hints_only_for_ack():
synth_calls: list[tuple[str, dict[str, object] | None]] = []