feat(voice): start replies from stable streaming partials

2026-04-19 00:13:58 +05:00
parent 770ba4e925
commit 9cd553bf92
8 changed files with 748 additions and 53 deletions
@@ -1451,6 +1451,37 @@ def test_voice_v2_streaming_duplex_early_plan_returns_fast_safe_reply_without_ll
    assert "оператор" in decision["reply_text"].lower()


+def test_voice_v2_streaming_duplex_early_plan_returns_domain_followup_without_llm(monkeypatch):
+    monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_streaming_duplex")
+
+    def _unexpected_llm(messages):
+        raise AssertionError(f"LLM should not be called for early plan: {messages!r}")
+
+    monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm)
+
+    decision = voice_module._voice_decision(
+        language="ru",
+        customer=None,
+        interaction=SimpleNamespace(interaction_id="int_voice_early_schedule", status="new", queue_id="que_voice", subject="unknown"),
+        transcript_text="РњРЅРµ РЅР°РґРѕ СѓР·РЅР°С‚СЊ РіСЂР°С„РёРє СЂР°Р±РѕС‚С‹",
+        transcript_window=[],
+        kb_results=[],
+        disclosure_required=False,
+        request_metadata={
+            "voice_v2_enabled": True,
+            "reply_phase": "early_plan",
+            "response_plan_id": "rsp_early_schedule",
+            "early_intent": "schedule",
+        },
+    )
+
+    assert decision["model"] == "voice_early_plan_domain"
+    assert decision["reply_text"]
+    assert decision["needs_handoff"] is False
+    assert decision["metadata"]["reply_phase"] == "early_plan"
+    assert decision["metadata"]["early_intent"] == "schedule"
+
+
 def test_voice_decision_hearing_check_keeps_active_topic_without_llm(monkeypatch):
    def _unexpected_llm(messages):
        raise AssertionError(f"LLM should not be called for hearing check: {messages!r}")
@@ -399,6 +399,65 @@ def test_elevenlabs_realtime_streaming_provider_returns_partial_and_final():
    assert websocket.sent_payloads[1]["commit"] is True


+def test_elevenlabs_realtime_streaming_provider_finalizes_from_stable_partial_on_timeout():
+    class _FakeRealtimeWebSocket:
+        def __init__(self) -> None:
+            self.sent_payloads: list[dict] = []
+            self.incoming: queue.Queue[str | None] = queue.Queue()
+
+        def send(self, raw_payload: str) -> None:
+            payload = json.loads(raw_payload)
+            self.sent_payloads.append(payload)
+            if not payload.get("commit"):
+                partial = json.dumps(
+                    {
+                        "message_type": "partial_transcript",
+                        "text": "need schedule",
+                        "language_code": "ru",
+                    }
+                )
+                self.incoming.put(partial)
+                self.incoming.put(partial)
+
+        def recv(self) -> str:
+            item = self.incoming.get(timeout=1)
+            if item is None:
+                raise RuntimeError("closed")
+            return item
+
+        def close(self) -> None:
+            self.incoming.put(None)
+
+    websocket = _FakeRealtimeWebSocket()
+
+    provider = asr_module.ElevenLabsRealtimeStreamingASRProvider(
+        api_base="https://api.elevenlabs.example",
+        api_key="asr-key",
+        timeout_seconds=1,
+        finalize_timeout_seconds=0.25,
+        websocket_factory=lambda url, *, header, timeout: websocket,
+    )
+    stream_id = provider.open_stream("session-1", language_hint="ru")
+    provider.push_pcm(stream_id, b"\x01\x00" * 160)
+
+    partial = None
+    for _ in range(20):
+        partial = provider.poll_partial(stream_id)
+        if partial is not None and partial.is_stable:
+            break
+        time.sleep(0.02)
+
+    assert partial is not None
+    assert partial.is_stable is True
+
+    final = provider.finalize(stream_id)
+    provider.close_stream(stream_id)
+
+    assert final.text == "need schedule"
+    assert final.language == "ru"
+    assert websocket.sent_payloads[-1]["commit"] is True
+
+
 def test_yandex_grpc_streaming_provider_returns_partial_and_final():
    calls: list[dict] = []

@@ -1847,6 +1847,263 @@ def test_media_runtime_streaming_partial_poll_does_not_block_turn_close_ack():
    assert "main" in events


+def test_media_runtime_voice_v2_uses_early_plan_before_slow_final_decision():
+    events: list[str | tuple[str, str]] = []
+
+    class _StreamingProvider(StreamingASRProvider):
+        name = "streaming-sidecar"
+        supports_streaming = True
+
+        def finalize(self, stream_id: str) -> ASRTranscription:
+            assert stream_id == "stream-1"
+            return ASRTranscription(text="work schedule", language="ru", confidence=0.9)
+
+        def close_stream(self, stream_id: str) -> None:
+            assert stream_id == "stream-1"
+
+    def _process_turn(session_id, transcript_text, language, barge_in, metadata):
+        del session_id, transcript_text, barge_in
+        if metadata and metadata.get("reply_phase") == "early_plan":
+            events.append("early_plan")
+            return VoiceAITurnDecisionOut(
+                language=language or "ru",
+                intent="schedule",
+                reply_text="early reply",
+                confidence=0.8,
+                needs_handoff=False,
+                handoff_reason=None,
+                case_action="keep_open",
+                kb_refs=[],
+                summary_text="early ready",
+                model="early",
+                latency_ms=1,
+                status="active",
+            )
+        events.append("final_start")
+        time.sleep(0.25)
+        events.append("final_done")
+        return VoiceAITurnDecisionOut(
+            language=language or "ru",
+            intent="schedule",
+            reply_text="final reply",
+            confidence=0.9,
+            needs_handoff=False,
+            handoff_reason=None,
+            case_action="keep_open",
+            kb_refs=[],
+            summary_text="final ready",
+            model="final",
+            latency_ms=1,
+            status="active",
+        )
+
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=2000,
+        asr_provider=_StubASRProvider(),
+        streaming_asr_provider=_StreamingProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: None,
+        process_turn=_process_turn,
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_speak_reply(
+        actor: MediaActor,
+        text: str,
+        *,
+        is_greeting: bool,
+        style_hints: dict[str, object] | None = None,
+        reply_phase: str | None = "main",
+    ) -> None:
+        del actor, is_greeting, style_hints
+        events.append((str(reply_phase), text))
+
+    runtime._speak_reply = _fake_speak_reply  # type: ignore[method-assign]
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_early_plan",
+                call_id="call_media_runtime_early_plan",
+                interaction_id="int_media_runtime_early_plan",
+                ai_session_id="ais_media_runtime_early_plan",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=True,
+                voice_v2_duplex=True,
+                voice_v2_streaming_asr_backend="local_sidecar",
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
+            frame_ms=20,
+            frame_bytes=320,
+        )
+        actor.asr_streaming_enabled = True
+        actor.asr_stream_id = "stream-1"
+        actor.partial_transcript = "work schedule"
+        actor.stable_partial_transcript = "work schedule"
+        actor.partial_intent = "schedule"
+        actor.stable_partial_intent = "schedule"
+        pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
+        await runtime._process_utterance(actor, pcm_frame * 40, False)
+        for _ in range(50):
+            if "final_done" in events:
+                break
+            await asyncio.sleep(0.01)
+
+    asyncio.run(_scenario())
+
+    assert ("main", "early reply") in events
+    assert ("main", "final reply") not in events
+    assert events.index(("main", "early reply")) < events.index("final_done")
+
+
+def test_media_runtime_voice_v2_uses_partial_as_final_when_streaming_finalize_fails():
+    captured: list[tuple[str, dict | None]] = []
+
+    class _CountingASRProvider(ASRProvider):
+        name = "counting-asr"
+
+        def __init__(self) -> None:
+            self.transcribe_count = 0
+
+        def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
+            del audio_bytes
+            self.transcribe_count += 1
+            return ASRTranscription(text="batch text", language=language_hint or "ru", confidence=0.9)
+
+    class _FailingStreamingProvider(StreamingASRProvider):
+        name = "failing-streaming"
+        supports_streaming = True
+
+        def finalize(self, stream_id: str) -> ASRTranscription:
+            assert stream_id == "stream-1"
+            raise StreamingASRUnavailable("finalize timeout")
+
+        def close_stream(self, stream_id: str) -> None:
+            assert stream_id == "stream-1"
+
+    asr_provider = _CountingASRProvider()
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=2000,
+        asr_provider=asr_provider,
+        streaming_asr_provider=_FailingStreamingProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: None,
+        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
+            captured.append((transcript_text, metadata))
+            or VoiceAITurnDecisionOut(
+                language=language or "ru",
+                intent="schedule",
+                reply_text="reply",
+                confidence=0.8,
+                needs_handoff=False,
+                handoff_reason=None,
+                case_action="keep_open",
+                kb_refs=[],
+                summary_text="reply ready",
+                model="stub-voice",
+                latency_ms=1,
+                status="active",
+            )
+        ),
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_speak_reply(
+        actor: MediaActor,
+        text: str,
+        *,
+        is_greeting: bool,
+        style_hints: dict[str, object] | None = None,
+        reply_phase: str | None = "main",
+    ) -> None:
+        del actor, text, is_greeting, style_hints, reply_phase
+
+    runtime._speak_reply = _fake_speak_reply  # type: ignore[method-assign]
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_partial_final",
+                call_id="call_media_runtime_partial_final",
+                interaction_id="int_media_runtime_partial_final",
+                ai_session_id="ais_media_runtime_partial_final",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=True,
+                voice_v2_duplex=True,
+                voice_v2_streaming_asr_backend="local_sidecar",
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
+            frame_ms=20,
+            frame_bytes=320,
+        )
+        actor.asr_streaming_enabled = True
+        actor.asr_stream_id = "stream-1"
+        actor.partial_transcript = "work schedule"
+        actor.stable_partial_transcript = "work schedule"
+        actor.partial_intent = "schedule"
+        actor.stable_partial_intent = "schedule"
+        pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
+        await runtime._process_utterance(actor, pcm_frame * 40, False)
+
+    asyncio.run(_scenario())
+
+    assert asr_provider.transcribe_count == 0
+    final_turns = [item for item in captured if item[1] and item[1].get("reply_phase") == "final"]
+    assert final_turns
+    assert final_turns[0][0] == "work schedule"
+    assert final_turns[0][1]["transcript_source"] == "streaming_partial_after_finalize_failure"
+
+
 def test_media_runtime_merges_thinking_continuation_into_current_utterance():
    runtime = AudioSocketMediaRuntime(
        enabled=True,
@@ -36,7 +36,7 @@ class _StubTTSProvider(TTSProvider):

 def test_media_runtime_voice_v2_uses_streaming_sidecar_for_partial_and_final_asr_stable():
    reply_starts: list[tuple[str, str | None, float]] = []
-    turns: list[str] = []
+    turns: list[tuple[str, str]] = []

    class _BatchASRProvider(ASRProvider):
        name = "batch-asr"
@@ -131,7 +131,7 @@ def test_media_runtime_voice_v2_uses_streaming_sidecar_for_partial_and_final_asr
        plan_reply=lambda session_id, text, metadata, kind: None,
        record_latency=lambda session_id, metric, latency_ms: None,
        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
-            turns.append(transcript_text)
+            turns.append((str((metadata or {}).get("reply_phase") or "final"), transcript_text))
            or VoiceAITurnDecisionOut(
                language=language or "ru",
                intent="schedule",
@@ -191,7 +191,7 @@ def test_media_runtime_voice_v2_uses_streaming_sidecar_for_partial_and_final_asr

    phases = [phase for _, phase, _ in reply_starts]
    finalize_started_at = next(ts for name, ts in streaming_provider.events if name == "finalize")
-    assert turns == ["need work schedule"]
+    assert [text for phase, text in turns if phase == "final"] == ["need work schedule"]
    assert "open" in [name for name, _ in streaming_provider.events]
    assert "close" in [name for name, _ in streaming_provider.events]
    assert phases[:2] == ["ack", "main"]
@@ -48,6 +48,10 @@ def test_voice_start_name_outcome_rejects_garbage_name_inside_mixed_request():
    assert source == "none"


+def test_normalize_name_candidate_drops_trailing_asr_initial():
+    assert voice_policy._normalize_name_candidate("\u0415\u0440\u043d\u0443\u0440 \u0418") == "\u0415\u0440\u043d\u0443\u0440"
+
+
 def test_display_name_looks_trusted_rejects_phone_and_accepts_real_name():
    customer = SimpleNamespace(display_name="+77010000001")
    assert voice_policy._display_name_looks_trusted(customer, "+77010000001", None) is False