feat(voice): add phase 1.1 partial asr fast ack

2026-04-11 01:24:21 +05:00
parent ba0e1143ca
commit c39fad7e2d
12 changed files with 852 additions and 60 deletions
@@ -1300,6 +1300,62 @@ def test_voice_llm_guarded_decision_uses_operator_style_without_ai_or_kb(monkeyp
    assert "Do not say or imply that you are an AI" in system_prompt


+def test_voice_v2_fast_conversational_adds_ack_metadata_and_compacts_reply(monkeypatch):
+    monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_fast_conversational")
+    monkeypatch.setattr(ai_module, "_ai_provider", lambda: "openai_compatible")
+
+    def _fake_structured(messages):
+        del messages
+        return {
+            "language": "ru",
+            "intent": "kb_answer",
+            "reply_text": (
+                "Сейчас сориентирую по графику работы филиала в Алматы. "
+                "Он работает с понедельника по пятницу с 9:00 до 18:00. "
+                "Если нужно, подскажу и по субботе."
+            ),
+            "confidence": 0.91,
+            "needs_handoff": False,
+            "handoff_reason": None,
+            "case_action": "keep_open",
+            "kb_refs": ["kba_voice_v2"],
+            "_model": "gpt-test",
+            "_latency_ms": 35,
+            "_finish_reason": "stop",
+        }
+
+    monkeypatch.setattr(ai_module, "_request_structured_model_decision", _fake_structured)
+
+    decision = voice_module._voice_decision(
+        language="ru",
+        customer=SimpleNamespace(customer_id="cus_voice_v2", display_name="Ернор"),
+        interaction=SimpleNamespace(interaction_id="int_voice_v2", status="new", queue_id="que_voice", subject="hours"),
+        transcript_text="Хочу узнать график работы филиала в Алматы",
+        transcript_window=[
+            SimpleNamespace(
+                speaker="caller",
+                text="Хочу узнать график работы филиала в Алматы",
+                sequence_no=1,
+                source_type="voice_asr",
+                barge_in_interrupted=False,
+                created_at=utc_now_iso(),
+            )
+        ],
+        kb_results=[SimpleNamespace(article_id="kba_voice_v2", title="График", body="Будни 9:00-18:00")],
+        disclosure_required=False,
+        customer_name_value="Ернор",
+        customer_name_status="name_obtained",
+        request_metadata={"voice_v2_enabled": True, "response_plan_id": "rsp_test"},
+    )
+
+    assert decision["intent"] == "kb_answer"
+    assert decision["metadata"]["voice_v2_enabled"] is True
+    assert decision["metadata"]["early_intent"] == "schedule"
+    assert decision["metadata"]["ack_kind"] == "understanding"
+    assert decision["metadata"]["response_plan_id"] == "rsp_test"
+    assert len(decision["reply_text"]) <= 180
+
+
 def test_ai_enqueue_creates_outbound_ai_reply_and_delivery_flow(monkeypatch):
    monkeypatch.setenv("AI_TELEGRAM_ENABLED", "1")
    monkeypatch.setenv("AI_PROVIDER", "stub")
@@ -107,6 +107,7 @@ def test_media_runtime_streams_greeting_and_turn():
        set_state=lambda session_id, state, handoff_reason, metadata: states.append((session_id, state)),
        get_pending_greeting=lambda session_id: "greeting" if session_id == "avs_media_runtime" else None,
        mark_reply_delivered=lambda session_id, text, is_greeting: delivered.append((session_id, text, is_greeting)),
+        plan_reply=lambda session_id, text, metadata, kind: None,
        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
            turns.append((session_id, transcript_text, barge_in))
            or VoiceAITurnDecisionOut(
@@ -201,6 +202,7 @@ def test_media_runtime_waits_for_late_registration():
        set_state=lambda session_id, state, handoff_reason, metadata: None,
        get_pending_greeting=lambda session_id: "greeting" if session_id == "avs_media_runtime_late" else None,
        mark_reply_delivered=lambda session_id, text, is_greeting: delivered.append((session_id, text, is_greeting)),
+        plan_reply=lambda session_id, text, metadata, kind: None,
        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
            language=language or "ru",
            intent="answer",
@@ -303,6 +305,7 @@ def test_media_runtime_sends_keepalive_while_tts_is_slow():
        set_state=lambda session_id, state, handoff_reason, metadata: None,
        get_pending_greeting=lambda session_id: "greeting" if session_id == "avs_media_runtime_keepalive" else None,
        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: None,
        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
            language=language or "ru",
            intent="answer",
@@ -369,6 +372,7 @@ def test_media_runtime_sends_keepalive_before_registration_is_ready():
        set_state=lambda session_id, state, handoff_reason, metadata: None,
        get_pending_greeting=lambda session_id: "greeting" if session_id == "avs_media_runtime_prereg" else None,
        mark_reply_delivered=lambda session_id, text, is_greeting: delivered.append((session_id, text, is_greeting)),
+        plan_reply=lambda session_id, text, metadata, kind: None,
        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
            language=language or "ru",
            intent="answer",
@@ -453,6 +457,7 @@ def test_media_runtime_starts_handoff_before_handoff_tts_finishes():
        set_state=lambda session_id, state, handoff_reason, metadata: None,
        get_pending_greeting=lambda session_id: None,
        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: None,
        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
            language=language or "ru",
            intent="handoff_request",
@@ -512,3 +517,208 @@ def test_media_runtime_starts_handoff_before_handoff_tts_finishes():

    assert handoff_started.is_set()
    assert events.index("handoff") < events.index("speak_end")
+
+
+def test_media_runtime_voice_v2_plays_short_ack_before_main_reply():
+    planned: list[tuple[str, str, str, dict | None]] = []
+    delivered: list[tuple[str, str, bool]] = []
+
+    class _ScheduleASRProvider(ASRProvider):
+        name = "schedule-asr"
+
+        def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
+            del audio_bytes
+            return ASRTranscription(text="Хочу узнать график работы", language=language_hint or "ru", confidence=0.9)
+
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=400,
+        asr_provider=_ScheduleASRProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: delivered.append((session_id, text, is_greeting)),
+        plan_reply=lambda session_id, text, metadata, kind: planned.append((session_id, text, kind, metadata)),
+        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: (
+            time.sleep(0.25)
+            or VoiceAITurnDecisionOut(
+                language=language or "ru",
+                intent="clarification",
+                reply_text="Подскажите, пожалуйста, какой город вас интересует?",
+                confidence=0.9,
+                needs_handoff=False,
+                handoff_reason=None,
+                case_action="keep_open",
+                kb_refs=[],
+                summary_text="reply ready",
+                model="stub-voice",
+                latency_ms=1,
+                status="active",
+                metadata={"early_intent": "schedule", "ack_kind": "understanding"},
+            )
+        ),
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_write_audio_packet(current_actor, pcm_frame: bytes) -> None:
+        del current_actor, pcm_frame
+
+    runtime._write_audio_packet = _fake_write_audio_packet  # type: ignore[method-assign]
+    pcm_frame = (1000).to_bytes(2, "little", signed=True) * 160
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_v2",
+                call_id="call_media_runtime_v2",
+                interaction_id="int_media_runtime_v2",
+                ai_session_id="ais_media_runtime_v2",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=False,
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=400),
+            frame_ms=20,
+            frame_bytes=320,
+        )
+        await runtime._process_utterance(actor, pcm_frame, False)
+
+    asyncio.run(_scenario())
+
+    assert [item[2] for item in planned] == ["ack", "reply"]
+    assert planned[0][1] == "Сейчас сориентирую."
+    assert planned[1][1].startswith("Подскажите, пожалуйста")
+    assert delivered == [
+        ("avs_media_runtime_v2", "Сейчас сориентирую.", False),
+        ("avs_media_runtime_v2", "Подскажите, пожалуйста, какой город вас интересует?", False),
+    ]
+
+
+def test_media_runtime_voice_v2_uses_partial_asr_to_start_ack_before_full_asr():
+    timings: dict[str, float] = {}
+    speak_events: list[tuple[str, float]] = []
+
+    class _PartialAwareASRProvider(ASRProvider):
+        name = "partial-aware-asr"
+
+        def transcribe_partial(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
+            assert audio_bytes
+            timings["partial_ready"] = time.monotonic()
+            return ASRTranscription(text="work schedule", language=language_hint or "ru", confidence=0.8)
+
+        def transcribe(self, audio_bytes: bytes, *, language_hint: str | None = None) -> ASRTranscription:
+            assert audio_bytes
+            timings["full_started"] = time.monotonic()
+            time.sleep(0.35)
+            timings["full_finished"] = time.monotonic()
+            return ASRTranscription(text="work schedule", language=language_hint or "ru", confidence=0.9)
+
+    runtime = AudioSocketMediaRuntime(
+        enabled=True,
+        host="127.0.0.1",
+        port=0,
+        frame_ms=20,
+        idle_timeout_seconds=2.0,
+        registration_wait_timeout_seconds=0.5,
+        min_speech_ms=40,
+        trailing_silence_ms=40,
+        max_turn_ms=2000,
+        asr_provider=_PartialAwareASRProvider(),
+        tts_provider=_StubTTSProvider(),
+        load_registration_by_media_uuid=lambda value: None,
+        mark_media_connected=lambda session_id, value: None,
+        mark_media_ended=lambda session_id, reason: None,
+        touch_media_frame=lambda session_id: None,
+        set_state=lambda session_id, state, handoff_reason, metadata: None,
+        get_pending_greeting=lambda session_id: None,
+        mark_reply_delivered=lambda session_id, text, is_greeting: None,
+        plan_reply=lambda session_id, text, metadata, kind: None,
+        process_turn=lambda session_id, transcript_text, language, barge_in, metadata: VoiceAITurnDecisionOut(
+            language=language or "ru",
+            intent="clarification",
+            reply_text="РџРѕРґСЃРєР°Р¶РёС‚Рµ, РєР°РєРѕР№ РіРѕСЂРѕРґ РІР°СЃ РёРЅС‚РµСЂРµСЃСѓРµС‚?",
+            confidence=0.9,
+            needs_handoff=False,
+            handoff_reason=None,
+            case_action="keep_open",
+            kb_refs=[],
+            summary_text="reply ready",
+            model="stub-voice",
+            latency_ms=1,
+            status="active",
+        ),
+        request_handoff=lambda session_id, customer_request_text, decision: None,
+        handle_media_error=lambda session_id, message, metadata: None,
+    )
+
+    async def _fake_speak_text(current_actor, text: str, *, is_greeting: bool) -> None:
+        del current_actor, is_greeting
+        speak_events.append((text, time.monotonic()))
+        await asyncio.sleep(0)
+
+    runtime._speak_text = _fake_speak_text  # type: ignore[method-assign]
+
+    async def _scenario() -> None:
+        actor = MediaActor(
+            registration=MediaRegistration(
+                voice_session_id="avs_media_runtime_v2_partial",
+                call_id="call_media_runtime_v2_partial",
+                interaction_id="int_media_runtime_v2_partial",
+                ai_session_id="ais_media_runtime_v2_partial",
+                language="ru",
+                media_uuid=str(uuid.uuid4()),
+                queue_code="voice_lab_ai",
+                queue_id="que_voice_lab_ai",
+                agent_profile="voice_support",
+                voice_v2_enabled=True,
+                voice_v2_ack_mode="immediate_short",
+                voice_v2_streaming_tts=True,
+                voice_v2_partial_asr=True,
+            ),
+            reader=asyncio.StreamReader(),
+            writer=None,  # type: ignore[arg-type]
+            vad=EnergyVAD(frame_ms=20, min_speech_ms=40, trailing_silence_ms=40, max_turn_ms=2000),
+            frame_ms=20,
+            frame_bytes=320,
+            state="listening",
+        )
+        speech_frame = (1000).to_bytes(2, "little", signed=True) * 160
+        silence_frame = b"\x00\x00" * 160
+        for _ in range(35):
+            await runtime._handle_pcm(actor, speech_frame)
+        for _ in range(20):
+            if actor.partial_transcript:
+                break
+            await asyncio.sleep(0.01)
+        assert actor.partial_transcript == "work schedule"
+        for _ in range(2):
+            await runtime._handle_pcm(actor, silence_frame)
+        pcm_bytes, barge_in = await asyncio.wait_for(actor.turn_queue.get(), timeout=0.2)
+        await runtime._process_utterance(actor, pcm_bytes, barge_in)
+
+    asyncio.run(_scenario())
+
+    assert timings["partial_ready"] < timings["full_finished"]
+    assert speak_events
+    assert speak_events[0][0] == runtime._ack_text("ru", "understanding")
+    assert speak_events[0][1] < timings["full_finished"]