diff --git a/deployment/aimaq.env.production b/deployment/aimaq.env.production index e933fed..5bebf23 100644 --- a/deployment/aimaq.env.production +++ b/deployment/aimaq.env.production @@ -13,6 +13,7 @@ AI_API_BASE=https://api.openai.com/v1 AI_API_KEY=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 AI_MODEL=gpt-4o-mini AI_TIMEOUT_SECONDS=30 +AI_VOICE_AI_TIMEOUT_SECONDS=10 AI_WEB_SEARCH_ENABLED=1 AI_WEB_SEARCH_MAX_RESULTS=5 AI_WEB_SEARCH_GL=kz diff --git a/docs/backlog/streaming-tts-choppy-audio.md b/docs/backlog/streaming-tts-choppy-audio.md new file mode 100644 index 0000000..ff97237 --- /dev/null +++ b/docs/backlog/streaming-tts-choppy-audio.md @@ -0,0 +1,47 @@ +# Streaming TTS produces choppy, syllable-by-syllable audio + +**Status:** open, not started. `AI_VOICE_V2_STREAMING_TTS` is `0` (disabled) in +`deployment/aimaq.env.production` until this is fixed — see commit `26d718b` +(enabled) and `b8c922c` (reverted after live testing on the Creator plan). + +## Symptom + +With `AI_VOICE_V2_STREAMING_TTS=1`, live calls sound robotic / read +syllable-by-syllable ("роботизированно, читает по слогам"), reported by +Didar 2026-08-29 after testing on a paid ElevenLabs Creator plan (so it is +not a quota/concurrency artifact — that was ruled out separately the same +week). + +## Root cause + +`services/ai_voice_runtime_service/media_runtime.py`, `_speak_reply` +(~line 1969) only buffers once, at the very start of a reply: + +```python +if prebuffered: + interrupted = await _write_pcm_frames(pcm_8k) # fed straight through + ... +``` + +`_tts_stream_prebuffer_ms` (200ms) absorbs jitter only until the first +`_tts_stream_prebuffer_bytes` have arrived. After that, every network chunk +from ElevenLabs is written to the AudioSocket the moment it arrives, with no +ongoing cushion. `eleven_turbo_v2_5` (and flash models generally) deliver +audio over the wire in uneven bursts, not a smooth constant stream — any +gap between bursts mid-utterance becomes literal dead air in the outbound +audio, which is what reads as "robotic"/"syllable by syllable". + +## Fix direction + +Replace the one-shot prebuffer with a rolling buffer maintained for the +whole utterance: keep ~150-200ms of decoded PCM queued ahead of what's +being paced out via `_FramePacer`, refilling from the producer queue +continuously, instead of switching to pass-through after the first fill. + +## Before re-enabling + +1. Implement the rolling buffer above. +2. Re-test live with `AI_VOICE_V2_STREAMING_TTS=1` on the current Creator + plan and confirm no gaps/choppiness across a few real calls. +3. Only then flip `AI_VOICE_V2_STREAMING_TTS` back to `1` in + `deployment/aimaq.env.production`. diff --git a/services/ai_orchestrator_service/app.py b/services/ai_orchestrator_service/app.py index fb91aa0..bfa14fa 100644 --- a/services/ai_orchestrator_service/app.py +++ b/services/ai_orchestrator_service/app.py @@ -1475,6 +1475,14 @@ def _ai_timeout_seconds() -> float: return max(3.0, _float_env("AI_TIMEOUT_SECONDS", 20.0)) +def _ai_voice_timeout_seconds() -> float: + return max(3.0, _float_env("AI_VOICE_AI_TIMEOUT_SECONDS", _ai_timeout_seconds())) + + +def _ai_decision_max_tokens() -> int: + return max(64, int(_float_env("AI_DECISION_MAX_TOKENS", 500))) + + def _ai_telegram_enabled() -> bool: return _bool_env("AI_TELEGRAM_ENABLED", False) @@ -2339,17 +2347,23 @@ def _openai_prompt( ] -def _request_structured_model_decision(messages: list[dict[str, str]]) -> dict[str, Any]: +def _request_structured_model_decision( + messages: list[dict[str, str]], + *, + timeout_seconds: float | None = None, +) -> dict[str, Any]: if not _ai_api_base() or not _ai_api_key(): raise RuntimeError("AI_API_BASE / AI_API_KEY are required for openai_compatible provider") started = time.perf_counter() payload = { "model": _ai_model(), "temperature": 0.2, + "max_tokens": _ai_decision_max_tokens(), "response_format": {"type": "json_object"}, "messages": messages, } - with httpx.Client(timeout=_ai_timeout_seconds()) as client: + effective_timeout = timeout_seconds if timeout_seconds is not None else _ai_timeout_seconds() + with httpx.Client(timeout=effective_timeout) as client: response = client.post( f"{_ai_api_base()}/chat/completions", json=payload, diff --git a/services/ai_orchestrator_service/voice.py b/services/ai_orchestrator_service/voice.py index 3c2bc22..19daeff 100644 --- a/services/ai_orchestrator_service/voice.py +++ b/services/ai_orchestrator_service/voice.py @@ -1664,7 +1664,8 @@ def _voice_llm_decision( name_value=name_value, name_status=name_status, operator_config=operator_config, - ) + ), + timeout_seconds=app._ai_voice_timeout_seconds(), ) except Exception: return None diff --git a/tests/test_ai_orchestrator_service.py b/tests/test_ai_orchestrator_service.py index 70dbb36..399abdd 100644 --- a/tests/test_ai_orchestrator_service.py +++ b/tests/test_ai_orchestrator_service.py @@ -1307,7 +1307,7 @@ def test_voice_llm_guarded_decision_uses_operator_style_without_ai_or_kb(monkeyp monkeypatch.setattr(ai_module, "_ai_provider", lambda: "openai_compatible") captured: dict[str, object] = {} - def _fake_structured(messages): + def _fake_structured(messages, **kwargs): captured["messages"] = messages return { "language": "ru", @@ -1351,7 +1351,7 @@ def test_voice_v2_fast_conversational_adds_ack_metadata_and_compacts_reply(monke monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_fast_conversational") monkeypatch.setattr(ai_module, "_ai_provider", lambda: "openai_compatible") - def _fake_structured(messages): + def _fake_structured(messages, **kwargs): del messages return { "language": "ru", @@ -1406,7 +1406,7 @@ def test_voice_v2_fast_conversational_adds_ack_metadata_and_compacts_reply(monke def test_voice_v2_off_domain_request_returns_fast_operator_fallback_without_llm(monkeypatch): monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_fast_conversational") - def _unexpected_llm(messages): + def _unexpected_llm(messages, **kwargs): raise AssertionError(f"LLM should not be called for off-domain fallback: {messages!r}") monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm) @@ -1446,7 +1446,7 @@ def test_voice_v2_off_domain_request_returns_fast_operator_fallback_without_llm( def test_voice_v2_streaming_duplex_early_plan_returns_fast_safe_reply_without_llm(monkeypatch): monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_streaming_duplex") - def _unexpected_llm(messages): + def _unexpected_llm(messages, **kwargs): raise AssertionError(f"LLM should not be called for early plan: {messages!r}") monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm) @@ -1477,7 +1477,7 @@ def test_voice_v2_streaming_duplex_early_plan_returns_fast_safe_reply_without_ll def test_voice_v2_streaming_duplex_early_plan_returns_domain_followup_without_llm(monkeypatch): monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_streaming_duplex") - def _unexpected_llm(messages): + def _unexpected_llm(messages, **kwargs): raise AssertionError(f"LLM should not be called for early plan: {messages!r}") monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm) @@ -1506,7 +1506,7 @@ def test_voice_v2_streaming_duplex_early_plan_returns_domain_followup_without_ll def test_voice_decision_hearing_check_keeps_active_topic_without_llm(monkeypatch): - def _unexpected_llm(messages): + def _unexpected_llm(messages, **kwargs): raise AssertionError(f"LLM should not be called for hearing check: {messages!r}") monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm) @@ -1593,7 +1593,7 @@ def test_voice_postprocess_reply_reuses_active_topic_after_frustration_turn(): def test_turn_voice_session_early_plan_does_not_persist_partial_turns(monkeypatch): monkeypatch.setenv("AI_VOICE_POLICY_MODE", "v2_streaming_duplex") - def _unexpected_llm(messages): + def _unexpected_llm(messages, **kwargs): raise AssertionError(f"LLM should not be called for early plan: {messages!r}") monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm) @@ -1710,7 +1710,7 @@ def test_voice_postprocess_reply_uses_summary_context_when_raw_window_lost_topic def test_voice_decision_uses_summary_city_slot_instead_of_asking_city_again(monkeypatch): - def _unexpected_llm(messages): + def _unexpected_llm(messages, **kwargs): raise AssertionError(f"LLM should not be called when summary slot prompt is enough: {messages!r}") monkeypatch.setattr(ai_module, "_request_structured_model_decision", _unexpected_llm)