Merge pull request 'Play a filler ack during slow voice decisions and cache KB search rows' (#2) from voice-latency-llm-tts-improvements into main

Reviewed-on: #2

services/ai_orchestrator_service/app.py

@@ -8,11 +8,12 @@ import math
 import os
 import re
 import time
+from threading import Lock
 from typing import Any
 
 import httpx
 from fastapi import Depends, FastAPI, HTTPException, Query
-from sqlalchemy import select
+from sqlalchemy import func, select
 
 from services.shared.core import Role, new_id, utc_now_iso
 from services.shared.db import get_session
@@ -1909,13 +1910,61 @@ def _article_snippet(article: KBArticleRow, limit: int = 240) -> str:
     return f"{raw[: limit - 3]}..."
 
 
-def _kb_search(session, text: str, *, language: str | None = None) -> list[KBArticleRow]:
-    if not str(text or "").strip():
-        return []
+_KB_ROWS_CACHE: dict[str, tuple[tuple[int, int, str, int], list[KBArticleRow]]] = {}
+_KB_ROWS_CACHE_LOCK = Lock()
+
+
+def _kb_rows_fingerprint(session, language: str | None) -> tuple[int, int, str, int]:
+    # A cheap aggregate (row content lengths, computed server-side - no
+    # title/body/tags_json actually transferred) that changes on any
+    # insert, update, or delete, so the cache below can never serve stale
+    # results. count/max_id/max_updated_at alone aren't sufficient: ids can
+    # be reused after a row is deleted and updated_at has 1-second
+    # resolution, so two different row sets can otherwise share a
+    # fingerprint if they happen to be written within the same second.
+    content_length = func.length(KBArticleRow.title) + func.length(KBArticleRow.body) + func.length(
+        KBArticleRow.tags_json
+    )
+    stmt = select(
+        func.count(KBArticleRow.id),
+        func.max(KBArticleRow.id),
+        func.max(KBArticleRow.updated_at),
+        func.sum(content_length),
+    )
+    if language is not None:
+        stmt = stmt.where(KBArticleRow.language == normalize_kb_language(language))
+    count, max_id, max_updated_at, total_length = session.execute(stmt).one()
+    return (int(count or 0), int(max_id or 0), str(max_updated_at or ""), int(total_length or 0))
+
+
+def _load_kb_rows_cached(session, language: str | None) -> list[KBArticleRow]:
+    # Every voice/chat turn re-runs KB search, and this table rarely changes
+    # mid-call, so avoid re-fetching + re-transferring the whole table from
+    # the DB (title/body/tags_json for every article) when nothing changed.
+    cache_key = normalize_kb_language(language) if language is not None else "__all__"
+    fingerprint = _kb_rows_fingerprint(session, language)
+
+    with _KB_ROWS_CACHE_LOCK:
+        cached = _KB_ROWS_CACHE.get(cache_key)
+    if cached is not None and cached[0] == fingerprint:
+        return cached[1]
+
     stmt = select(KBArticleRow).order_by(KBArticleRow.id.desc())
     if language is not None:
         stmt = stmt.where(KBArticleRow.language == normalize_kb_language(language))
-    rows = session.execute(stmt).scalars().all()
+    rows = list(session.execute(stmt).scalars().all())
+    for row in rows:
+        session.expunge(row)
+
+    with _KB_ROWS_CACHE_LOCK:
+        _KB_ROWS_CACHE[cache_key] = (fingerprint, rows)
+    return rows
+
+
+def _kb_search(session, text: str, *, language: str | None = None) -> list[KBArticleRow]:
+    if not str(text or "").strip():
+        return []
+    rows = _load_kb_rows_cached(session, language)
     return search_kb_rows(rows, text, limit=_ai_max_kb_results())
 
 

services/ai_voice_runtime_service/media_runtime.py

@@ -193,6 +193,7 @@ class AudioSocketMediaRuntime:
         self._partial_asr_min_ms = 320
         self._immediate_ack_min_ms = 700
         self._v2_ack_post_gap_seconds = 0.10
+        self._v1_ack_wait_seconds = 0.6
         self._partial_poll_interval_seconds = 0.20
         self._streaming_asr_partial_poll_enabled = (
             str(os.getenv("AI_VOICE_V2_STREAMING_ASR_PARTIAL_POLL_ENABLED", "0")).strip().lower()
@@ -1619,7 +1620,18 @@ class AudioSocketMediaRuntime:
                     early_plan_decision = await self._take_early_plan_decision(actor, timeout_seconds=0.12)
                     decision = early_plan_decision if early_plan_decision is not None else await decision_task
         else:
-            decision = await decision_task
+            try:
+                decision = await asyncio.wait_for(asyncio.shield(decision_task), timeout=self._v1_ack_wait_seconds)
+            except asyncio.TimeoutError:
+                if not actor.early_ack_started:
+                    await self._emit_early_ack(
+                        actor,
+                        language=transcription.language or actor.registration.language,
+                        metadata=metadata,
+                        ack_source="v1_decision_timeout",
+                        ack_kind="unknown",
+                    )
+                decision = await decision_task
         handoff_task: asyncio.Task | None = None
         if decision.needs_handoff:
             await self._set_actor_state(actor, "handoff_requested", decision.handoff_reason)