fix: filter RU/KZ stopwords from KB search so filler words can't false-match
deploy / deploy (push) Successful in 30s

search_kb_rows had no relevance floor: any exact-token hit, however
generic, scored above zero and could win as the top/only result. A
caller utterance as thin as a bare "да" (confirming the language) could
exact-match that same common word inside an unrelated FAQ article's
body and get returned as "the" answer, which then got read back
almost verbatim — this is what surfaced live as the AI unprompted
launching into a voucher-activation explanation right after the
customer confirmed Russian, having said nothing else.

tokenize_kb_text now drops a curated set of RU/KZ greetings,
confirmations, pronouns, and particles. A stopword-only query naturally
falls through to the existing "no query tokens -> no results" path
instead of returning a coincidental match; genuine single-content-word
queries (e.g. "ваучер") are unaffected. Applies to every channel that
calls _kb_search (voice, Telegram, WhatsApp), not just voice.
This commit is contained in:
2026-08-31 01:08:11 +05:00
parent 3c5c233071
commit 9bf367abf4
2 changed files with 88 additions and 1 deletions
+33 -1
View File
@@ -19,6 +19,34 @@ _MIN_SOFT_MATCH_LENGTH = 4
_NON_WORD_RE = re.compile(r"[^\w]+", re.UNICODE)
_SPACE_RE = re.compile(r"\s+")
# Common RU/KZ greetings, confirmations, pronouns, and particles that carry no
# topical signal on their own. Without this, a caller utterance as thin as
# "да" or "хорошо" could still exact-token-match some unrelated KB article
# that happens to contain that word in its body, and get returned as the
# top/only search result — read back to the caller as if it were the answer
# to their question. Filtering these keeps _score_row's exact-token match
# meaningful: a match now requires an actual content word.
_STOPWORDS = frozenset(
{
# RU: greetings / confirmations / fillers
"алло", "ага", "да", "неа", "нет", "ой", "ок", "окей", "угу", "ясно",
"ладно", "хорошо", "понял", "поняла", "привет", "здравствуйте",
"добрый", "день", "вечер", "утро", "слышу", "слышно", "спасибо",
"пожалуйста", "извините", "простите", "алло",
# RU: pronouns / conjunctions / particles with no topical content
"я", "ты", "вы", "мы", "он", "она", "они", "это", "то", "и", "а",
"но", "или", "что", "как", "где", "когда", "если", "чтобы", "для",
"из", "по", "на", "в", "с", "у", "о", "же", "ли", "бы", "не", "ну",
"вот", "просто", "есть", "быть", "можно", "нужно", "надо", "уже",
# KZ: greetings / confirmations / fillers
"иә", "ия", "жоқ", "жарайды", "түсінікті", "рахмет", "сәлем",
"сәлеметсіз", "бе", "кешіріңіз",
# KZ: pronouns / conjunctions / particles
"мен", "сен", "сіз", "біз", "ол", "олар", "және", "бірақ", "немесе",
"не", "қалай", "қайда", "қашан", "үшін", "туралы",
}
)
class KBSearchRow(Protocol):
id: int
@@ -38,7 +66,11 @@ def normalize_kb_text(value: str | None) -> str:
def tokenize_kb_text(value: str | None) -> list[str]:
return [token for token in normalize_kb_text(value).split(" ") if len(token) >= _MIN_TOKEN_LENGTH]
return [
token
for token in normalize_kb_text(value).split(" ")
if len(token) >= _MIN_TOKEN_LENGTH and token not in _STOPWORDS
]
def search_kb_rows(