fix: filter RU/KZ stopwords from KB search so filler words can't false-match
deploy / deploy (push) Successful in 30s

search_kb_rows had no relevance floor: any exact-token hit, however
generic, scored above zero and could win as the top/only result. A
caller utterance as thin as a bare "да" (confirming the language) could
exact-match that same common word inside an unrelated FAQ article's
body and get returned as "the" answer, which then got read back
almost verbatim — this is what surfaced live as the AI unprompted
launching into a voucher-activation explanation right after the
customer confirmed Russian, having said nothing else.

tokenize_kb_text now drops a curated set of RU/KZ greetings,
confirmations, pronouns, and particles. A stopword-only query naturally
falls through to the existing "no query tokens -> no results" path
instead of returning a coincidental match; genuine single-content-word
queries (e.g. "ваучер") are unaffected. Applies to every channel that
calls _kb_search (voice, Telegram, WhatsApp), not just voice.
This commit is contained in:
2026-08-31 01:08:11 +05:00
parent 3c5c233071
commit 9bf367abf4
2 changed files with 88 additions and 1 deletions
+33 -1
View File
@@ -19,6 +19,34 @@ _MIN_SOFT_MATCH_LENGTH = 4
_NON_WORD_RE = re.compile(r"[^\w]+", re.UNICODE) _NON_WORD_RE = re.compile(r"[^\w]+", re.UNICODE)
_SPACE_RE = re.compile(r"\s+") _SPACE_RE = re.compile(r"\s+")
# Common RU/KZ greetings, confirmations, pronouns, and particles that carry no
# topical signal on their own. Without this, a caller utterance as thin as
# "да" or "хорошо" could still exact-token-match some unrelated KB article
# that happens to contain that word in its body, and get returned as the
# top/only search result — read back to the caller as if it were the answer
# to their question. Filtering these keeps _score_row's exact-token match
# meaningful: a match now requires an actual content word.
_STOPWORDS = frozenset(
{
# RU: greetings / confirmations / fillers
"алло", "ага", "да", "неа", "нет", "ой", "ок", "окей", "угу", "ясно",
"ладно", "хорошо", "понял", "поняла", "привет", "здравствуйте",
"добрый", "день", "вечер", "утро", "слышу", "слышно", "спасибо",
"пожалуйста", "извините", "простите", "алло",
# RU: pronouns / conjunctions / particles with no topical content
"я", "ты", "вы", "мы", "он", "она", "они", "это", "то", "и", "а",
"но", "или", "что", "как", "где", "когда", "если", "чтобы", "для",
"из", "по", "на", "в", "с", "у", "о", "же", "ли", "бы", "не", "ну",
"вот", "просто", "есть", "быть", "можно", "нужно", "надо", "уже",
# KZ: greetings / confirmations / fillers
"иә", "ия", "жоқ", "жарайды", "түсінікті", "рахмет", "сәлем",
"сәлеметсіз", "бе", "кешіріңіз",
# KZ: pronouns / conjunctions / particles
"мен", "сен", "сіз", "біз", "ол", "олар", "және", "бірақ", "немесе",
"не", "қалай", "қайда", "қашан", "үшін", "туралы",
}
)
class KBSearchRow(Protocol): class KBSearchRow(Protocol):
id: int id: int
@@ -38,7 +66,11 @@ def normalize_kb_text(value: str | None) -> str:
def tokenize_kb_text(value: str | None) -> list[str]: def tokenize_kb_text(value: str | None) -> list[str]:
return [token for token in normalize_kb_text(value).split(" ") if len(token) >= _MIN_TOKEN_LENGTH] return [
token
for token in normalize_kb_text(value).split(" ")
if len(token) >= _MIN_TOKEN_LENGTH and token not in _STOPWORDS
]
def search_kb_rows( def search_kb_rows(
+55
View File
@@ -114,6 +114,61 @@ def test_search_kb_rows_prefers_title_and_tags_over_body_only_mentions():
assert results[0].title == rows[0].title assert results[0].title == rows[0].title
def test_search_kb_rows_ignores_stopword_only_query():
# A caller confirming the language ("да") should never surface an
# unrelated FAQ article just because that article's body happens to
# contain the word "да" somewhere in ordinary prose.
rows = [
_row(
row_id=1,
title="Активация ваучера",
body="Да, подтвердите СМС с номера 1414 командой 21*1.",
tags=["ваучер"],
),
]
assert search_kb_rows(rows, "да", limit=5) == []
assert search_kb_rows(rows, "Хорошо", limit=5) == []
def test_search_kb_rows_still_matches_real_content_word_amid_fillers():
rows = [
_row(
row_id=1,
title="Активация ваучера",
body="Подтвердите СМС с номера 1414 командой 21*1.",
tags=["ваучер"],
),
_row(
row_id=2,
title="График работы",
body="Филиалы работают с 9 до 18.",
tags=["график"],
),
]
results = search_kb_rows(rows, "да, у меня вопрос про ваучер", limit=5)
assert results
assert results[0].title == "Активация ваучера"
def test_search_kb_rows_single_real_word_query_still_matches():
rows = [
_row(
row_id=1,
title="Активация ваучера",
body="Подтвердите СМС с номера 1414 командой 21*1.",
tags=["ваучер"],
),
]
results = search_kb_rows(rows, "ваучер", limit=5)
assert results
assert results[0].title == "Активация ваучера"
def test_search_kb_rows_breaks_ties_by_newer_id(): def test_search_kb_rows_breaks_ties_by_newer_id():
older = _row( older = _row(
row_id=10, row_id=10,