fix: filter RU/KZ stopwords from KB search so filler words can't false-match
deploy / deploy (push) Successful in 30s
deploy / deploy (push) Successful in 30s
search_kb_rows had no relevance floor: any exact-token hit, however generic, scored above zero and could win as the top/only result. A caller utterance as thin as a bare "да" (confirming the language) could exact-match that same common word inside an unrelated FAQ article's body and get returned as "the" answer, which then got read back almost verbatim — this is what surfaced live as the AI unprompted launching into a voucher-activation explanation right after the customer confirmed Russian, having said nothing else. tokenize_kb_text now drops a curated set of RU/KZ greetings, confirmations, pronouns, and particles. A stopword-only query naturally falls through to the existing "no query tokens -> no results" path instead of returning a coincidental match; genuine single-content-word queries (e.g. "ваучер") are unaffected. Applies to every channel that calls _kb_search (voice, Telegram, WhatsApp), not just voice.
This commit is contained in:
@@ -19,6 +19,34 @@ _MIN_SOFT_MATCH_LENGTH = 4
|
|||||||
_NON_WORD_RE = re.compile(r"[^\w]+", re.UNICODE)
|
_NON_WORD_RE = re.compile(r"[^\w]+", re.UNICODE)
|
||||||
_SPACE_RE = re.compile(r"\s+")
|
_SPACE_RE = re.compile(r"\s+")
|
||||||
|
|
||||||
|
# Common RU/KZ greetings, confirmations, pronouns, and particles that carry no
|
||||||
|
# topical signal on their own. Without this, a caller utterance as thin as
|
||||||
|
# "да" or "хорошо" could still exact-token-match some unrelated KB article
|
||||||
|
# that happens to contain that word in its body, and get returned as the
|
||||||
|
# top/only search result — read back to the caller as if it were the answer
|
||||||
|
# to their question. Filtering these keeps _score_row's exact-token match
|
||||||
|
# meaningful: a match now requires an actual content word.
|
||||||
|
_STOPWORDS = frozenset(
|
||||||
|
{
|
||||||
|
# RU: greetings / confirmations / fillers
|
||||||
|
"алло", "ага", "да", "неа", "нет", "ой", "ок", "окей", "угу", "ясно",
|
||||||
|
"ладно", "хорошо", "понял", "поняла", "привет", "здравствуйте",
|
||||||
|
"добрый", "день", "вечер", "утро", "слышу", "слышно", "спасибо",
|
||||||
|
"пожалуйста", "извините", "простите", "алло",
|
||||||
|
# RU: pronouns / conjunctions / particles with no topical content
|
||||||
|
"я", "ты", "вы", "мы", "он", "она", "они", "это", "то", "и", "а",
|
||||||
|
"но", "или", "что", "как", "где", "когда", "если", "чтобы", "для",
|
||||||
|
"из", "по", "на", "в", "с", "у", "о", "же", "ли", "бы", "не", "ну",
|
||||||
|
"вот", "просто", "есть", "быть", "можно", "нужно", "надо", "уже",
|
||||||
|
# KZ: greetings / confirmations / fillers
|
||||||
|
"иә", "ия", "жоқ", "жарайды", "түсінікті", "рахмет", "сәлем",
|
||||||
|
"сәлеметсіз", "бе", "кешіріңіз",
|
||||||
|
# KZ: pronouns / conjunctions / particles
|
||||||
|
"мен", "сен", "сіз", "біз", "ол", "олар", "және", "бірақ", "немесе",
|
||||||
|
"не", "қалай", "қайда", "қашан", "үшін", "туралы",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class KBSearchRow(Protocol):
|
class KBSearchRow(Protocol):
|
||||||
id: int
|
id: int
|
||||||
@@ -38,7 +66,11 @@ def normalize_kb_text(value: str | None) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def tokenize_kb_text(value: str | None) -> list[str]:
|
def tokenize_kb_text(value: str | None) -> list[str]:
|
||||||
return [token for token in normalize_kb_text(value).split(" ") if len(token) >= _MIN_TOKEN_LENGTH]
|
return [
|
||||||
|
token
|
||||||
|
for token in normalize_kb_text(value).split(" ")
|
||||||
|
if len(token) >= _MIN_TOKEN_LENGTH and token not in _STOPWORDS
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def search_kb_rows(
|
def search_kb_rows(
|
||||||
|
|||||||
@@ -114,6 +114,61 @@ def test_search_kb_rows_prefers_title_and_tags_over_body_only_mentions():
|
|||||||
assert results[0].title == rows[0].title
|
assert results[0].title == rows[0].title
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_kb_rows_ignores_stopword_only_query():
|
||||||
|
# A caller confirming the language ("да") should never surface an
|
||||||
|
# unrelated FAQ article just because that article's body happens to
|
||||||
|
# contain the word "да" somewhere in ordinary prose.
|
||||||
|
rows = [
|
||||||
|
_row(
|
||||||
|
row_id=1,
|
||||||
|
title="Активация ваучера",
|
||||||
|
body="Да, подтвердите СМС с номера 1414 командой 21*1.",
|
||||||
|
tags=["ваучер"],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
assert search_kb_rows(rows, "да", limit=5) == []
|
||||||
|
assert search_kb_rows(rows, "Хорошо", limit=5) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_kb_rows_still_matches_real_content_word_amid_fillers():
|
||||||
|
rows = [
|
||||||
|
_row(
|
||||||
|
row_id=1,
|
||||||
|
title="Активация ваучера",
|
||||||
|
body="Подтвердите СМС с номера 1414 командой 21*1.",
|
||||||
|
tags=["ваучер"],
|
||||||
|
),
|
||||||
|
_row(
|
||||||
|
row_id=2,
|
||||||
|
title="График работы",
|
||||||
|
body="Филиалы работают с 9 до 18.",
|
||||||
|
tags=["график"],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
results = search_kb_rows(rows, "да, у меня вопрос про ваучер", limit=5)
|
||||||
|
|
||||||
|
assert results
|
||||||
|
assert results[0].title == "Активация ваучера"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_kb_rows_single_real_word_query_still_matches():
|
||||||
|
rows = [
|
||||||
|
_row(
|
||||||
|
row_id=1,
|
||||||
|
title="Активация ваучера",
|
||||||
|
body="Подтвердите СМС с номера 1414 командой 21*1.",
|
||||||
|
tags=["ваучер"],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
results = search_kb_rows(rows, "ваучер", limit=5)
|
||||||
|
|
||||||
|
assert results
|
||||||
|
assert results[0].title == "Активация ваучера"
|
||||||
|
|
||||||
|
|
||||||
def test_search_kb_rows_breaks_ties_by_newer_id():
|
def test_search_kb_rows_breaks_ties_by_newer_id():
|
||||||
older = _row(
|
older = _row(
|
||||||
row_id=10,
|
row_id=10,
|
||||||
|
|||||||
Reference in New Issue
Block a user