diff --git a/services/shared/kb_search.py b/services/shared/kb_search.py index 1eca81e..6ce587e 100644 --- a/services/shared/kb_search.py +++ b/services/shared/kb_search.py @@ -19,6 +19,34 @@ _MIN_SOFT_MATCH_LENGTH = 4 _NON_WORD_RE = re.compile(r"[^\w]+", re.UNICODE) _SPACE_RE = re.compile(r"\s+") +# Common RU/KZ greetings, confirmations, pronouns, and particles that carry no +# topical signal on their own. Without this, a caller utterance as thin as +# "да" or "хорошо" could still exact-token-match some unrelated KB article +# that happens to contain that word in its body, and get returned as the +# top/only search result — read back to the caller as if it were the answer +# to their question. Filtering these keeps _score_row's exact-token match +# meaningful: a match now requires an actual content word. +_STOPWORDS = frozenset( + { + # RU: greetings / confirmations / fillers + "алло", "ага", "да", "неа", "нет", "ой", "ок", "окей", "угу", "ясно", + "ладно", "хорошо", "понял", "поняла", "привет", "здравствуйте", + "добрый", "день", "вечер", "утро", "слышу", "слышно", "спасибо", + "пожалуйста", "извините", "простите", "алло", + # RU: pronouns / conjunctions / particles with no topical content + "я", "ты", "вы", "мы", "он", "она", "они", "это", "то", "и", "а", + "но", "или", "что", "как", "где", "когда", "если", "чтобы", "для", + "из", "по", "на", "в", "с", "у", "о", "же", "ли", "бы", "не", "ну", + "вот", "просто", "есть", "быть", "можно", "нужно", "надо", "уже", + # KZ: greetings / confirmations / fillers + "иә", "ия", "жоқ", "жарайды", "түсінікті", "рахмет", "сәлем", + "сәлеметсіз", "бе", "кешіріңіз", + # KZ: pronouns / conjunctions / particles + "мен", "сен", "сіз", "біз", "ол", "олар", "және", "бірақ", "немесе", + "не", "қалай", "қайда", "қашан", "үшін", "туралы", + } +) + class KBSearchRow(Protocol): id: int @@ -38,7 +66,11 @@ def normalize_kb_text(value: str | None) -> str: def tokenize_kb_text(value: str | None) -> list[str]: - return [token for token in normalize_kb_text(value).split(" ") if len(token) >= _MIN_TOKEN_LENGTH] + return [ + token + for token in normalize_kb_text(value).split(" ") + if len(token) >= _MIN_TOKEN_LENGTH and token not in _STOPWORDS + ] def search_kb_rows( diff --git a/tests/test_kb_search.py b/tests/test_kb_search.py index eeda4c3..47bd9ff 100644 --- a/tests/test_kb_search.py +++ b/tests/test_kb_search.py @@ -114,6 +114,61 @@ def test_search_kb_rows_prefers_title_and_tags_over_body_only_mentions(): assert results[0].title == rows[0].title +def test_search_kb_rows_ignores_stopword_only_query(): + # A caller confirming the language ("да") should never surface an + # unrelated FAQ article just because that article's body happens to + # contain the word "да" somewhere in ordinary prose. + rows = [ + _row( + row_id=1, + title="Активация ваучера", + body="Да, подтвердите СМС с номера 1414 командой 21*1.", + tags=["ваучер"], + ), + ] + + assert search_kb_rows(rows, "да", limit=5) == [] + assert search_kb_rows(rows, "Хорошо", limit=5) == [] + + +def test_search_kb_rows_still_matches_real_content_word_amid_fillers(): + rows = [ + _row( + row_id=1, + title="Активация ваучера", + body="Подтвердите СМС с номера 1414 командой 21*1.", + tags=["ваучер"], + ), + _row( + row_id=2, + title="График работы", + body="Филиалы работают с 9 до 18.", + tags=["график"], + ), + ] + + results = search_kb_rows(rows, "да, у меня вопрос про ваучер", limit=5) + + assert results + assert results[0].title == "Активация ваучера" + + +def test_search_kb_rows_single_real_word_query_still_matches(): + rows = [ + _row( + row_id=1, + title="Активация ваучера", + body="Подтвердите СМС с номера 1414 командой 21*1.", + tags=["ваучер"], + ), + ] + + results = search_kb_rows(rows, "ваучер", limit=5) + + assert results + assert results[0].title == "Активация ваучера" + + def test_search_kb_rows_breaks_ties_by_newer_id(): older = _row( row_id=10,