sync: migrate ai-operator to Gitea (2026-08-10)
This commit is contained in:
konturai-ops
@@ -0,0 +1,130 @@
|
||||
package language
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"ai-operator/internal/dialogue/state"
|
||||
)
|
||||
|
||||
type Detector struct{}
|
||||
|
||||
func NewDetector() *Detector { return &Detector{} }
|
||||
func (d *Detector) Normalize(input string) string { return Normalize(input) }
|
||||
func (d *Detector) DetectToolLanguage(value string) DetectionResult {
|
||||
return d.Detect(value, SourceToolArgs)
|
||||
}
|
||||
|
||||
func (d *Detector) Detect(input string, source DetectionSource) DetectionResult {
|
||||
n := Normalize(input)
|
||||
res := DetectionResult{Language: LanguageUnknown, Intent: IntentNotLanguage, Source: source, NormalizedText: n, ReasonCode: "no_match"}
|
||||
if n == "" {
|
||||
res.ReasonCode = "empty_input"
|
||||
res.NeedsClarification = true
|
||||
return res
|
||||
}
|
||||
if len([]rune(n)) < 2 {
|
||||
res.ReasonCode = "too_short"
|
||||
return res
|
||||
}
|
||||
if containsAny(n, ambiguousPhrases) || mentionsBoth(n) {
|
||||
res.Intent = IntentAmbiguous
|
||||
res.ReasonCode = "ambiguous_ru_kk"
|
||||
res.Confidence = 0.4
|
||||
res.NeedsClarification = true
|
||||
return res
|
||||
}
|
||||
if containsAny(n, falsePositivePhrases) {
|
||||
res.ReasonCode = "no_match"
|
||||
return res
|
||||
}
|
||||
if reason, ok := ruExact[n]; ok {
|
||||
return result(LanguageRU, IntentLanguageSelect, 0.99, n, n, source, reason, false)
|
||||
}
|
||||
if reason, ok := kkExact[n]; ok {
|
||||
return result(LanguageKK, IntentLanguageSelect, 0.95, n, n, source, reason, false)
|
||||
}
|
||||
if d.IsExplicitChangeRequest(n) {
|
||||
if p := phraseMatch(n, explicitRu); p != "" {
|
||||
return result(LanguageRU, IntentLanguageChange, 0.95, p, n, source, "explicit_language_request", false)
|
||||
}
|
||||
if p := phraseMatch(n, explicitKK); p != "" {
|
||||
return result(LanguageKK, IntentLanguageChange, 0.95, p, n, source, "explicit_language_request", false)
|
||||
}
|
||||
return DetectionResult{Language: LanguageUnknown, Intent: IntentAmbiguous, Confidence: 0.5, NormalizedText: n, Source: source, ReasonCode: "explicit_language_request", NeedsClarification: true}
|
||||
}
|
||||
if p := exactOrPhrase(n, ruPhrases); p != "" {
|
||||
conf := 0.95
|
||||
if n == "я русский" {
|
||||
conf = 0.55
|
||||
}
|
||||
return result(LanguageRU, IntentLanguageSelect, conf, p, n, source, "exact_phrase", conf < MediumConfidence)
|
||||
}
|
||||
if p := exactOrPhrase(n, kkPhrases); p != "" {
|
||||
return result(LanguageKK, IntentLanguageSelect, 0.95, p, n, source, "exact_phrase", false)
|
||||
}
|
||||
if p := exactOrPhrase(n, ruASR); p != "" {
|
||||
return result(LanguageRU, IntentLanguageSelect, 0.80, p, n, source, "weak_match", false)
|
||||
}
|
||||
if p := exactOrPhrase(n, kkASR); p != "" {
|
||||
return result(LanguageKK, IntentLanguageSelect, 0.80, p, n, source, "weak_match", false)
|
||||
}
|
||||
if n == "я русский" {
|
||||
return result(LanguageRU, IntentAmbiguous, 0.55, n, n, source, "weak_match", true)
|
||||
}
|
||||
if n == "я казах" {
|
||||
return result(LanguageKK, IntentAmbiguous, 0.55, n, n, source, "weak_match", true)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (d *Detector) IsExplicitChangeRequest(input string) bool {
|
||||
n := Normalize(input)
|
||||
return containsAny(n, explicitChangeMarkers) || containsAny(n, explicitRu) || containsAny(n, explicitKK)
|
||||
}
|
||||
|
||||
func ShouldApplyLanguageDetection(ctx SelectionContext, res DetectionResult) LanguageDecision {
|
||||
if ctx.CurrentState == state.StateEnded || ctx.CurrentState == state.StateHandoff || ctx.CurrentState == state.StateClosing {
|
||||
return LanguageDecision{ReasonCode: "state_not_allowed", MessageKey: "language.change.denied"}
|
||||
}
|
||||
if res.NeedsClarification || res.Intent == IntentAmbiguous {
|
||||
return LanguageDecision{NeedsClarification: true, ReasonCode: res.ReasonCode, MessageKey: "language.ask_clarify"}
|
||||
}
|
||||
if res.Language != LanguageRU && res.Language != LanguageKK {
|
||||
return LanguageDecision{ReasonCode: res.ReasonCode, MessageKey: "language.not_understood"}
|
||||
}
|
||||
switch ctx.CurrentState {
|
||||
case state.StateLanguageSelection:
|
||||
if res.Confidence >= MediumConfidence && res.Intent == IntentLanguageSelect {
|
||||
return LanguageDecision{Apply: true, Language: res.Language, ReasonCode: res.ReasonCode, MessageKey: "language.selected." + string(res.Language)}
|
||||
}
|
||||
case state.StateRegionSelection:
|
||||
if res.Confidence >= HighConfidence && (res.Intent == IntentLanguageChange || res.Intent == IntentLanguageSelect) {
|
||||
return LanguageDecision{Apply: true, Language: res.Language, ReasonCode: res.ReasonCode, MessageKey: "language.changed." + string(res.Language)}
|
||||
}
|
||||
case state.StateReadyToHelp, state.StateQuestionAnswering:
|
||||
if ctx.AllowChange && res.Intent == IntentLanguageChange && res.Confidence >= HighConfidence {
|
||||
return LanguageDecision{Apply: true, Language: res.Language, ReasonCode: res.ReasonCode, MessageKey: "language.changed." + string(res.Language)}
|
||||
}
|
||||
}
|
||||
return LanguageDecision{ReasonCode: "mixed_language_no_explicit_switch", MessageKey: "language.not_understood"}
|
||||
}
|
||||
|
||||
func result(lang Language, intent DetectionIntent, conf float64, phrase, norm string, src DetectionSource, reason string, clarify bool) DetectionResult {
|
||||
return DetectionResult{Language: lang, Intent: intent, Confidence: conf, MatchedPhrase: phrase, NormalizedText: norm, Source: src, ReasonCode: reason, NeedsClarification: clarify}
|
||||
}
|
||||
func exactOrPhrase(n string, phrases []string) string {
|
||||
for _, p := range phrases {
|
||||
p = Normalize(p)
|
||||
if n == p || strings.Contains(n, p) && len([]rune(p)) > 4 {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func phraseMatch(n string, phrases []string) string { return exactOrPhrase(n, phrases) }
|
||||
func containsAny(n string, phrases []string) bool { return exactOrPhrase(n, phrases) != "" }
|
||||
func mentionsBoth(n string) bool {
|
||||
ru := exactOrPhrase(n, append(ruPhrases, ruASR...)) != "" || strings.Contains(" "+n+" ", " ru ")
|
||||
kk := exactOrPhrase(n, append(kkPhrases, kkASR...)) != "" || strings.Contains(" "+n+" ", " kk ")
|
||||
return ru && kk
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package language
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"ai-operator/internal/dialogue/state"
|
||||
)
|
||||
|
||||
func TestNormalize(t *testing.T) {
|
||||
cases := map[string]string{
|
||||
" RU ": "ru",
|
||||
"Қазақша!": "қазақша",
|
||||
"по-русски": "по русски",
|
||||
"ёж тест": "еж тест",
|
||||
"kk.": "kk",
|
||||
"қазақ тілі": "қазақ тілі",
|
||||
}
|
||||
for in, want := range cases {
|
||||
if got := Normalize(in); got != want {
|
||||
t.Fatalf("Normalize(%q)=%q want=%q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRUDetection(t *testing.T) {
|
||||
d := NewDetector()
|
||||
inputs := []string{"ru", "rus", "русский", "русский язык", "на русском", "по русски", "по-русски", "хочу русский", "хочу на русском", "давайте на русском", "говорите на русском", "продолжим на русском", "выбираю русский", "мне русский", "нужен русский", "русский пожалуйста", "russian", "in russian", "speak russian", "russki", "russkiy", "po russki", "руский", "русски", "порусски", "по руски"}
|
||||
for _, in := range inputs {
|
||||
got := d.Detect(in, SourceUserText)
|
||||
if got.Language != LanguageRU || got.Confidence < MediumConfidence || got.NeedsClarification {
|
||||
t.Fatalf("%q => %+v", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestKKDetection(t *testing.T) {
|
||||
d := NewDetector()
|
||||
inputs := []string{"kk", "kz", "қазақша", "қазақ тілі", "қазақ тілінде", "қазақша сөйлейік", "қазақша болсын", "қазақ тілін таңдадым", "қазақша жауап беріңіз", "маған қазақша", "мен қазақша", "қазакша", "казакша", "казахский", "казахский язык", "на казахском", "по казахски", "по-казахски", "хочу казахский", "говорите на казахском", "qazaqsha", "qazaq tili", "kazakh", "in kazakh", "kazaksha", "казақша", "показахски"}
|
||||
for _, in := range inputs {
|
||||
got := d.Detect(in, SourceUserText)
|
||||
if got.Language != LanguageKK || got.Confidence < MediumConfidence || got.NeedsClarification {
|
||||
t.Fatalf("%q => %+v", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAmbiguousAndFalsePositive(t *testing.T) {
|
||||
d := NewDetector()
|
||||
amb := []string{"я русский", "я казах", "русский или қазақша?", "можно русский, нет қазақша", "сначала русский потом казахский", "я не знаю русский или қазақша"}
|
||||
for _, in := range amb {
|
||||
got := d.Detect(in, SourceUserText)
|
||||
if !got.NeedsClarification || got.Intent != IntentAmbiguous {
|
||||
t.Fatalf("ambiguous %q => %+v", in, got)
|
||||
}
|
||||
}
|
||||
falsePos := []string{"Казахтелеком", "русский клиент спрашивает", "У меня вопрос на русском сайте", "Алматы русский театр", "тариф ru123", "abckkdef", "какой у меня тариф", "хочу узнать баланс", "оператор нужен"}
|
||||
for _, in := range falsePos {
|
||||
got := d.Detect(in, SourceUserText)
|
||||
if got.Language != LanguageUnknown || got.Intent != IntentNotLanguage {
|
||||
t.Fatalf("false positive %q => %+v", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplicitChangeAndPolicy(t *testing.T) {
|
||||
d := NewDetector()
|
||||
cases := map[string]Language{"перейдите на русский": LanguageRU, "переключите на русский": LanguageRU, "switch to russian": LanguageRU, "қазақшаға ауысайық": LanguageKK, "қазақша сөйлейік": LanguageKK, "switch to kazakh": LanguageKK}
|
||||
for in, want := range cases {
|
||||
got := d.Detect(in, SourceTranscript)
|
||||
if got.Intent != IntentLanguageChange || got.Language != want {
|
||||
t.Fatalf("change %q => %+v", in, got)
|
||||
}
|
||||
}
|
||||
clarify := d.Detect("сменить язык", SourceTranscript)
|
||||
if !clarify.NeedsClarification || clarify.Language != LanguageUnknown {
|
||||
t.Fatalf("expected clarification: %+v", clarify)
|
||||
}
|
||||
if !ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateLanguageSelection}, d.Detect("русский", SourceUserText)).Apply {
|
||||
t.Fatal("language selection did not apply ru")
|
||||
}
|
||||
if !ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateLanguageSelection}, d.Detect("қазақша", SourceUserText)).Apply {
|
||||
t.Fatal("language selection did not apply kk")
|
||||
}
|
||||
if ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateLanguageSelection}, d.Detect("какой у меня тариф", SourceUserText)).Apply {
|
||||
t.Fatal("business question applied language")
|
||||
}
|
||||
if !ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateLanguageSelection}, d.Detect("русский или қазақша?", SourceUserText)).NeedsClarification {
|
||||
t.Fatal("ambiguous did not request clarification")
|
||||
}
|
||||
if ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateRegionSelection, AllowChange: true}, d.Detect("русский клиент", SourceUserText)).Apply {
|
||||
t.Fatal("random mention switched language")
|
||||
}
|
||||
if !ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateReadyToHelp, AllowChange: true}, d.Detect("перейдите на казахский", SourceUserText)).Apply {
|
||||
t.Fatal("explicit ready switch did not apply")
|
||||
}
|
||||
if ShouldApplyLanguageDetection(SelectionContext{CurrentState: state.StateEnded, AllowChange: true}, d.Detect("русский", SourceUserText)).Apply {
|
||||
t.Fatal("ended state applied language")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package language
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var spaceRE = regexp.MustCompile(`\s+`)
|
||||
var edgePunctRE = regexp.MustCompile(`^[\s\.,!\?;:"'«»\(\)\[\]\{\}]+|[\s\.,!\?;:"'«»\(\)\[\]\{\}]+$`)
|
||||
|
||||
func Normalize(input string) string {
|
||||
s := strings.TrimSpace(strings.ToLower(input))
|
||||
s = strings.ReplaceAll(s, "ё", "е")
|
||||
s = strings.ReplaceAll(s, "-", " ")
|
||||
s = edgePunctRE.ReplaceAllString(s, "")
|
||||
s = strings.Map(func(r rune) rune {
|
||||
switch r {
|
||||
case '.', ',', '!', '?', ';', ':', '"', '\'', '«', '»', '(', ')', '[', ']', '{', '}':
|
||||
return ' '
|
||||
default:
|
||||
return r
|
||||
}
|
||||
}, s)
|
||||
s = spaceRE.ReplaceAllString(strings.TrimSpace(s), " ")
|
||||
return s
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package language
|
||||
|
||||
var ruExact = map[string]string{"ru": "exact_code", "rus": "exact_code"}
|
||||
var kkExact = map[string]string{"kk": "exact_code", "kz": "exact_code_alias"}
|
||||
|
||||
var ruPhrases = []string{
|
||||
"русский", "русский язык", "на русском", "по русски", "хочу русский", "хочу на русском", "давайте на русском", "говорите на русском", "продолжим на русском", "выбираю русский", "мне русский", "нужен русский", "рус", "русский пожалуйста", "можно на русском",
|
||||
"russian", "in russian", "speak russian", "russki", "russkiy", "po russki",
|
||||
}
|
||||
var ruASR = []string{"руский", "русски", "руссский", "руский язык", "порусски", "по руски"}
|
||||
|
||||
var kkPhrases = []string{
|
||||
"қазақша", "қазақ тілі", "қазақ тілінде", "қазақша сөйлейік", "қазақша сөйлесеміз", "қазақша болсын", "қазақ тілін таңдадым", "қазақша қызмет", "қазақша жауап беріңіз", "маған қазақша", "мен қазақша", "қазакша", "казакша", "казахский", "казахский язык", "на казахском", "по казахски", "хочу казахский", "говорите на казахском", "продолжим на казахском",
|
||||
"qazaqsha", "qazaq tili", "qazaq tilinde", "kazakh", "kazakh language", "in kazakh", "kazaksha", "qazaq",
|
||||
}
|
||||
var kkASR = []string{"казакша", "қазақшаа", "казақша", "казахски", "показахски", "по казахски"}
|
||||
|
||||
var ambiguousPhrases = []string{"я русский", "я казах", "русский или қазақша", "русский или казахский", "можно русский нет қазақша", "сначала русский потом казахский", "я не знаю русский или қазақша", "я не знаю русский или казахский"}
|
||||
var falsePositivePhrases = []string{"казахтелеком", "русский клиент", "русский клиент спрашивает", "у меня вопрос на русском сайте", "алматы русский театр", "какой у меня тариф", "хочу узнать баланс", "оператор нужен"}
|
||||
|
||||
var explicitChangeMarkers = []string{"сменить язык", "поменять язык", "давайте сменим язык", "можно сменить язык", "хочу сменить язык", "перейдите", "переключите", "switch language", "change language", "switch to", "тілді ауыстыру", "тілді өзгерту", "ауысайық", "ауысамын"}
|
||||
var explicitRu = []string{"говорите по русски", "говорите на русском", "перейдите на русский", "переключите на русский", "давайте на русском", "лучше на русском", "орысша сөйлейік", "орыс тіліне ауысайық", "switch to russian", "russian please", "русскийға ауысайық"}
|
||||
var explicitKK = []string{"қазақшаға ауысайық", "қазақша сөйлейік", "қазақша жауап беріңіз", "перейдите на казахский", "переключите на казахский", "switch to kazakh", "qazaqsha"}
|
||||
@@ -0,0 +1,60 @@
|
||||
package language
|
||||
|
||||
import "ai-operator/internal/dialogue/state"
|
||||
|
||||
type Language = state.Language
|
||||
|
||||
const (
|
||||
LanguageUnknown Language = state.LanguageUnknown
|
||||
LanguageRU Language = state.LanguageRU
|
||||
LanguageKK Language = state.LanguageKK
|
||||
)
|
||||
|
||||
type DetectionSource string
|
||||
|
||||
const (
|
||||
SourceUserText DetectionSource = "user_text"
|
||||
SourceToolArgs DetectionSource = "tool_args"
|
||||
SourceTranscript DetectionSource = "transcript"
|
||||
SourceCLI DetectionSource = "cli"
|
||||
)
|
||||
|
||||
type DetectionIntent string
|
||||
|
||||
const (
|
||||
IntentLanguageSelect DetectionIntent = "language_select"
|
||||
IntentLanguageChange DetectionIntent = "language_change"
|
||||
IntentNotLanguage DetectionIntent = "not_language"
|
||||
IntentAmbiguous DetectionIntent = "ambiguous"
|
||||
)
|
||||
|
||||
const (
|
||||
HighConfidence = 0.90
|
||||
MediumConfidence = 0.70
|
||||
LowConfidence = 0.50
|
||||
)
|
||||
|
||||
type DetectionResult struct {
|
||||
Language Language
|
||||
Intent DetectionIntent
|
||||
Confidence float64
|
||||
MatchedPhrase string
|
||||
NormalizedText string
|
||||
Source DetectionSource
|
||||
ReasonCode string
|
||||
NeedsClarification bool
|
||||
}
|
||||
|
||||
type SelectionContext struct {
|
||||
CurrentState state.ConversationState
|
||||
CurrentLanguage Language
|
||||
AllowChange bool
|
||||
}
|
||||
|
||||
type LanguageDecision struct {
|
||||
Apply bool
|
||||
Language Language
|
||||
NeedsClarification bool
|
||||
ReasonCode string
|
||||
MessageKey string
|
||||
}
|
||||
Reference in New Issue
Block a user