27 lines
658 B
Go
27 lines
658 B
Go
package language
|
||
|
||
import (
|
||
"regexp"
|
||
"strings"
|
||
)
|
||
|
||
var spaceRE = regexp.MustCompile(`\s+`)
|
||
var edgePunctRE = regexp.MustCompile(`^[\s\.,!\?;:"'«»\(\)\[\]\{\}]+|[\s\.,!\?;:"'«»\(\)\[\]\{\}]+$`)
|
||
|
||
func Normalize(input string) string {
|
||
s := strings.TrimSpace(strings.ToLower(input))
|
||
s = strings.ReplaceAll(s, "ё", "е")
|
||
s = strings.ReplaceAll(s, "-", " ")
|
||
s = edgePunctRE.ReplaceAllString(s, "")
|
||
s = strings.Map(func(r rune) rune {
|
||
switch r {
|
||
case '.', ',', '!', '?', ';', ':', '"', '\'', '«', '»', '(', ')', '[', ']', '{', '}':
|
||
return ' '
|
||
default:
|
||
return r
|
||
}
|
||
}, s)
|
||
s = spaceRE.ReplaceAllString(strings.TrimSpace(s), " ")
|
||
return s
|
||
}
|