sync: migrate ai-operator to Gitea (2026-08-10)
This commit is contained in:
konturai-ops
@@ -0,0 +1,16 @@
|
||||
package redaction
|
||||
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
openAIKeyPattern = regexp.MustCompile(`sk-[A-Za-z0-9_-]{8,}`)
|
||||
bearerPattern = regexp.MustCompile(`(?i)Bearer\s+[A-Za-z0-9._~+/-]+=*`)
|
||||
secretKVPattern = regexp.MustCompile(`(?i)(password|secret|api_key|token)=([^\s&]+)`)
|
||||
databaseURLPattern = regexp.MustCompile(`(?i)(postgres(?:ql)?://[^:\s/]+):([^@\s]+)@`)
|
||||
emailPattern = regexp.MustCompile(`(?i)\b([A-Z0-9._%+-])([A-Z0-9._%+-]*)(@[A-Z0-9.-]+\.[A-Z]{2,})\b`)
|
||||
phonePattern = regexp.MustCompile(`(?:\+7|8)\d{10}`)
|
||||
digits12Pattern = regexp.MustCompile(`\b\d{12}\b`)
|
||||
digits16Pattern = regexp.MustCompile(`\b\d{16}\b`)
|
||||
otpPattern = regexp.MustCompile(`(?i)((?:код(?: подтверждения)?|sms|смс|otp)[^0-9]{0,20})(\d{4,8})`)
|
||||
accountPattern = regexp.MustCompile(`(?i)((?:лицев(?:ой|ого)\s+сч[её]т)\D{0,20})(\d{5,20})`)
|
||||
)
|
||||
@@ -0,0 +1,136 @@
|
||||
package redaction
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type RedactionResult struct {
|
||||
Text string
|
||||
Applied bool
|
||||
Findings []Finding
|
||||
}
|
||||
|
||||
type Finding struct {
|
||||
Type string
|
||||
Count int
|
||||
}
|
||||
|
||||
type Redactor interface {
|
||||
RedactText(input string) RedactionResult
|
||||
RedactJSON(input map[string]any) map[string]any
|
||||
RedactBytesForLog(input []byte) string
|
||||
}
|
||||
|
||||
type DefaultRedactor struct{}
|
||||
|
||||
func New() DefaultRedactor { return DefaultRedactor{} }
|
||||
|
||||
func (DefaultRedactor) RedactText(input string) RedactionResult {
|
||||
out := input
|
||||
counts := map[string]int{}
|
||||
apply := func(kind string, fn func(string) string) {
|
||||
before := out
|
||||
out = fn(out)
|
||||
if out != before {
|
||||
counts[kind]++
|
||||
}
|
||||
}
|
||||
apply("database_url", func(s string) string { return databaseURLPattern.ReplaceAllString(s, `${1}:***MASKED***@`) })
|
||||
apply("api_key", func(s string) string { return openAIKeyPattern.ReplaceAllString(s, `sk-***MASKED***`) })
|
||||
apply("bearer", func(s string) string { return bearerPattern.ReplaceAllString(s, `Bearer ***MASKED***`) })
|
||||
apply("secret", func(s string) string { return secretKVPattern.ReplaceAllString(s, `${1}=***MASKED***`) })
|
||||
apply("email", func(s string) string { return emailPattern.ReplaceAllString(s, `${1}***${3}`) })
|
||||
apply("otp", func(s string) string {
|
||||
return otpPattern.ReplaceAllStringFunc(s, func(m string) string {
|
||||
if strings.Contains(strings.ToLower(m), "услуг") {
|
||||
return m
|
||||
}
|
||||
parts := otpPattern.FindStringSubmatch(m)
|
||||
if len(parts) != 3 {
|
||||
return m
|
||||
}
|
||||
return parts[1] + "CODE"
|
||||
})
|
||||
})
|
||||
apply("account", func(s string) string {
|
||||
return accountPattern.ReplaceAllStringFunc(s, func(m string) string {
|
||||
parts := accountPattern.FindStringSubmatch(m)
|
||||
if len(parts) != 3 {
|
||||
return m
|
||||
}
|
||||
return parts[1] + maskMiddle(parts[2], 0, 4, "****")
|
||||
})
|
||||
})
|
||||
apply("phone", func(s string) string {
|
||||
return phonePattern.ReplaceAllStringFunc(s, func(m string) string { return maskMiddle(m, 4, 4, "***") })
|
||||
})
|
||||
apply("iin", func(s string) string {
|
||||
return digits12Pattern.ReplaceAllStringFunc(s, func(m string) string { return maskMiddle(m, 4, 4, "****") })
|
||||
})
|
||||
apply("card", func(s string) string {
|
||||
return digits16Pattern.ReplaceAllStringFunc(s, func(m string) string { return maskMiddle(m, 4, 4, "********") })
|
||||
})
|
||||
findings := make([]Finding, 0, len(counts))
|
||||
for k, v := range counts {
|
||||
findings = append(findings, Finding{Type: k, Count: v})
|
||||
}
|
||||
return RedactionResult{Text: out, Applied: out != input, Findings: findings}
|
||||
}
|
||||
|
||||
func (r DefaultRedactor) RedactJSON(input map[string]any) map[string]any {
|
||||
return redactMap(r, input)
|
||||
}
|
||||
|
||||
func (DefaultRedactor) RedactBytesForLog(input []byte) string {
|
||||
return fmt.Sprintf("[bytes:%d redacted]", len(input))
|
||||
}
|
||||
|
||||
func redactMap(r DefaultRedactor, input map[string]any) map[string]any {
|
||||
out := map[string]any{}
|
||||
for k, v := range input {
|
||||
out[k] = redactValue(r, v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func redactValue(r DefaultRedactor, v any) any {
|
||||
switch x := v.(type) {
|
||||
case string:
|
||||
return r.RedactText(x).Text
|
||||
case map[string]any:
|
||||
return redactMap(r, x)
|
||||
case []any:
|
||||
out := make([]any, len(x))
|
||||
for i := range x {
|
||||
out[i] = redactValue(r, x[i])
|
||||
}
|
||||
return out
|
||||
case []string:
|
||||
out := make([]string, len(x))
|
||||
for i := range x {
|
||||
out[i] = r.RedactText(x[i]).Text
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
func maskMiddle(value string, keepStart, keepEnd int, mask string) string {
|
||||
if !utf8.ValidString(value) {
|
||||
return "***MASKED***"
|
||||
}
|
||||
r := []rune(value)
|
||||
if keepStart == 0 && keepEnd > 0 {
|
||||
if len(r) <= keepEnd {
|
||||
return mask
|
||||
}
|
||||
return mask + string(r[len(r)-keepEnd:])
|
||||
}
|
||||
if len(r) <= keepStart+keepEnd {
|
||||
return strings.Repeat("*", len(r))
|
||||
}
|
||||
return string(r[:keepStart]) + mask + string(r[len(r)-keepEnd:])
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package redaction
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRedactTextSensitiveValues(t *testing.T) {
|
||||
r := New()
|
||||
in := "phone +77771234567 iin 123456789012 card 4400123412341234 email test@example.com код 123456 sk-secretvalue Bearer abcdef password=qwerty postgres://u:pass@127.0.0.1/db"
|
||||
out := r.RedactText(in)
|
||||
for _, raw := range []string{"+77771234567", "123456789012", "4400123412341234", "test@example.com", "123456 sk-secretvalue", "Bearer abcdef", "password=qwerty", ":pass@"} {
|
||||
if strings.Contains(out.Text, raw) {
|
||||
t.Fatalf("raw sensitive value still present: %s in %s", raw, out.Text)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"+777***4567", "1234****9012", "4400********1234", "t***@example.com", "код CODE", "sk-***MASKED***"} {
|
||||
if !strings.Contains(out.Text, want) {
|
||||
t.Fatalf("missing masked value %q in %s", want, out.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactDoesNotOverRedactOrdinaryNumbers(t *testing.T) {
|
||||
out := New().RedactText("3 рабочих дня, код услуги 5104")
|
||||
if !strings.Contains(out.Text, "3 рабочих дня") || !strings.Contains(out.Text, "5104") {
|
||||
t.Fatalf("over-redacted ordinary text: %s", out.Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedactJSONRecursive(t *testing.T) {
|
||||
out := New().RedactJSON(map[string]any{"a": map[string]any{"phone": "+77771234567"}, "b": []any{"test@example.com"}})
|
||||
s := strings.Join([]string{out["a"].(map[string]any)["phone"].(string), out["b"].([]any)[0].(string)}, " ")
|
||||
if strings.Contains(s, "+77771234567") || strings.Contains(s, "test@example.com") {
|
||||
t.Fatalf("recursive redaction failed: %v", out)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user