sync: migrate ai-operator to Gitea (2026-08-10)
This commit is contained in:
konturai-ops
@@ -0,0 +1,45 @@
|
||||
package kb
|
||||
|
||||
import "unicode/utf8"
|
||||
|
||||
type ChunkerConfig struct{ Target, Min, Max, Overlap int }
|
||||
|
||||
func DefaultChunkerConfig() ChunkerConfig {
|
||||
return ChunkerConfig{Target: 1200, Min: 200, Max: 2000, Overlap: 150}
|
||||
}
|
||||
func ChunkDocument(doc Document, cfg ChunkerConfig) []Chunk {
|
||||
text := BuildChunkText(doc)
|
||||
if cfg.Target <= 0 {
|
||||
cfg = DefaultChunkerConfig()
|
||||
}
|
||||
rs := []rune(text)
|
||||
if len(rs) == 0 {
|
||||
return nil
|
||||
}
|
||||
var chunks []Chunk
|
||||
start := 0
|
||||
idx := 0
|
||||
for start < len(rs) {
|
||||
end := start + cfg.Target
|
||||
if end > len(rs) {
|
||||
end = len(rs)
|
||||
}
|
||||
if end-start > cfg.Max {
|
||||
end = start + cfg.Max
|
||||
}
|
||||
content := string(rs[start:end])
|
||||
if !utf8.ValidString(content) {
|
||||
content = string([]rune(content))
|
||||
}
|
||||
chunks = append(chunks, Chunk{ChunkIndex: idx, Language: doc.Language, RegionCode: doc.RegionCode, Title: doc.Title, Content: content, ContentHash: ContentHash(content), CharCount: len([]rune(content)), TokenEstimate: len([]rune(content)) / 4, Metadata: map[string]any{"external_id": doc.ExternalID, "source_uri": doc.SourceURI}})
|
||||
idx++
|
||||
if end == len(rs) {
|
||||
break
|
||||
}
|
||||
start = end - cfg.Overlap
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
package kb
|
||||
|
||||
func citationFor(r SearchResult) Citation {
|
||||
return Citation{DocumentTitle: r.Title, SourceURI: r.SourceURI, ChunkID: r.ChunkID, RegionCode: r.RegionCode, Language: r.Language, Source: r.Citation.Source}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package kb
|
||||
@@ -0,0 +1,174 @@
|
||||
package kb
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"ai-operator/internal/dialogue/region"
|
||||
)
|
||||
|
||||
var requiredFields = []string{"external_id", "title", "language", "region_code", "status", "content", "source_hash"}
|
||||
|
||||
func ParseJSONLFile(path string, allowDisabled bool) ([]Document, []string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return ParseJSONL(f, path, allowDisabled)
|
||||
}
|
||||
|
||||
func ParseJSONL(r io.Reader, sourcePath string, allowDisabled bool) ([]Document, []string, error) {
|
||||
resolver := region.NewDefaultResolver()
|
||||
sc := bufio.NewScanner(r)
|
||||
sc.Buffer(make([]byte, 1024), 10*1024*1024)
|
||||
var docs []Document
|
||||
var errs []string
|
||||
line := 0
|
||||
for sc.Scan() {
|
||||
line++
|
||||
raw := bytes.TrimSpace(sc.Bytes())
|
||||
if len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
errs = append(errs, fmt.Sprintf("%s:%d invalid json", sourcePath, line))
|
||||
continue
|
||||
}
|
||||
missing := ""
|
||||
for _, k := range requiredFields {
|
||||
if strings.TrimSpace(asString(m[k])) == "" {
|
||||
missing = k
|
||||
break
|
||||
}
|
||||
}
|
||||
if missing != "" {
|
||||
errs = append(errs, fmt.Sprintf("%s:%d missing %s", sourcePath, line, missing))
|
||||
continue
|
||||
}
|
||||
lang := asString(m["language"])
|
||||
if lang != "ru" && lang != "kk" {
|
||||
errs = append(errs, fmt.Sprintf("%s:%d invalid language", sourcePath, line))
|
||||
continue
|
||||
}
|
||||
regCode := asString(m["region_code"])
|
||||
if regCode != "global" {
|
||||
reg, ok := resolver.GetByCode(regCode)
|
||||
if !ok || (!reg.Enabled && !allowDisabled) {
|
||||
errs = append(errs, fmt.Sprintf("%s:%d invalid region_code", sourcePath, line))
|
||||
continue
|
||||
}
|
||||
}
|
||||
status := asString(m["status"])
|
||||
if status != "draft" && status != "published" && status != "archived" {
|
||||
errs = append(errs, fmt.Sprintf("%s:%d invalid status", sourcePath, line))
|
||||
continue
|
||||
}
|
||||
metadata := map[string]any{}
|
||||
if mm, ok := m["metadata"].(map[string]any); ok {
|
||||
for k, v := range mm {
|
||||
metadata[k] = v
|
||||
}
|
||||
}
|
||||
known := map[string]bool{"external_id": true, "title": true, "language": true, "region_code": true, "category": true, "status": true, "content": true, "question": true, "short_answer": true, "full_answer": true, "keywords": true, "source": true, "source_type": true, "source_uri": true, "source_hash": true, "metadata": true}
|
||||
for k, v := range m {
|
||||
if !known[k] {
|
||||
metadata[k] = v
|
||||
}
|
||||
}
|
||||
docs = append(docs, Document{ExternalID: asString(m["external_id"]), Title: asString(m["title"]), Language: lang, RegionCode: regCode, Category: defaultString(asString(m["category"]), "general"), SourceType: defaultString(asString(m["source_type"]), "jsonl"), SourceURI: defaultString(asString(m["source_uri"]), filepath.Base(sourcePath)), SourceHash: asString(m["source_hash"]), Status: status, Content: asString(m["content"]), Question: asString(m["question"]), ShortAnswer: asString(m["short_answer"]), FullAnswer: asString(m["full_answer"]), Source: asString(m["source"]), Keywords: nonNilStrings(asStringSlice(m["keywords"])), Metadata: metadata})
|
||||
}
|
||||
return docs, errs, sc.Err()
|
||||
}
|
||||
|
||||
func LoadJSONLDocuments(ctx context.Context, root string, allowDisabled bool) ([]Document, []string, error) {
|
||||
var docs []Document
|
||||
var errs []string
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".jsonl") {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
dd, ee, err := ParseJSONLFile(path, allowDisabled)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
docs = append(docs, dd...)
|
||||
errs = append(errs, ee...)
|
||||
return nil
|
||||
})
|
||||
return docs, errs, err
|
||||
}
|
||||
|
||||
func BuildChunkText(doc Document) string {
|
||||
parts := []string{doc.Title}
|
||||
if doc.Question != "" {
|
||||
parts = append(parts, "Вопрос: "+doc.Question)
|
||||
}
|
||||
if doc.ShortAnswer != "" {
|
||||
parts = append(parts, "Краткий ответ: "+doc.ShortAnswer)
|
||||
}
|
||||
if doc.FullAnswer != "" {
|
||||
parts = append(parts, "Расширенное пояснение: "+doc.FullAnswer)
|
||||
}
|
||||
if len(doc.Keywords) > 0 {
|
||||
parts = append(parts, "Ключевые слова: "+strings.Join(doc.Keywords, ", "))
|
||||
}
|
||||
if doc.Source != "" {
|
||||
parts = append(parts, "Источник: "+doc.Source)
|
||||
}
|
||||
if doc.Content != "" {
|
||||
parts = append(parts, doc.Content)
|
||||
}
|
||||
return strings.Join(parts, "\n\n")
|
||||
}
|
||||
func ContentHash(s string) string { h := sha256.Sum256([]byte(s)); return hex.EncodeToString(h[:]) }
|
||||
func asString(v any) string {
|
||||
if s, ok := v.(string); ok {
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func defaultString(v, d string) string {
|
||||
if v == "" {
|
||||
return d
|
||||
}
|
||||
return v
|
||||
}
|
||||
func asStringSlice(v any) []string {
|
||||
arr, ok := v.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
out := []string{}
|
||||
for _, x := range arr {
|
||||
if s := asString(x); s != "" {
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func nonNilStrings(v []string) []string {
|
||||
if v == nil {
|
||||
return []string{}
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package kb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"ai-operator/internal/config"
|
||||
"ai-operator/internal/embedding"
|
||||
)
|
||||
|
||||
func TestParseJSONLTolerant(t *testing.T) {
|
||||
line := `{"external_id":"id1","title":"T","language":"ru","region_code":"global","status":"published","content":"hello","source_hash":"abc","keywords":["a"],"extra":"x","metadata":{"m":1}}`
|
||||
docs, errs, err := ParseJSONL(strings.NewReader(line), "test.jsonl", false)
|
||||
if err != nil || len(errs) != 0 || len(docs) != 1 {
|
||||
t.Fatalf("docs=%d errs=%v err=%v", len(docs), errs, err)
|
||||
}
|
||||
if docs[0].Metadata["extra"] != "x" || len(docs[0].Keywords) != 1 {
|
||||
t.Fatalf("metadata/keywords not parsed: %+v", docs[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONLMissingRequiredSkipped(t *testing.T) {
|
||||
docs, errs, err := ParseJSONL(strings.NewReader(`{"title":"T"}`), "bad.jsonl", false)
|
||||
if err != nil || len(docs) != 0 || len(errs) != 1 {
|
||||
t.Fatalf("docs=%d errs=%v err=%v", len(docs), errs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkDocumentUTF8(t *testing.T) {
|
||||
doc := Document{ExternalID: "x", Title: "Заголовок", Language: "ru", RegionCode: "global", Content: strings.Repeat("Қазақша текст. ", 200)}
|
||||
chunks := ChunkDocument(doc, DefaultChunkerConfig())
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("expected multiple chunks")
|
||||
}
|
||||
for _, ch := range chunks {
|
||||
if ch.Content == "" || ch.CharCount == 0 || !strings.Contains(ch.Title, "Заголовок") {
|
||||
t.Fatalf("bad chunk %+v", ch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceValidationNoAnswer(t *testing.T) {
|
||||
s := NewService(fakeRepo{}, fakeEmbed{}, testKBConfig())
|
||||
resp, _ := s.Search(context.Background(), SearchRequest{Query: "", Language: "ru", RegionCode: "global"})
|
||||
if resp.ReasonCode != "query_too_short" {
|
||||
t.Fatalf("bad empty response %+v", resp)
|
||||
}
|
||||
resp, _ = s.Search(context.Background(), SearchRequest{Query: "none", Language: "ru", RegionCode: "global"})
|
||||
if resp.ReasonCode != "no_relevant_knowledge" {
|
||||
t.Fatalf("bad no answer %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeRepo struct{}
|
||||
|
||||
func (fakeRepo) Health(context.Context) (Health, error) { return Health{}, nil }
|
||||
func (fakeRepo) UpsertDocument(context.Context, Document, []Chunk) error { return nil }
|
||||
func (fakeRepo) Search(context.Context, SearchRequest, []float32) ([]SearchResult, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
type fakeEmbed struct{}
|
||||
|
||||
func (fakeEmbed) Embed(context.Context, []string) ([]embedding.Vector, error) {
|
||||
return []embedding.Vector{make(embedding.Vector, 1536)}, nil
|
||||
}
|
||||
func (fakeEmbed) Dimensions() int { return 1536 }
|
||||
func (fakeEmbed) Model() string { return "fake" }
|
||||
func (fakeEmbed) ProviderName() string { return "fake" }
|
||||
func testKBConfig() config.KBConfig {
|
||||
return config.KBConfig{DefaultLimit: 5, MaxLimit: 10, MinScore: 0.2, QueryMaxChars: 1000, CrossLanguageFallback: true}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package kb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Document struct {
|
||||
ID, ExternalID, Title, Language, RegionCode, Category, SourceType, SourceURI, SourceHash, Status string
|
||||
Content, Question, ShortAnswer, FullAnswer, Source string
|
||||
Keywords []string
|
||||
Metadata map[string]any
|
||||
ValidFrom, ValidTo *time.Time
|
||||
Priority int
|
||||
}
|
||||
|
||||
type Chunk struct {
|
||||
ID, DocumentID string
|
||||
ChunkIndex int
|
||||
Language, RegionCode, Title, Content, ContentHash string
|
||||
TokenEstimate, CharCount int
|
||||
Embedding []float32
|
||||
EmbeddingModel, EmbeddingProvider string
|
||||
Metadata map[string]any
|
||||
}
|
||||
|
||||
type SearchRequest struct {
|
||||
Query, Language, RegionCode, CallID string
|
||||
Limit int
|
||||
MinScore float64
|
||||
IncludeGlobal bool
|
||||
CrossLanguageFallback bool
|
||||
}
|
||||
type Citation struct{ DocumentTitle, SourceURI, ChunkID, RegionCode, Language, Source string }
|
||||
type SearchResult struct {
|
||||
DocumentID, ChunkID, Title, Content, Language, SourceLanguage, RegionCode, Category, SourceURI string
|
||||
Score, VectorScore, KeywordScore float64
|
||||
CrossLanguageFallback bool
|
||||
Metadata map[string]any
|
||||
Citation Citation
|
||||
}
|
||||
type IngestResult struct {
|
||||
DocsSeen, DocsIngested, ChunksCreated, Skipped int
|
||||
Errors []string
|
||||
}
|
||||
type Health struct {
|
||||
DBReachable, VectorExtension, MigrationsApplied bool
|
||||
Documents, Chunks int
|
||||
}
|
||||
|
||||
type Repository interface {
|
||||
Health(ctx context.Context) (Health, error)
|
||||
UpsertDocument(ctx context.Context, doc Document, chunks []Chunk) error
|
||||
Search(ctx context.Context, req SearchRequest, queryEmbedding []float32) ([]SearchResult, error)
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package kb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"ai-operator/internal/embedding"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type PostgresRepository struct{ pool *pgxpool.Pool }
|
||||
|
||||
func NewPostgresRepository(pool *pgxpool.Pool) *PostgresRepository {
|
||||
return &PostgresRepository{pool: pool}
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) Health(ctx context.Context) (Health, error) {
|
||||
h := Health{}
|
||||
if err := r.pool.Ping(ctx); err != nil {
|
||||
return h, err
|
||||
}
|
||||
h.DBReachable = true
|
||||
_ = r.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname='vector')`).Scan(&h.VectorExtension)
|
||||
var mig int
|
||||
_ = r.pool.QueryRow(ctx, `SELECT count(*) FROM information_schema.tables WHERE table_name IN ('schema_migrations','knowledge_documents','knowledge_chunks','knowledge_ingest_runs','knowledge_search_logs')`).Scan(&mig)
|
||||
h.MigrationsApplied = mig == 5
|
||||
_ = r.pool.QueryRow(ctx, `SELECT count(*) FROM knowledge_documents`).Scan(&h.Documents)
|
||||
_ = r.pool.QueryRow(ctx, `SELECT count(*) FROM knowledge_chunks`).Scan(&h.Chunks)
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) UpsertDocument(ctx context.Context, doc Document, chunks []Chunk) error {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
docID := stableUUID(doc.ExternalID)
|
||||
meta, _ := json.Marshal(doc.Metadata)
|
||||
_, err = tx.Exec(ctx, `INSERT INTO knowledge_documents(id,external_id,title,language,region_code,category,source_type,source_uri,source_hash,status,content,question,short_answer,full_answer,keywords,source,metadata,priority,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,now()) ON CONFLICT(external_id) DO UPDATE SET title=EXCLUDED.title, language=EXCLUDED.language, region_code=EXCLUDED.region_code, category=EXCLUDED.category, source_type=EXCLUDED.source_type, source_uri=EXCLUDED.source_uri, source_hash=EXCLUDED.source_hash, status=EXCLUDED.status, content=EXCLUDED.content, question=EXCLUDED.question, short_answer=EXCLUDED.short_answer, full_answer=EXCLUDED.full_answer, keywords=EXCLUDED.keywords, source=EXCLUDED.source, metadata=EXCLUDED.metadata, priority=EXCLUDED.priority, updated_at=now()`, docID, doc.ExternalID, doc.Title, doc.Language, doc.RegionCode, doc.Category, doc.SourceType, doc.SourceURI, doc.SourceHash, doc.Status, doc.Content, nullable(doc.Question), nullable(doc.ShortAnswer), nullable(doc.FullAnswer), nonNilKeywords(doc.Keywords), nullable(doc.Source), meta, doc.Priority)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `DELETE FROM knowledge_chunks WHERE document_id=$1`, docID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, ch := range chunks {
|
||||
chID := stableUUID(fmt.Sprintf("%s:%d:%s", doc.ExternalID, ch.ChunkIndex, ch.ContentHash))
|
||||
cm, _ := json.Marshal(ch.Metadata)
|
||||
_, err = tx.Exec(ctx, `INSERT INTO knowledge_chunks(id,document_id,chunk_index,language,region_code,title,content,content_hash,token_estimate,char_count,embedding,embedding_model,embedding_provider,metadata,tsv) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11::vector,$12,$13,$14,to_tsvector('simple',$7))`, chID, docID, ch.ChunkIndex, ch.Language, ch.RegionCode, ch.Title, ch.Content, ch.ContentHash, ch.TokenEstimate, ch.CharCount, embedding.VectorLiteral(ch.Embedding), ch.EmbeddingModel, ch.EmbeddingProvider, cm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) Search(ctx context.Context, req SearchRequest, queryEmbedding []float32) ([]SearchResult, error) {
|
||||
limit := req.Limit
|
||||
if limit <= 0 {
|
||||
limit = 5
|
||||
}
|
||||
if limit > 10 {
|
||||
limit = 10
|
||||
}
|
||||
langs := []string{req.Language}
|
||||
regions := []string{req.RegionCode}
|
||||
if req.IncludeGlobal {
|
||||
regions = append(regions, "global")
|
||||
}
|
||||
vec := embedding.VectorLiteral(queryEmbedding)
|
||||
rows, err := r.pool.Query(ctx, `WITH q AS (SELECT plainto_tsquery('simple', $1) query) SELECT d.id::text, c.id::text, c.title, c.content, c.language, c.region_code, d.category, coalesce(d.source_uri,''), coalesce(d.source,''), (1 - ((c.embedding <=> $2::vector)/2.0))::float8 AS vector_score, ts_rank_cd(c.tsv, q.query)::float8 AS keyword_score, CASE WHEN c.region_code=$3 THEN 1.0 ELSE 0.6 END AS region_boost FROM knowledge_chunks c JOIN knowledge_documents d ON d.id=c.document_id, q WHERE d.status='published' AND c.language=ANY($4) AND c.region_code=ANY($5) AND (d.valid_from IS NULL OR d.valid_from<=now()) AND (d.valid_to IS NULL OR d.valid_to>=now()) AND (c.tsv @@ q.query OR c.content ILIKE '%' || $1 || '%' OR c.embedding IS NOT NULL) ORDER BY ((0.65*(1-((c.embedding <=> $2::vector)/2.0))) + (0.25*ts_rank_cd(c.tsv,q.query)) + (0.10*CASE WHEN c.region_code=$3 THEN 1.0 ELSE 0.6 END)) DESC LIMIT $6`, sanitizeQuery(req.Query), vec, req.RegionCode, langs, regions, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []SearchResult
|
||||
for rows.Next() {
|
||||
var r0 SearchResult
|
||||
var source string
|
||||
var v, k, boost float64
|
||||
if err := rows.Scan(&r0.DocumentID, &r0.ChunkID, &r0.Title, &r0.Content, &r0.Language, &r0.RegionCode, &r0.Category, &r0.SourceURI, &source, &v, &k, &boost); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r0.SourceLanguage = r0.Language
|
||||
r0.VectorScore = v
|
||||
r0.KeywordScore = k
|
||||
r0.Score = 0.65*v + 0.25*k + 0.10*boost
|
||||
r0.Citation = Citation{DocumentTitle: r0.Title, SourceURI: r0.SourceURI, ChunkID: r0.ChunkID, RegionCode: r0.RegionCode, Language: r0.Language, Source: source}
|
||||
if r0.Score >= req.MinScore {
|
||||
out = append(out, r0)
|
||||
}
|
||||
}
|
||||
_ = r.logSearch(ctx, req, out, false)
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) logSearch(ctx context.Context, req SearchRequest, results []SearchResult, fallback bool) error {
|
||||
top := 0.0
|
||||
if len(results) > 0 {
|
||||
top = results[0].Score
|
||||
}
|
||||
_, err := r.pool.Exec(ctx, `INSERT INTO knowledge_search_logs(id,call_id,query,language,region_code,result_count,top_score,cross_language_fallback_used) VALUES($1,$2,$3,$4,$5,$6,$7,$8)`, stableUUID(fmt.Sprintf("%s:%s:%d", req.CallID, req.Query, len(results))), nullable(req.CallID), truncate(req.Query, 1000), req.Language, req.RegionCode, len(results), top, fallback)
|
||||
return err
|
||||
}
|
||||
|
||||
func nullable(s string) any {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return nil
|
||||
}
|
||||
return s
|
||||
}
|
||||
func sanitizeQuery(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r < 32 {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, truncate(strings.TrimSpace(s), 1000))
|
||||
}
|
||||
func truncate(s string, n int) string {
|
||||
r := []rune(s)
|
||||
if len(r) > n {
|
||||
return string(r[:n])
|
||||
}
|
||||
return s
|
||||
}
|
||||
func stableUUID(s string) string {
|
||||
h := ContentHash(s)
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32])
|
||||
}
|
||||
|
||||
var _ pgx.Tx
|
||||
|
||||
func nonNilKeywords(v []string) []string {
|
||||
if v == nil {
|
||||
return []string{}
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
package kb
|
||||
@@ -0,0 +1 @@
|
||||
package kb
|
||||
@@ -0,0 +1,126 @@
|
||||
package kb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"ai-operator/internal/config"
|
||||
"ai-operator/internal/embedding"
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
repo Repository
|
||||
embed embedding.Provider
|
||||
cfg config.KBConfig
|
||||
}
|
||||
|
||||
func NewService(repo Repository, embed embedding.Provider, cfg config.KBConfig) *Service {
|
||||
return &Service{repo: repo, embed: embed, cfg: cfg}
|
||||
}
|
||||
|
||||
type SearchResponse struct {
|
||||
OK bool
|
||||
ReasonCode, MessageKey string
|
||||
Results []SearchResult
|
||||
Citations []Citation
|
||||
CrossLanguageFallbackUsed bool
|
||||
}
|
||||
|
||||
func (s *Service) Search(ctx context.Context, req SearchRequest) (SearchResponse, error) {
|
||||
q := strings.TrimSpace(req.Query)
|
||||
if q == "" {
|
||||
return SearchResponse{OK: false, ReasonCode: "query_too_short", MessageKey: "knowledge.query_too_short"}, nil
|
||||
}
|
||||
if s.cfg.QueryMaxChars > 0 && len([]rune(q)) > s.cfg.QueryMaxChars {
|
||||
return SearchResponse{OK: false, ReasonCode: "query_too_long", MessageKey: "knowledge.query_too_long"}, nil
|
||||
}
|
||||
if req.Language != "ru" && req.Language != "kk" {
|
||||
return SearchResponse{OK: false, ReasonCode: "language_required", MessageKey: "knowledge.search_denied_language"}, nil
|
||||
}
|
||||
if req.RegionCode == "" {
|
||||
return SearchResponse{OK: false, ReasonCode: "region_required", MessageKey: "knowledge.search_denied_region"}, nil
|
||||
}
|
||||
if req.Limit <= 0 {
|
||||
req.Limit = s.cfg.DefaultLimit
|
||||
}
|
||||
if req.Limit <= 0 {
|
||||
req.Limit = 5
|
||||
}
|
||||
if s.cfg.MaxLimit > 0 && req.Limit > s.cfg.MaxLimit {
|
||||
req.Limit = s.cfg.MaxLimit
|
||||
}
|
||||
if req.MinScore == 0 {
|
||||
req.MinScore = s.cfg.MinScore
|
||||
}
|
||||
emb, err := s.embed.Embed(ctx, []string{q})
|
||||
if err != nil {
|
||||
return SearchResponse{OK: false, ReasonCode: "knowledge_base_unavailable", MessageKey: "knowledge.unavailable"}, err
|
||||
}
|
||||
req.Query = q
|
||||
req.IncludeGlobal = true
|
||||
results, err := s.repo.Search(ctx, req, emb[0])
|
||||
if err != nil {
|
||||
return SearchResponse{OK: false, ReasonCode: "knowledge_base_unavailable", MessageKey: "knowledge.unavailable"}, err
|
||||
}
|
||||
fallback := false
|
||||
if len(results) == 0 && req.Language == "kk" && (req.CrossLanguageFallback || s.cfg.CrossLanguageFallback) {
|
||||
ruReq := req
|
||||
ruReq.Language = "ru"
|
||||
results, err = s.repo.Search(ctx, ruReq, emb[0])
|
||||
if err != nil {
|
||||
return SearchResponse{OK: false, ReasonCode: "knowledge_base_unavailable", MessageKey: "knowledge.unavailable"}, err
|
||||
}
|
||||
if len(results) > 0 {
|
||||
fallback = true
|
||||
for i := range results {
|
||||
results[i].CrossLanguageFallback = true
|
||||
results[i].SourceLanguage = "ru"
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return SearchResponse{OK: false, ReasonCode: "no_relevant_knowledge", MessageKey: "knowledge.no_answer"}, nil
|
||||
}
|
||||
cites := make([]Citation, 0, len(results))
|
||||
for _, r := range results {
|
||||
cites = append(cites, r.Citation)
|
||||
}
|
||||
return SearchResponse{OK: true, ReasonCode: "ok", MessageKey: "knowledge.results_found", Results: results, Citations: cites, CrossLanguageFallbackUsed: fallback}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Health(ctx context.Context) (Health, error) { return s.repo.Health(ctx) }
|
||||
|
||||
func (s *Service) IngestJSONL(ctx context.Context, path string, allowDisabled bool) (IngestResult, error) {
|
||||
docs, parseErrs, err := LoadJSONLDocuments(ctx, path, allowDisabled)
|
||||
if err != nil {
|
||||
return IngestResult{}, err
|
||||
}
|
||||
res := IngestResult{DocsSeen: len(docs) + len(parseErrs), Errors: parseErrs}
|
||||
for _, doc := range docs {
|
||||
chunks := ChunkDocument(doc, DefaultChunkerConfig())
|
||||
texts := make([]string, len(chunks))
|
||||
for i, ch := range chunks {
|
||||
texts[i] = ch.Content
|
||||
}
|
||||
vecs, err := s.embed.Embed(ctx, texts)
|
||||
if err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("%s: embedding failed", doc.ExternalID))
|
||||
res.Skipped++
|
||||
continue
|
||||
}
|
||||
for i := range chunks {
|
||||
chunks[i].Embedding = vecs[i]
|
||||
chunks[i].EmbeddingModel = s.embed.Model()
|
||||
chunks[i].EmbeddingProvider = s.embed.ProviderName()
|
||||
}
|
||||
if err := s.repo.UpsertDocument(ctx, doc, chunks); err != nil {
|
||||
res.Errors = append(res.Errors, fmt.Sprintf("%s: %v", doc.ExternalID, err))
|
||||
res.Skipped++
|
||||
continue
|
||||
}
|
||||
res.DocsIngested++
|
||||
res.ChunksCreated += len(chunks)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
Reference in New Issue
Block a user