74 lines
2.7 KiB
Go
74 lines
2.7 KiB
Go
package kb
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
"ai-operator/internal/config"
|
|
"ai-operator/internal/embedding"
|
|
)
|
|
|
|
func TestParseJSONLTolerant(t *testing.T) {
|
|
line := `{"external_id":"id1","title":"T","language":"ru","region_code":"global","status":"published","content":"hello","source_hash":"abc","keywords":["a"],"extra":"x","metadata":{"m":1}}`
|
|
docs, errs, err := ParseJSONL(strings.NewReader(line), "test.jsonl", false)
|
|
if err != nil || len(errs) != 0 || len(docs) != 1 {
|
|
t.Fatalf("docs=%d errs=%v err=%v", len(docs), errs, err)
|
|
}
|
|
if docs[0].Metadata["extra"] != "x" || len(docs[0].Keywords) != 1 {
|
|
t.Fatalf("metadata/keywords not parsed: %+v", docs[0])
|
|
}
|
|
}
|
|
|
|
func TestParseJSONLMissingRequiredSkipped(t *testing.T) {
|
|
docs, errs, err := ParseJSONL(strings.NewReader(`{"title":"T"}`), "bad.jsonl", false)
|
|
if err != nil || len(docs) != 0 || len(errs) != 1 {
|
|
t.Fatalf("docs=%d errs=%v err=%v", len(docs), errs, err)
|
|
}
|
|
}
|
|
|
|
func TestChunkDocumentUTF8(t *testing.T) {
|
|
doc := Document{ExternalID: "x", Title: "Заголовок", Language: "ru", RegionCode: "global", Content: strings.Repeat("Қазақша текст. ", 200)}
|
|
chunks := ChunkDocument(doc, DefaultChunkerConfig())
|
|
if len(chunks) < 2 {
|
|
t.Fatalf("expected multiple chunks")
|
|
}
|
|
for _, ch := range chunks {
|
|
if ch.Content == "" || ch.CharCount == 0 || !strings.Contains(ch.Title, "Заголовок") {
|
|
t.Fatalf("bad chunk %+v", ch)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestServiceValidationNoAnswer(t *testing.T) {
|
|
s := NewService(fakeRepo{}, fakeEmbed{}, testKBConfig())
|
|
resp, _ := s.Search(context.Background(), SearchRequest{Query: "", Language: "ru", RegionCode: "global"})
|
|
if resp.ReasonCode != "query_too_short" {
|
|
t.Fatalf("bad empty response %+v", resp)
|
|
}
|
|
resp, _ = s.Search(context.Background(), SearchRequest{Query: "none", Language: "ru", RegionCode: "global"})
|
|
if resp.ReasonCode != "no_relevant_knowledge" {
|
|
t.Fatalf("bad no answer %+v", resp)
|
|
}
|
|
}
|
|
|
|
type fakeRepo struct{}
|
|
|
|
func (fakeRepo) Health(context.Context) (Health, error) { return Health{}, nil }
|
|
func (fakeRepo) UpsertDocument(context.Context, Document, []Chunk) error { return nil }
|
|
func (fakeRepo) Search(context.Context, SearchRequest, []float32) ([]SearchResult, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
type fakeEmbed struct{}
|
|
|
|
func (fakeEmbed) Embed(context.Context, []string) ([]embedding.Vector, error) {
|
|
return []embedding.Vector{make(embedding.Vector, 1536)}, nil
|
|
}
|
|
func (fakeEmbed) Dimensions() int { return 1536 }
|
|
func (fakeEmbed) Model() string { return "fake" }
|
|
func (fakeEmbed) ProviderName() string { return "fake" }
|
|
func testKBConfig() config.KBConfig {
|
|
return config.KBConfig{DefaultLimit: 5, MaxLimit: 10, MinScore: 0.2, QueryMaxChars: 1000, CrossLanguageFallback: true}
|
|
}
|