46 lines
1.1 KiB
Go
46 lines
1.1 KiB
Go
package kb
|
|
|
|
import "unicode/utf8"
|
|
|
|
type ChunkerConfig struct{ Target, Min, Max, Overlap int }
|
|
|
|
func DefaultChunkerConfig() ChunkerConfig {
|
|
return ChunkerConfig{Target: 1200, Min: 200, Max: 2000, Overlap: 150}
|
|
}
|
|
func ChunkDocument(doc Document, cfg ChunkerConfig) []Chunk {
|
|
text := BuildChunkText(doc)
|
|
if cfg.Target <= 0 {
|
|
cfg = DefaultChunkerConfig()
|
|
}
|
|
rs := []rune(text)
|
|
if len(rs) == 0 {
|
|
return nil
|
|
}
|
|
var chunks []Chunk
|
|
start := 0
|
|
idx := 0
|
|
for start < len(rs) {
|
|
end := start + cfg.Target
|
|
if end > len(rs) {
|
|
end = len(rs)
|
|
}
|
|
if end-start > cfg.Max {
|
|
end = start + cfg.Max
|
|
}
|
|
content := string(rs[start:end])
|
|
if !utf8.ValidString(content) {
|
|
content = string([]rune(content))
|
|
}
|
|
chunks = append(chunks, Chunk{ChunkIndex: idx, Language: doc.Language, RegionCode: doc.RegionCode, Title: doc.Title, Content: content, ContentHash: ContentHash(content), CharCount: len([]rune(content)), TokenEstimate: len([]rune(content)) / 4, Metadata: map[string]any{"external_id": doc.ExternalID, "source_uri": doc.SourceURI}})
|
|
idx++
|
|
if end == len(rs) {
|
|
break
|
|
}
|
|
start = end - cfg.Overlap
|
|
if start < 0 {
|
|
start = 0
|
|
}
|
|
}
|
|
return chunks
|
|
}
|