sync: migrate ai-operator to Gitea (2026-08-10)

internal/kb/chunker.go

@@ -0,0 +1,45 @@
+package kb
+
+import "unicode/utf8"
+
+type ChunkerConfig struct{ Target, Min, Max, Overlap int }
+
+func DefaultChunkerConfig() ChunkerConfig {
+	return ChunkerConfig{Target: 1200, Min: 200, Max: 2000, Overlap: 150}
+}
+func ChunkDocument(doc Document, cfg ChunkerConfig) []Chunk {
+	text := BuildChunkText(doc)
+	if cfg.Target <= 0 {
+		cfg = DefaultChunkerConfig()
+	}
+	rs := []rune(text)
+	if len(rs) == 0 {
+		return nil
+	}
+	var chunks []Chunk
+	start := 0
+	idx := 0
+	for start < len(rs) {
+		end := start + cfg.Target
+		if end > len(rs) {
+			end = len(rs)
+		}
+		if end-start > cfg.Max {
+			end = start + cfg.Max
+		}
+		content := string(rs[start:end])
+		if !utf8.ValidString(content) {
+			content = string([]rune(content))
+		}
+		chunks = append(chunks, Chunk{ChunkIndex: idx, Language: doc.Language, RegionCode: doc.RegionCode, Title: doc.Title, Content: content, ContentHash: ContentHash(content), CharCount: len([]rune(content)), TokenEstimate: len([]rune(content)) / 4, Metadata: map[string]any{"external_id": doc.ExternalID, "source_uri": doc.SourceURI}})
+		idx++
+		if end == len(rs) {
+			break
+		}
+		start = end - cfg.Overlap
+		if start < 0 {
+			start = 0
+		}
+	}
+	return chunks
+}