sync: migrate ai-operator to Gitea (2026-08-10)

internal/ai/pipeline/text_chunker.go

@@ -0,0 +1,120 @@
+package pipeline
+
+import (
+	"regexp"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+type TextChunker struct {
+	MinChars     int
+	MaxChars     int
+	StartAfter   int
+	FirstTimeout time.Duration
+	Punctuation  bool
+	buf          strings.Builder
+	firstAt      time.Time
+}
+
+func NewTextChunker(minChars, maxChars, startAfter int, firstTimeout time.Duration, punctuation bool) *TextChunker {
+	if minChars <= 0 {
+		minChars = 30
+	}
+	if maxChars < minChars {
+		maxChars = 160
+	}
+	if startAfter <= 0 {
+		startAfter = 60
+	}
+	if firstTimeout <= 0 {
+		firstTimeout = 1200 * time.Millisecond
+	}
+	return &TextChunker{MinChars: minChars, MaxChars: maxChars, StartAfter: startAfter, FirstTimeout: firstTimeout, Punctuation: punctuation}
+}
+
+func (c *TextChunker) Add(delta string, final bool) []string {
+	delta = normalizeForSpeech(delta)
+	if delta != "" {
+		if c.buf.Len() == 0 {
+			c.firstAt = time.Now()
+		}
+		c.buf.WriteString(delta)
+	}
+	var out []string
+	for {
+		s := strings.TrimSpace(c.buf.String())
+		if s == "" {
+			c.buf.Reset()
+			return out
+		}
+		emitAt := c.emitIndex(s, final)
+		if emitAt <= 0 {
+			return out
+		}
+		chunk := strings.TrimSpace(s[:emitAt])
+		out = append(out, chunk)
+		rest := strings.TrimSpace(s[emitAt:])
+		c.buf.Reset()
+		c.buf.WriteString(rest)
+		if rest == "" {
+			return out
+		}
+	}
+}
+
+func (c *TextChunker) Flush() []string { return c.Add("", true) }
+
+func (c *TextChunker) emitIndex(s string, final bool) int {
+	if final {
+		return len(s)
+	}
+	if c.Punctuation && utf8.RuneCountInString(s) >= c.MinChars {
+		if idx := lastSentenceBoundary(s, c.MaxChars); idx > 0 {
+			return idx
+		}
+	}
+	if utf8.RuneCountInString(s) >= c.MaxChars {
+		return byteIndexByRunes(s, c.MaxChars)
+	}
+	if utf8.RuneCountInString(s) >= c.StartAfter && time.Since(c.firstAt) >= c.FirstTimeout {
+		return len(s)
+	}
+	return 0
+}
+
+func lastSentenceBoundary(s string, maxRunes int) int {
+	limit := byteIndexByRunes(s, maxRunes)
+	if limit <= 0 || limit > len(s) {
+		limit = len(s)
+	}
+	last := -1
+	for i, r := range s[:limit] {
+		if r == '.' || r == '?' || r == '!' || r == '…' {
+			last = i + len(string(r))
+		}
+	}
+	return last
+}
+
+func byteIndexByRunes(s string, n int) int {
+	if n <= 0 {
+		return 0
+	}
+	i := 0
+	for pos := range s {
+		if i == n {
+			return pos
+		}
+		i++
+	}
+	return len(s)
+}
+
+var markdownPattern = regexp.MustCompile(`[*_` + "`" + `#>\[\]]`)
+
+func normalizeForSpeech(s string) string {
+	s = markdownPattern.ReplaceAllString(s, "")
+	s = strings.ReplaceAll(s, "\n", " ")
+	return s
+}