sync: migrate ai-operator to Gitea (2026-08-10)
This commit is contained in:
konturai-ops
@@ -0,0 +1,111 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"ai-operator/internal/ai/llm"
|
||||
"ai-operator/internal/ai/stt"
|
||||
"ai-operator/internal/ai/tts"
|
||||
"ai-operator/internal/media"
|
||||
)
|
||||
|
||||
type FakeSTT struct {
|
||||
events chan stt.Event
|
||||
audio int
|
||||
}
|
||||
|
||||
func NewFakeSTT() *FakeSTT { return &FakeSTT{events: make(chan stt.Event, 16)} }
|
||||
|
||||
func (f *FakeSTT) Start(ctx context.Context, req stt.StreamRequest) error { return nil }
|
||||
|
||||
func (f *FakeSTT) SendAudio(ctx context.Context, chunk media.AudioChunk) error {
|
||||
f.audio += len(chunk.Data)
|
||||
f.events <- stt.Event{Type: stt.EventPartialTranscript, Text: "Сколько стоит", At: time.Now().UTC()}
|
||||
f.events <- stt.Event{Type: stt.EventCommittedTranscript, Text: "Сколько стоит первичное подключение газа?", Language: "ru", At: time.Now().UTC()}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeSTT) Events() <-chan stt.Event { return f.events }
|
||||
|
||||
func (f *FakeSTT) Close(ctx context.Context) error {
|
||||
close(f.events)
|
||||
return nil
|
||||
}
|
||||
|
||||
type FakeLLM struct {
|
||||
Text string
|
||||
}
|
||||
|
||||
func (f FakeLLM) StreamGenerate(ctx context.Context, req llm.GenerateRequest) (<-chan llm.Event, error) {
|
||||
out := make(chan llm.Event, 8)
|
||||
text := f.Text
|
||||
if text == "" {
|
||||
text = "Первичное подключение газа к газовому оборудованию осуществляется бесплатно."
|
||||
}
|
||||
go func() {
|
||||
defer close(out)
|
||||
parts := strings.SplitAfter(text, " ")
|
||||
for _, p := range parts {
|
||||
if p != "" {
|
||||
out <- llm.Event{Type: llm.EventTextDelta, Text: p}
|
||||
}
|
||||
}
|
||||
out <- llm.Event{Type: llm.EventDone}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
type FakeTTSFactory struct {
|
||||
mu sync.Mutex
|
||||
Texts []string
|
||||
Starts int
|
||||
Cancels int
|
||||
}
|
||||
|
||||
func (f *FakeTTSFactory) New(language string) tts.StreamingTTS {
|
||||
return &FakeTTS{factory: f, audio: make(chan tts.AudioChunk, 16)}
|
||||
}
|
||||
|
||||
type FakeTTS struct {
|
||||
factory *FakeTTSFactory
|
||||
audio chan tts.AudioChunk
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (f *FakeTTS) Start(ctx context.Context, req tts.TTSStreamRequest) error {
|
||||
f.factory.mu.Lock()
|
||||
f.factory.Starts++
|
||||
f.factory.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeTTS) SendText(ctx context.Context, text string, flush bool) error {
|
||||
f.factory.mu.Lock()
|
||||
if text != "" {
|
||||
f.factory.Texts = append(f.factory.Texts, text)
|
||||
}
|
||||
f.factory.mu.Unlock()
|
||||
if text != "" {
|
||||
select {
|
||||
case f.audio <- tts.AudioChunk{Data: []byte{0, 1, 0, 1}, Timestamp: time.Now().UTC()}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if flush && text == "" {
|
||||
f.Close(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *FakeTTS) Audio() <-chan tts.AudioChunk { return f.audio }
|
||||
|
||||
func (f *FakeTTS) Close(ctx context.Context) error {
|
||||
if !f.closed {
|
||||
close(f.audio)
|
||||
f.closed = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package pipeline
|
||||
|
||||
import "time"
|
||||
|
||||
type LatencyMetrics struct {
|
||||
CallID string
|
||||
TurnID string
|
||||
TurnStartedAt time.Time
|
||||
STTFirstPartialAt time.Time
|
||||
STTCommittedAt time.Time
|
||||
LLMFirstTokenAt time.Time
|
||||
TTSFirstAudioAt time.Time
|
||||
FirstAudibleAudioAt time.Time
|
||||
CompletedAt time.Time
|
||||
STTChars int
|
||||
LLMChars int
|
||||
TTSAudioBytes int
|
||||
Interruptions int
|
||||
}
|
||||
|
||||
func (m LatencyMetrics) Snapshot() map[string]any {
|
||||
return map[string]any{
|
||||
"call_id": m.CallID,
|
||||
"turn_id": m.TurnID,
|
||||
"stt_first_partial_ms": sinceMS(m.TurnStartedAt, m.STTFirstPartialAt),
|
||||
"stt_committed_ms": sinceMS(m.TurnStartedAt, m.STTCommittedAt),
|
||||
"llm_first_token_ms": sinceMS(m.STTCommittedAt, m.LLMFirstTokenAt),
|
||||
"tts_first_audio_ms": sinceMS(m.LLMFirstTokenAt, m.TTSFirstAudioAt),
|
||||
"first_audible_audio_ms": sinceMS(m.STTCommittedAt, m.FirstAudibleAudioAt),
|
||||
"total_turn_ms": sinceMS(m.TurnStartedAt, m.CompletedAt),
|
||||
"stt_chars": m.STTChars,
|
||||
"llm_chars": m.LLMChars,
|
||||
"tts_audio_bytes": m.TTSAudioBytes,
|
||||
"interruptions_count": m.Interruptions,
|
||||
}
|
||||
}
|
||||
|
||||
func sinceMS(start, end time.Time) any {
|
||||
if start.IsZero() || end.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return end.Sub(start).Milliseconds()
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"ai-operator/internal/ai/tts"
|
||||
"ai-operator/internal/config"
|
||||
)
|
||||
|
||||
func NaturalizeForVoice(text string, language string, cfg config.NaturalnessConfig) string {
|
||||
return tts.NaturalizeForVoice(text, language, cfg)
|
||||
}
|
||||
|
||||
func RemoveAudioTags(text string) string {
|
||||
return tts.RemoveAudioTags(text)
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"ai-operator/internal/ai"
|
||||
"ai-operator/internal/config"
|
||||
"ai-operator/internal/media"
|
||||
)
|
||||
|
||||
func TestTextChunker(t *testing.T) {
|
||||
c := NewTextChunker(10, 80, 20, time.Millisecond, true)
|
||||
chunks := c.Add("Первое предложение. Второе", false)
|
||||
if len(chunks) != 1 || chunks[0] != "Первое предложение." {
|
||||
t.Fatalf("chunks=%v", chunks)
|
||||
}
|
||||
flush := c.Flush()
|
||||
if len(flush) != 1 || flush[0] != "Второе" {
|
||||
t.Fatalf("flush=%v", flush)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNaturalizer(t *testing.T) {
|
||||
cfg := config.NaturalnessConfig{Enabled: true, AudioTagsEnabled: true, AllowNonverbalTags: true, MaxAudioTagsPerResponse: 2}
|
||||
got := NaturalizeForVoice("Здравствуйте, меня зовут Жанна. Чем могу помочь?", "ru", cfg)
|
||||
if got == "" || got == "Здравствуйте, меня зовут Жанна. Чем могу помочь?" {
|
||||
t.Fatalf("not naturalized: %q", got)
|
||||
}
|
||||
if stripped := RemoveAudioTags(got); stripped == "" || stripped == got {
|
||||
t.Fatalf("tags not stripped: %q", stripped)
|
||||
}
|
||||
cfg.AllowCough = false
|
||||
if got := NaturalizeForVoice("Первичное подключение газа бесплатно.", "ru", cfg); containsRune(got, "cough") {
|
||||
t.Fatalf("cough added unexpectedly: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamingProviderFakeEndToEnd(t *testing.T) {
|
||||
cfg := config.Config{
|
||||
STT: config.STTConfig{Model: "scribe_realtime_v2", SampleRate: 16000, InputFormat: "pcm_16000"},
|
||||
LLM: config.LLMConfig{StreamChunkMinChars: 10, StreamChunkMaxChars: 160, FirstChunkTimeoutMS: 1, MaxOutputTokens: 100, Temperature: 0.2},
|
||||
Eleven: config.ElevenLabsConfig{VoiceIDRU: "voice", TTSModelID: "eleven_flash_v2_5", TTSOutputFormat: "pcm_16000", TTSSampleRate: 16000},
|
||||
Natural: config.NaturalnessConfig{Enabled: false},
|
||||
Pipeline: config.PipelineConfig{InitialGreeting: false, BargeIn: true, MaxTurnSeconds: 2, TTSStartAfterChars: 20, TTSStartAfterPunctuation: true},
|
||||
}
|
||||
factory := &FakeTTSFactory{}
|
||||
p := NewStreamingProviderWithDeps(cfg, nil, NewFakeSTT(), FakeLLM{}, factory.New)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := p.StartSession(ctx, ai.VoiceSessionConfig{CallID: "c1", SystemPrompt: "prompt"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := p.SendAudio(ctx, media.AudioChunk{Data: []byte{0, 0}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tool := false
|
||||
audio := false
|
||||
for !audio {
|
||||
select {
|
||||
case ev := <-p.Events():
|
||||
if ev.Type == ai.VoiceEventToolCall {
|
||||
tool = true
|
||||
_ = p.SendToolResult(ctx, ai.ToolResult{CallID: ev.CallID, ToolCallID: ev.ToolCall.ID, Result: map[string]any{"answer_text": "Первичное подключение бесплатно."}})
|
||||
}
|
||||
if ev.Type == ai.VoiceEventAssistantAudioDelta {
|
||||
audio = true
|
||||
}
|
||||
case <-ctx.Done():
|
||||
t.Fatal("timeout")
|
||||
}
|
||||
}
|
||||
if !tool || !audio || len(factory.Texts) == 0 {
|
||||
t.Fatalf("tool=%t audio=%t texts=%v", tool, audio, factory.Texts)
|
||||
}
|
||||
}
|
||||
|
||||
func containsRune(s, sub string) bool {
|
||||
for i := 0; i+len(sub) <= len(s); i++ {
|
||||
if s[i:i+len(sub)] == sub {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"ai-operator/internal/ai"
|
||||
"ai-operator/internal/ai/llm"
|
||||
"ai-operator/internal/ai/stt"
|
||||
"ai-operator/internal/ai/tts"
|
||||
"ai-operator/internal/config"
|
||||
"ai-operator/internal/media"
|
||||
"ai-operator/internal/tools"
|
||||
)
|
||||
|
||||
type TTSFactory func(language string) tts.StreamingTTS
|
||||
|
||||
type StreamingProvider struct {
|
||||
cfg config.Config
|
||||
logger *slog.Logger
|
||||
stt stt.StreamingProvider
|
||||
llm llm.StreamingLLM
|
||||
ttsFactory TTSFactory
|
||||
|
||||
events chan ai.VoiceEvent
|
||||
mu sync.Mutex
|
||||
sttMu sync.Mutex
|
||||
stats ai.VoiceProviderStats
|
||||
callID string
|
||||
prompt string
|
||||
closed bool
|
||||
sttReq stt.StreamRequest
|
||||
|
||||
pending map[string]chan ai.ToolResult
|
||||
turnSeq int
|
||||
ttsCancel context.CancelFunc
|
||||
sttRestartWindow time.Time
|
||||
sttRestartCount int
|
||||
sttDisabledUntil time.Time
|
||||
}
|
||||
|
||||
func NewStreamingProvider(cfg config.Config, logger *slog.Logger) *StreamingProvider {
|
||||
return NewStreamingProviderWithDeps(cfg, logger, stt.NewElevenLabsRealtime(cfg), llm.NewOpenAIStreaming(cfg), func(language string) tts.StreamingTTS {
|
||||
return tts.NewElevenLabsWS(cfg)
|
||||
})
|
||||
}
|
||||
|
||||
func NewStreamingProviderWithDeps(cfg config.Config, logger *slog.Logger, sttProvider stt.StreamingProvider, llmProvider llm.StreamingLLM, ttsFactory TTSFactory) *StreamingProvider {
|
||||
return &StreamingProvider{cfg: cfg, logger: logger, stt: sttProvider, llm: llmProvider, ttsFactory: ttsFactory, events: make(chan ai.VoiceEvent, 128), pending: map[string]chan ai.ToolResult{}}
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) StartSession(ctx context.Context, sc ai.VoiceSessionConfig) error {
|
||||
if p.stt == nil || p.llm == nil || p.ttsFactory == nil {
|
||||
return errors.New("pipeline streaming provider dependencies are not configured")
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.events = make(chan ai.VoiceEvent, 128)
|
||||
p.pending = map[string]chan ai.ToolResult{}
|
||||
p.closed = false
|
||||
p.callID = sc.CallID
|
||||
p.prompt = sc.SystemPrompt
|
||||
p.stats = ai.VoiceProviderStats{}
|
||||
now := time.Now().UTC()
|
||||
p.stats.StartedAt = &now
|
||||
p.mu.Unlock()
|
||||
req := stt.StreamRequest{CallID: sc.CallID, Model: p.cfg.STT.Model, LanguageCode: p.cfg.STT.LanguageCode, LanguageAuto: p.cfg.STT.LanguageAuto, SampleRate: p.cfg.STT.SampleRate, InputFormat: p.cfg.STT.InputFormat}
|
||||
p.sttReq = req
|
||||
if err := p.stt.Start(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventSessionStarted, CallID: sc.CallID, At: now})
|
||||
go p.sttLoop()
|
||||
if p.cfg.Pipeline.InitialGreeting {
|
||||
go p.speakText(context.Background(), "greeting", "ru", "Здравствуйте, меня зовут Жанна. Я AI-оператор QazAimaqGas. Чем могу помочь?", nil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) SendAudio(ctx context.Context, chunk media.AudioChunk) error {
|
||||
p.mu.Lock()
|
||||
closed := p.closed
|
||||
sttDisabled := time.Now().Before(p.sttDisabledUntil)
|
||||
p.stats.InputAudioFrames++
|
||||
p.stats.InputAudioBytes += int64(len(chunk.Data))
|
||||
p.mu.Unlock()
|
||||
if closed || sttDisabled {
|
||||
return nil
|
||||
}
|
||||
err := p.stt.SendAudio(ctx, chunk)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if isClosedWebSocketError(err) {
|
||||
if restartErr := p.restartSTT(ctx); restartErr != nil {
|
||||
p.log("pipeline stt restart failed", "channel_id", p.callID, "error", restartErr)
|
||||
return nil
|
||||
}
|
||||
if retryErr := p.stt.SendAudio(ctx, chunk); retryErr != nil {
|
||||
p.log("pipeline stt audio send failed after restart", "channel_id", p.callID, "error", retryErr)
|
||||
p.disableSTTBriefly()
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) SendToolResult(ctx context.Context, result ai.ToolResult) error {
|
||||
p.mu.Lock()
|
||||
ch := p.pending[result.ToolCallID]
|
||||
p.mu.Unlock()
|
||||
if ch == nil {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case ch <- result:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) Close(ctx context.Context) error {
|
||||
p.cancelTTS()
|
||||
if p.stt != nil {
|
||||
_ = p.stt.Close(ctx)
|
||||
}
|
||||
p.mu.Lock()
|
||||
now := time.Now().UTC()
|
||||
p.stats.ClosedAt = &now
|
||||
p.mu.Unlock()
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventClosed, CallID: p.callID, At: now})
|
||||
p.closeEvents()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) Events() <-chan ai.VoiceEvent { return p.events }
|
||||
|
||||
func (p *StreamingProvider) Stats() ai.VoiceProviderStats {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
return p.stats
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) sttLoop() {
|
||||
for ev := range p.stt.Events() {
|
||||
switch ev.Type {
|
||||
case stt.EventPartialTranscript:
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventUserTranscriptDelta, CallID: p.callID, Text: ev.Text, At: ev.At, Metadata: map[string]any{"language": ev.Language}})
|
||||
if p.cfg.Pipeline.BargeIn && strings.TrimSpace(ev.Text) != "" {
|
||||
p.interrupt()
|
||||
}
|
||||
case stt.EventSpeechStarted:
|
||||
if p.cfg.Pipeline.BargeIn {
|
||||
p.interrupt()
|
||||
}
|
||||
case stt.EventFinalTranscript, stt.EventCommittedTranscript:
|
||||
text := strings.TrimSpace(ev.Text)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventUserTranscriptDone, CallID: p.callID, Text: text, At: ev.At, Metadata: map[string]any{"language": ev.Language}})
|
||||
go p.handleTurn(text, ev.Language)
|
||||
case stt.EventError:
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventError, CallID: p.callID, Error: ev.Error, At: ev.At})
|
||||
case stt.EventClosed:
|
||||
if ev.Error != "" {
|
||||
p.log("pipeline stt websocket closed", "channel_id", p.callID, "reason", ev.Error)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) handleTurn(userText, language string) {
|
||||
if language == "" {
|
||||
language = inferLanguage(userText)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.turnSeq++
|
||||
turnID := fmt.Sprintf("turn-%d", p.turnSeq)
|
||||
p.mu.Unlock()
|
||||
metrics := &LatencyMetrics{CallID: p.callID, TurnID: turnID, TurnStartedAt: time.Now().UTC(), STTCommittedAt: time.Now().UTC(), STTChars: len([]rune(userText))}
|
||||
|
||||
toolID := turnID + "-kb"
|
||||
ch := make(chan ai.ToolResult, 1)
|
||||
p.mu.Lock()
|
||||
p.pending[toolID] = ch
|
||||
p.mu.Unlock()
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventToolCall, CallID: p.callID, ToolCall: &ai.ToolCall{ID: toolID, Name: tools.SearchKnowledgeBase, Arguments: map[string]any{"query": userText, "limit": 5}}, At: time.Now().UTC()})
|
||||
var toolResult ai.ToolResult
|
||||
select {
|
||||
case toolResult = <-ch:
|
||||
case <-time.After(time.Duration(p.cfg.Pipeline.MaxTurnSeconds) * time.Second):
|
||||
toolResult = ai.ToolResult{CallID: p.callID, ToolCallID: toolID, Error: "tool_timeout", Result: map[string]any{"message": "База знаний временно недоступна."}}
|
||||
}
|
||||
p.mu.Lock()
|
||||
delete(p.pending, toolID)
|
||||
p.mu.Unlock()
|
||||
answer := answerFromToolResult(toolResult, language)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(p.cfg.Pipeline.MaxTurnSeconds)*time.Second)
|
||||
defer cancel()
|
||||
llmEvents, err := p.llm.StreamGenerate(ctx, llm.GenerateRequest{CallID: p.callID, SystemPrompt: p.prompt, Messages: []llm.Message{{Role: "user", Content: llm.BuildAnswerPrompt(userText, answer)}}, MaxTokens: p.cfg.LLM.MaxOutputTokens, Temperature: p.cfg.LLM.Temperature})
|
||||
if err != nil {
|
||||
_ = p.speakText(ctx, turnID, language, answer, metrics)
|
||||
return
|
||||
}
|
||||
_ = p.speakLLMStream(ctx, turnID, language, llmEvents, metrics)
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) speakLLMStream(ctx context.Context, turnID, language string, events <-chan llm.Event, metrics *LatencyMetrics) error {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
p.setTTSCancel(cancel)
|
||||
defer p.clearTTSCancel(cancel)
|
||||
ttsStream := p.ttsFactory(language)
|
||||
if err := ttsStream.Start(ctx, tts.TTSStreamRequest{CallID: p.callID, Language: language, VoiceID: p.voiceID(language), ModelID: p.cfg.Eleven.TTSModelID, OutputFormat: p.cfg.Eleven.TTSOutputFormat, SampleRate: p.cfg.Eleven.TTSSampleRate}); err != nil {
|
||||
return err
|
||||
}
|
||||
defer ttsStream.Close(context.Background())
|
||||
audioDone := p.forwardTTSAudio(ctx, ttsStream, metrics)
|
||||
chunker := NewTextChunker(p.cfg.LLM.StreamChunkMinChars, p.cfg.LLM.StreamChunkMaxChars, p.cfg.Pipeline.TTSStartAfterChars, time.Duration(p.cfg.LLM.FirstChunkTimeoutMS)*time.Millisecond, p.cfg.Pipeline.TTSStartAfterPunctuation)
|
||||
for ev := range events {
|
||||
switch ev.Type {
|
||||
case llm.EventTextDelta:
|
||||
if metrics.LLMFirstTokenAt.IsZero() {
|
||||
metrics.LLMFirstTokenAt = time.Now().UTC()
|
||||
}
|
||||
metrics.LLMChars += len([]rune(ev.Text))
|
||||
for _, chunk := range chunker.Add(ev.Text, false) {
|
||||
if err := ttsStream.SendText(ctx, NaturalizeForVoice(chunk, language, p.cfg.Natural), true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
case llm.EventError:
|
||||
return errors.New(ev.Error)
|
||||
case llm.EventDone:
|
||||
for _, chunk := range chunker.Flush() {
|
||||
if err := ttsStream.SendText(ctx, NaturalizeForVoice(chunk, language, p.cfg.Natural), true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_ = ttsStream.SendText(ctx, "", true)
|
||||
select {
|
||||
case <-audioDone:
|
||||
case <-time.After(ttsFinalDrain(p.cfg.Pipeline.MaxTurnSeconds)):
|
||||
case <-ctx.Done():
|
||||
}
|
||||
metrics.CompletedAt = time.Now().UTC()
|
||||
p.log("pipeline turn latency", "metrics", metrics.Snapshot())
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) speakText(ctx context.Context, turnID, language, text string, metrics *LatencyMetrics) error {
|
||||
ch := make(chan llm.Event, 4)
|
||||
ch <- llm.Event{Type: llm.EventTextDelta, Text: text}
|
||||
ch <- llm.Event{Type: llm.EventDone}
|
||||
close(ch)
|
||||
if metrics == nil {
|
||||
metrics = &LatencyMetrics{CallID: p.callID, TurnID: turnID, TurnStartedAt: time.Now().UTC(), STTCommittedAt: time.Now().UTC()}
|
||||
}
|
||||
return p.speakLLMStream(ctx, turnID, language, ch, metrics)
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) forwardTTSAudio(ctx context.Context, stream tts.StreamingTTS, metrics *LatencyMetrics) <-chan struct{} {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for audio := range stream.Audio() {
|
||||
pcm := audio.Data
|
||||
if len(pcm)%2 != 0 {
|
||||
p.log("trimmed odd trailing byte from tts pcm", "channel_id", p.callID, "bytes", len(pcm))
|
||||
pcm = pcm[:len(pcm)-1]
|
||||
}
|
||||
if len(pcm) == 0 {
|
||||
continue
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
if metrics.TTSFirstAudioAt.IsZero() {
|
||||
metrics.TTSFirstAudioAt = now
|
||||
metrics.FirstAudibleAudioAt = now
|
||||
}
|
||||
metrics.TTSAudioBytes += len(pcm)
|
||||
p.mu.Lock()
|
||||
p.stats.OutputAudioFrames++
|
||||
p.stats.OutputAudioBytes += int64(len(pcm))
|
||||
p.mu.Unlock()
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventAssistantAudioDelta, CallID: p.callID, Audio: pcm, At: now})
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
}
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventAssistantAudioDone, CallID: p.callID, At: time.Now().UTC()})
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) restartSTT(ctx context.Context) error {
|
||||
p.sttMu.Lock()
|
||||
defer p.sttMu.Unlock()
|
||||
now := time.Now()
|
||||
p.mu.Lock()
|
||||
if now.Before(p.sttDisabledUntil) {
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
if p.sttRestartWindow.IsZero() || now.Sub(p.sttRestartWindow) > 5*time.Second {
|
||||
p.sttRestartWindow = now
|
||||
p.sttRestartCount = 0
|
||||
}
|
||||
p.sttRestartCount++
|
||||
if p.sttRestartCount > 3 {
|
||||
p.sttDisabledUntil = now.Add(10 * time.Second)
|
||||
p.mu.Unlock()
|
||||
p.log("pipeline stt restart suppressed", "channel_id", p.callID, "cooldown", "10s")
|
||||
return errors.New("stt restart suppressed")
|
||||
}
|
||||
p.mu.Unlock()
|
||||
p.mu.Lock()
|
||||
if p.closed {
|
||||
p.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
req := p.sttReq
|
||||
p.mu.Unlock()
|
||||
_ = p.stt.Close(context.Background())
|
||||
if err := p.stt.Start(ctx, req); err != nil {
|
||||
return err
|
||||
}
|
||||
go p.sttLoop()
|
||||
p.log("pipeline stt websocket restarted", "channel_id", p.callID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) disableSTTBriefly() {
|
||||
p.mu.Lock()
|
||||
p.sttDisabledUntil = time.Now().Add(10 * time.Second)
|
||||
p.mu.Unlock()
|
||||
p.log("pipeline stt disabled after repeated websocket close", "channel_id", p.callID, "cooldown", "10s")
|
||||
}
|
||||
|
||||
func isClosedWebSocketError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
s := strings.ToLower(err.Error())
|
||||
return strings.Contains(s, "websocket: close") || strings.Contains(s, "close sent") || strings.Contains(s, "not connected") || strings.Contains(s, "closed network connection")
|
||||
}
|
||||
|
||||
func ttsFinalDrain(maxTurnSeconds int) time.Duration {
|
||||
if maxTurnSeconds <= 0 || maxTurnSeconds > 8 {
|
||||
return 8 * time.Second
|
||||
}
|
||||
return time.Duration(maxTurnSeconds) * time.Second
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) interrupt() {
|
||||
p.cancelTTS()
|
||||
p.emit(ai.VoiceEvent{Type: ai.VoiceEventInterruption, CallID: p.callID, At: time.Now().UTC()})
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) setTTSCancel(cancel context.CancelFunc) {
|
||||
p.mu.Lock()
|
||||
p.ttsCancel = cancel
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) clearTTSCancel(cancel context.CancelFunc) {
|
||||
p.mu.Lock()
|
||||
if fmt.Sprintf("%p", p.ttsCancel) == fmt.Sprintf("%p", cancel) {
|
||||
p.ttsCancel = nil
|
||||
}
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) cancelTTS() {
|
||||
p.mu.Lock()
|
||||
cancel := p.ttsCancel
|
||||
p.ttsCancel = nil
|
||||
p.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) voiceID(language string) string {
|
||||
if language == "kk" && p.cfg.Eleven.VoiceIDKK != "" {
|
||||
return p.cfg.Eleven.VoiceIDKK
|
||||
}
|
||||
if language == "kk" && p.cfg.Eleven.VoiceIDKK == "" {
|
||||
p.log("kk voice id missing, using ru voice fallback")
|
||||
}
|
||||
return p.cfg.Eleven.VoiceIDRU
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) emit(ev ai.VoiceEvent) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.stats.EventsSent++
|
||||
if p.closed {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case p.events <- ev:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) closeEvents() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if !p.closed {
|
||||
close(p.events)
|
||||
p.closed = true
|
||||
}
|
||||
}
|
||||
|
||||
func (p *StreamingProvider) log(msg string, args ...any) {
|
||||
if p.logger != nil {
|
||||
p.logger.Info(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
func answerFromToolResult(result ai.ToolResult, language string) string {
|
||||
if result.Result == nil {
|
||||
return noAnswer(language)
|
||||
}
|
||||
b, _ := json.Marshal(result.Result)
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(b, &m)
|
||||
for _, key := range []string{"answer_text", "message"} {
|
||||
if s, ok := m[key].(string); ok && strings.TrimSpace(s) != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
if result.Error == "region_required_for_question" {
|
||||
if language == "kk" {
|
||||
return "Қалаңызды немесе облысыңызды нақтылап жіберіңізші."
|
||||
}
|
||||
return "Подскажите, пожалуйста, ваш город или область?"
|
||||
}
|
||||
return noAnswer(language)
|
||||
}
|
||||
|
||||
func noAnswer(language string) string {
|
||||
if language == "kk" {
|
||||
return "Бұл сұрақ бойынша білім базасында нақты ақпарат жоқ."
|
||||
}
|
||||
return "В базе знаний нет точной информации по этому вопросу."
|
||||
}
|
||||
|
||||
func inferLanguage(text string) string {
|
||||
for _, r := range text {
|
||||
if strings.ContainsRune("әғқңөұүіһӘҒҚҢӨҰҮІҺ", r) {
|
||||
return "kk"
|
||||
}
|
||||
}
|
||||
return "ru"
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
package pipeline
|
||||
|
||||
// Streaming session orchestration lives in streaming_provider.go. This file is
|
||||
// intentionally kept as the package boundary for future per-call session state.
|
||||
@@ -0,0 +1,120 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type TextChunker struct {
|
||||
MinChars int
|
||||
MaxChars int
|
||||
StartAfter int
|
||||
FirstTimeout time.Duration
|
||||
Punctuation bool
|
||||
buf strings.Builder
|
||||
firstAt time.Time
|
||||
}
|
||||
|
||||
func NewTextChunker(minChars, maxChars, startAfter int, firstTimeout time.Duration, punctuation bool) *TextChunker {
|
||||
if minChars <= 0 {
|
||||
minChars = 30
|
||||
}
|
||||
if maxChars < minChars {
|
||||
maxChars = 160
|
||||
}
|
||||
if startAfter <= 0 {
|
||||
startAfter = 60
|
||||
}
|
||||
if firstTimeout <= 0 {
|
||||
firstTimeout = 1200 * time.Millisecond
|
||||
}
|
||||
return &TextChunker{MinChars: minChars, MaxChars: maxChars, StartAfter: startAfter, FirstTimeout: firstTimeout, Punctuation: punctuation}
|
||||
}
|
||||
|
||||
func (c *TextChunker) Add(delta string, final bool) []string {
|
||||
delta = normalizeForSpeech(delta)
|
||||
if delta != "" {
|
||||
if c.buf.Len() == 0 {
|
||||
c.firstAt = time.Now()
|
||||
}
|
||||
c.buf.WriteString(delta)
|
||||
}
|
||||
var out []string
|
||||
for {
|
||||
s := strings.TrimSpace(c.buf.String())
|
||||
if s == "" {
|
||||
c.buf.Reset()
|
||||
return out
|
||||
}
|
||||
emitAt := c.emitIndex(s, final)
|
||||
if emitAt <= 0 {
|
||||
return out
|
||||
}
|
||||
chunk := strings.TrimSpace(s[:emitAt])
|
||||
out = append(out, chunk)
|
||||
rest := strings.TrimSpace(s[emitAt:])
|
||||
c.buf.Reset()
|
||||
c.buf.WriteString(rest)
|
||||
if rest == "" {
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *TextChunker) Flush() []string { return c.Add("", true) }
|
||||
|
||||
func (c *TextChunker) emitIndex(s string, final bool) int {
|
||||
if final {
|
||||
return len(s)
|
||||
}
|
||||
if c.Punctuation && utf8.RuneCountInString(s) >= c.MinChars {
|
||||
if idx := lastSentenceBoundary(s, c.MaxChars); idx > 0 {
|
||||
return idx
|
||||
}
|
||||
}
|
||||
if utf8.RuneCountInString(s) >= c.MaxChars {
|
||||
return byteIndexByRunes(s, c.MaxChars)
|
||||
}
|
||||
if utf8.RuneCountInString(s) >= c.StartAfter && time.Since(c.firstAt) >= c.FirstTimeout {
|
||||
return len(s)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func lastSentenceBoundary(s string, maxRunes int) int {
|
||||
limit := byteIndexByRunes(s, maxRunes)
|
||||
if limit <= 0 || limit > len(s) {
|
||||
limit = len(s)
|
||||
}
|
||||
last := -1
|
||||
for i, r := range s[:limit] {
|
||||
if r == '.' || r == '?' || r == '!' || r == '…' {
|
||||
last = i + len(string(r))
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
|
||||
func byteIndexByRunes(s string, n int) int {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
i := 0
|
||||
for pos := range s {
|
||||
if i == n {
|
||||
return pos
|
||||
}
|
||||
i++
|
||||
}
|
||||
return len(s)
|
||||
}
|
||||
|
||||
var markdownPattern = regexp.MustCompile(`[*_` + "`" + `#>\[\]]`)
|
||||
|
||||
func normalizeForSpeech(s string) string {
|
||||
s = markdownPattern.ReplaceAllString(s, "")
|
||||
s = strings.ReplaceAll(s, "\n", " ")
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user