sync: migrate ai-operator to Gitea (2026-08-10)

internal/ai/openai/realtime/event_builder.go

@@ -0,0 +1,44 @@
+package realtime
+
+import (
+	"ai-operator/internal/ai"
+	"ai-operator/internal/audio"
+	"ai-operator/internal/config"
+	"encoding/json"
+)
+
+func BuildSessionUpdate(cfg config.OpenAIConfig, instructions string) ([]byte, error) {
+	session := map[string]any{"type": "realtime", "model": cfg.RealtimeModel, "instructions": instructions, "audio": map[string]any{"input": map[string]any{"format": map[string]any{"type": "audio/pcm", "rate": cfg.RealtimeInputSampleRate}, "turn_detection": map[string]any{"type": cfg.RealtimeTurnDetection}}, "output": map[string]any{"format": map[string]any{"type": "audio/pcm", "rate": cfg.RealtimeOutputSampleRate}}}, "reasoning": map[string]any{"effort": cfg.RealtimeReasoningEffort}}
+	if cfg.RealtimeVoice != "" {
+		session["audio"].(map[string]any)["output"].(map[string]any)["voice"] = cfg.RealtimeVoice
+	}
+	return json.Marshal(map[string]any{"type": "session.update", "session": session})
+}
+func BuildAudioAppend(pcm []byte) ([]byte, error) {
+	return json.Marshal(map[string]any{"type": "input_audio_buffer.append", "audio": audio.Base64Encode(pcm)})
+}
+func BuildCommit() ([]byte, error) {
+	return json.Marshal(map[string]any{"type": "input_audio_buffer.commit"})
+}
+func BuildResponseCreate() ([]byte, error) {
+	return json.Marshal(map[string]any{"type": "response.create"})
+}
+func BuildResponseCreateWithInstructions(instructions string) ([]byte, error) {
+	if instructions == "" {
+		return BuildResponseCreate()
+	}
+	return json.Marshal(map[string]any{"type": "response.create", "response": map[string]any{"instructions": instructions}})
+}
+func BuildResponseCancel() ([]byte, error) {
+	return json.Marshal(map[string]any{"type": "response.cancel"})
+}
+func BuildInputAudioClear() ([]byte, error) {
+	return json.Marshal(map[string]any{"type": "input_audio_buffer.clear"})
+}
+func BuildToolResult(result ai.ToolResult) ([]byte, error) {
+	b, _ := json.Marshal(result.Result)
+	if result.Error != "" {
+		b = []byte(result.Error)
+	}
+	return json.Marshal(map[string]any{"type": "conversation.item.create", "item": map[string]any{"type": "function_call_output", "call_id": result.ToolCallID, "output": string(b)}})
+}

internal/ai/openai/realtime/events.go

@@ -0,0 +1,103 @@
+package realtime
+
+import (
+	"ai-operator/internal/ai"
+	"encoding/base64"
+	"encoding/json"
+	"time"
+)
+
+type GenericRealtimeEvent struct {
+	Type string
+	Raw  map[string]any
+}
+
+func ParseServerEvent(data []byte) (ai.VoiceEvent, GenericRealtimeEvent, error) {
+	var base map[string]any
+	if err := json.Unmarshal(data, &base); err != nil {
+		return ai.VoiceEvent{}, GenericRealtimeEvent{}, err
+	}
+	typ, _ := base["type"].(string)
+	ev := ai.VoiceEvent{At: time.Now().UTC(), Metadata: map[string]any{"openai_type": typ}}
+	switch typ {
+	case "session.created":
+		ev.Type = ai.VoiceEventSessionStarted
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "session.updated":
+		ev.Type = ai.VoiceEventSessionUpdated
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "error":
+		ev.Type = ai.VoiceEventError
+		if e, ok := base["error"].(map[string]any); ok {
+			ev.Error = toStr(e["message"])
+		} else {
+			ev.Error = "openai error"
+		}
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "input_audio_buffer.speech_started":
+		ev.Type = ai.VoiceEventInterruption
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "response.output_audio.delta":
+		ev.Type = ai.VoiceEventAssistantAudioDelta
+		b, err := base64.StdEncoding.DecodeString(toStr(base["delta"]))
+		if err != nil {
+			return ai.VoiceEvent{}, GenericRealtimeEvent{}, err
+		}
+		ev.Audio = b
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "response.output_audio.done":
+		ev.Type = ai.VoiceEventAssistantAudioDone
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "response.output_audio_transcript.delta", "response.output_text.delta":
+		ev.Type = ai.VoiceEventAssistantTranscriptDelta
+		ev.Text = toStr(base["delta"])
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "response.output_audio_transcript.done", "response.output_text.done":
+		ev.Type = ai.VoiceEventAssistantTranscriptDone
+		ev.Text = toStr(base["transcript"]) + toStr(base["text"])
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "rate_limits.updated":
+		ev.Type = ai.VoiceEventRateLimitsUpdated
+		ev.Metadata = base
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	case "response.done":
+		if tc := extractToolCall(base); tc != nil {
+			ev.Type = ai.VoiceEventToolCall
+			ev.ToolCall = tc
+			return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+		}
+		ev.Type = ai.VoiceEventAssistantAudioDone
+		return ev, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	default:
+		return ai.VoiceEvent{}, GenericRealtimeEvent{Type: typ, Raw: base}, nil
+	}
+}
+func extractToolCall(m map[string]any) *ai.ToolCall {
+	resp, ok := m["response"].(map[string]any)
+	if !ok {
+		return nil
+	}
+	outs, ok := resp["output"].([]any)
+	if !ok {
+		return nil
+	}
+	for _, o := range outs {
+		item, ok := o.(map[string]any)
+		if !ok {
+			continue
+		}
+		if toStr(item["type"]) == "function_call" {
+			raw := toStr(item["arguments"])
+			args := map[string]any{}
+			_ = json.Unmarshal([]byte(raw), &args)
+			return &ai.ToolCall{ID: toStr(item["call_id"]), Name: toStr(item["name"]), Arguments: args, RawArguments: raw}
+		}
+	}
+	return nil
+}
+func toStr(v any) string {
+	if s, ok := v.(string); ok {
+		return s
+	}
+	return ""
+}

internal/ai/openai/realtime/provider.go

@@ -0,0 +1,224 @@
+package realtime
+
+import (
+	"ai-operator/internal/ai"
+	"ai-operator/internal/audio"
+	"ai-operator/internal/config"
+	"ai-operator/internal/media"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"github.com/gorilla/websocket"
+	"log/slog"
+	"net/http"
+	"sync"
+	"time"
+)
+
+type Provider struct {
+	cfg                     config.Config
+	logger                  *slog.Logger
+	conn                    *websocket.Conn
+	events                  chan ai.VoiceEvent
+	mu                      sync.Mutex
+	stats                   ai.VoiceProviderStats
+	inputBytes, outputBytes int64
+	eventsClosed            bool
+}
+
+func NewProvider(cfg config.Config, logger *slog.Logger) *Provider {
+	return &Provider{cfg: cfg, logger: logger, events: make(chan ai.VoiceEvent, 64)}
+}
+func (p *Provider) StartSession(ctx context.Context, sc ai.VoiceSessionConfig) error {
+	if p.cfg.OpenAI.APIKey == "" {
+		return errors.New("OPENAI_API_KEY is required for OpenAI Realtime")
+	}
+	p.mu.Lock()
+	p.events = make(chan ai.VoiceEvent, 64)
+	p.eventsClosed = false
+	p.stats = ai.VoiceProviderStats{}
+	p.inputBytes = 0
+	p.outputBytes = 0
+	p.mu.Unlock()
+	u, _, err := BuildURL(p.cfg.OpenAI)
+	if err != nil {
+		return err
+	}
+	h := http.Header{}
+	h.Set("Authorization", "Bearer "+p.cfg.OpenAI.APIKey)
+	h.Set("OpenAI-Safety-Identifier", safetyID(sc.CallID))
+	d := websocket.Dialer{HandshakeTimeout: p.cfg.OpenAI.RealtimeConnectTimeout}
+	conn, _, err := d.DialContext(ctx, u, h)
+	if err != nil {
+		return err
+	}
+	p.conn = conn
+	now := time.Now().UTC()
+	p.mu.Lock()
+	p.stats.StartedAt = &now
+	p.mu.Unlock()
+	if err := p.waitFor(ctx, "session.created"); err != nil {
+		return err
+	}
+	instr := sc.SystemPrompt
+	if instr == "" {
+		instr = "You are a test voice provider adapter for an AI operator. For now, keep responses very short. Do not claim to access customer data. Do not answer business questions yet."
+	}
+	msg, err := BuildSessionUpdate(p.cfg.OpenAI, instr)
+	if err != nil {
+		return err
+	}
+	if err := conn.WriteMessage(websocket.TextMessage, msg); err != nil {
+		return err
+	}
+	_ = p.waitFor(ctx, "session.updated")
+	go p.readLoop(sc.CallID)
+	if p.cfg.OpenAI.RealtimeInitialGreeting {
+		greeting := "Say exactly this short natural greeting in a calm, natural contact-center voice, without asking to choose language or region: Здравствуйте, меня зовут Жанна. Я AI-оператор QazAimaqGas. Чем могу помочь?"
+		rc, err := BuildResponseCreateWithInstructions(greeting)
+		if err != nil {
+			return err
+		}
+		if err := conn.WriteMessage(websocket.TextMessage, rc); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+func (p *Provider) waitFor(ctx context.Context, want string) error {
+	deadline := time.After(p.cfg.OpenAI.RealtimeConnectTimeout)
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-deadline:
+			return errors.New("timeout waiting for " + want)
+		default:
+			_, data, err := p.conn.ReadMessage()
+			if err != nil {
+				return err
+			}
+			var m map[string]any
+			_ = jsonUnmarshal(data, &m)
+			if toStr(m["type"]) == want {
+				return nil
+			}
+		}
+	}
+}
+func (p *Provider) readLoop(callID string) {
+	defer p.closeEvents()
+	for {
+		_, data, err := p.conn.ReadMessage()
+		if err != nil {
+			p.emit(ai.VoiceEvent{Type: ai.VoiceEventClosed, CallID: callID, At: time.Now().UTC()})
+			return
+		}
+		ev, _, err := ParseServerEvent(data)
+		if err != nil {
+			p.emit(ai.VoiceEvent{Type: ai.VoiceEventError, CallID: callID, Error: err.Error(), At: time.Now().UTC()})
+			continue
+		}
+		if ev.Type != "" {
+			ev.CallID = callID
+			p.emit(ev)
+		}
+	}
+}
+func (p *Provider) SendAudio(ctx context.Context, ch media.AudioChunk) error {
+	if p.conn == nil {
+		return errors.New("openai realtime websocket not connected")
+	}
+	if !audio.IsPCM16Aligned(ch.Data) {
+		return errors.New("pcm16 audio is not aligned")
+	}
+	p.mu.Lock()
+	p.inputBytes += int64(len(ch.Data))
+	if p.inputBytes > int64(p.cfg.OpenAI.RealtimeMaxInputAudioBytes) {
+		p.mu.Unlock()
+		return errors.New("max input audio bytes exceeded")
+	}
+	p.stats.InputAudioFrames++
+	p.stats.InputAudioBytes += int64(len(ch.Data))
+	p.mu.Unlock()
+	chunks := audio.ChunkBytes(ch.Data, p.cfg.OpenAI.RealtimeMaxAudioChunkBytes)
+	for _, c := range chunks {
+		msg, err := BuildAudioAppend(c)
+		if err != nil {
+			return err
+		}
+		if err := p.conn.WriteMessage(websocket.TextMessage, msg); err != nil {
+			return err
+		}
+		p.mu.Lock()
+		p.stats.EventsSent++
+		p.mu.Unlock()
+	}
+	return nil
+}
+func (p *Provider) SendToolResult(ctx context.Context, r ai.ToolResult) error {
+	if p.conn == nil {
+		return errors.New("openai realtime websocket not connected")
+	}
+	msg, err := BuildToolResult(r)
+	if err != nil {
+		return err
+	}
+	if err := p.conn.WriteMessage(websocket.TextMessage, msg); err != nil {
+		return err
+	}
+	rc, _ := BuildResponseCreate()
+	return p.conn.WriteMessage(websocket.TextMessage, rc)
+}
+func (p *Provider) Close(ctx context.Context) error {
+	p.mu.Lock()
+	now := time.Now().UTC()
+	p.stats.ClosedAt = &now
+	p.mu.Unlock()
+	if p.conn != nil {
+		_ = p.conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
+		return p.conn.Close()
+	}
+	p.closeEvents()
+	return nil
+}
+func (p *Provider) Events() <-chan ai.VoiceEvent { return p.events }
+func (p *Provider) Stats() ai.VoiceProviderStats { p.mu.Lock(); defer p.mu.Unlock(); return p.stats }
+func (p *Provider) emit(e ai.VoiceEvent) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.stats.EventsReceived++
+	if e.Type == ai.VoiceEventAssistantAudioDelta {
+		p.outputBytes += int64(len(e.Audio))
+		p.stats.OutputAudioFrames++
+		p.stats.OutputAudioBytes += int64(len(e.Audio))
+	}
+	if e.Type == ai.VoiceEventToolCall {
+		p.stats.ToolCallsReceived++
+	}
+	if e.Type == ai.VoiceEventError {
+		p.stats.Errors++
+	}
+	if p.eventsClosed {
+		return
+	}
+	select {
+	case p.events <- e:
+	default:
+	}
+}
+func (p *Provider) closeEvents() {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if !p.eventsClosed {
+		close(p.events)
+		p.eventsClosed = true
+	}
+}
+func safetyID(callID string) string {
+	sum := sha256.Sum256([]byte(callID))
+	return "aiop-" + hex.EncodeToString(sum[:])[:16]
+}
+func jsonUnmarshal(b []byte, v any) error { return json.Unmarshal(b, v) }

internal/ai/openai/realtime/realtime_test.go

@@ -0,0 +1,161 @@
+package realtime
+
+import (
+	"ai-operator/internal/ai"
+	"ai-operator/internal/config"
+	"ai-operator/internal/media"
+	"context"
+	"encoding/json"
+	"github.com/gorilla/websocket"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestURLBuildersParsers(t *testing.T) {
+	cfg := config.OpenAIConfig{RealtimeURL: "wss://api.openai.com/v1/realtime", RealtimeModel: "gpt realtime", RealtimeVoice: "marin", RealtimeInputSampleRate: 24000, RealtimeOutputSampleRate: 24000, RealtimeTurnDetection: "server_vad", RealtimeReasoningEffort: "low"}
+	u, _, err := BuildURL(cfg)
+	if err != nil || !strings.Contains(u, "model=gpt+realtime") {
+		t.Fatalf("url=%s err=%v", u, err)
+	}
+	b, err := BuildSessionUpdate(cfg, "test")
+	if err != nil || strings.Contains(string(b), "sk-") {
+		t.Fatal(string(b))
+	}
+	if !strings.Contains(string(b), `"voice":"marin"`) {
+		t.Fatalf("session.update missing configured voice: %s", b)
+	}
+	for _, fn := range []func() ([]byte, error){func() ([]byte, error) { return BuildAudioAppend([]byte{1, 2}) }, BuildCommit, BuildResponseCreate, BuildResponseCancel, BuildInputAudioClear, func() ([]byte, error) {
+		return BuildToolResult(ai.ToolResult{ToolCallID: "tc", Result: map[string]string{"ok": "true"}})
+	}} {
+		x, err := fn()
+		if err != nil || !json.Valid(x) {
+			t.Fatalf("bad event %s %v", x, err)
+		}
+	}
+	ev, _, err := ParseServerEvent([]byte(`{"type":"response.output_audio.delta","delta":"AQI="}`))
+	if err != nil || ev.Type != ai.VoiceEventAssistantAudioDelta || len(ev.Audio) != 2 {
+		t.Fatalf("ev=%+v err=%v", ev, err)
+	}
+	ev, _, _ = ParseServerEvent([]byte(`{"type":"input_audio_buffer.speech_started"}`))
+	if ev.Type != ai.VoiceEventInterruption {
+		t.Fatal(ev.Type)
+	}
+	ev, _, _ = ParseServerEvent([]byte(`{"type":"response.done","response":{"output":[{"type":"function_call","call_id":"c","name":"set_language","arguments":"{\"language\":\"ru\"}"}]}}`))
+	if ev.ToolCall == nil || ev.ToolCall.Name != "set_language" {
+		t.Fatalf("tool=%+v", ev.ToolCall)
+	}
+	if _, _, err := ParseServerEvent([]byte(`bad`)); err == nil {
+		t.Fatal("want err")
+	}
+}
+func TestProviderFakeWebSocket(t *testing.T) {
+	up := websocket.Upgrader{}
+	var auth string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		auth = r.Header.Get("Authorization")
+		c, err := up.Upgrade(w, r, nil)
+		if err != nil {
+			return
+		}
+		defer c.Close()
+		_ = c.WriteJSON(map[string]any{"type": "session.created"})
+		_, msg, _ := c.ReadMessage()
+		if !strings.Contains(string(msg), "session.update") {
+			t.Errorf("msg=%s", msg)
+		}
+		_ = c.WriteJSON(map[string]any{"type": "session.updated"})
+		_, _, _ = c.ReadMessage()
+		_ = c.WriteJSON(map[string]any{"type": "response.output_audio.delta", "delta": "AQI="})
+		time.Sleep(20 * time.Millisecond)
+	}))
+	defer srv.Close()
+	cfg := config.Config{OpenAI: config.OpenAIConfig{APIKey: "sk-test", RealtimeURL: "ws" + strings.TrimPrefix(srv.URL, "http"), RealtimeModel: "m", RealtimeConnectTimeout: time.Second, RealtimeMaxAudioChunkBytes: 100, RealtimeMaxInputAudioBytes: 1000, RealtimeInputSampleRate: 24000, RealtimeOutputSampleRate: 24000, RealtimeTurnDetection: "server_vad", RealtimeReasoningEffort: "low"}}
+	p := NewProvider(cfg, nil)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	if err := p.StartSession(ctx, ai.VoiceSessionConfig{CallID: "c1"}); err != nil {
+		t.Fatal(err)
+	}
+	if auth != "Bearer sk-test" {
+		t.Fatal("auth missing")
+	}
+	if err := p.SendAudio(ctx, media.AudioChunk{Data: []byte{0, 0}}); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case ev := <-p.Events():
+		_ = ev
+	case <-time.After(time.Second):
+		t.Fatal("no event")
+	}
+	_ = p.Close(ctx)
+}
+func TestProviderCanStartAfterClose(t *testing.T) {
+	up := websocket.Upgrader{}
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		c, err := up.Upgrade(w, r, nil)
+		if err != nil {
+			return
+		}
+		defer c.Close()
+		_ = c.WriteJSON(map[string]any{"type": "session.created"})
+		_, msg, _ := c.ReadMessage()
+		if !strings.Contains(string(msg), "session.update") {
+			t.Errorf("msg=%s", msg)
+		}
+		_ = c.WriteJSON(map[string]any{"type": "session.updated"})
+		_ = c.WriteJSON(map[string]any{"type": "response.output_audio.delta", "delta": "AQI="})
+		for {
+			if _, _, err := c.ReadMessage(); err != nil {
+				return
+			}
+		}
+	}))
+	defer srv.Close()
+	cfg := config.Config{OpenAI: config.OpenAIConfig{APIKey: "sk-test", RealtimeURL: "ws" + strings.TrimPrefix(srv.URL, "http"), RealtimeModel: "m", RealtimeConnectTimeout: time.Second, RealtimeMaxAudioChunkBytes: 100, RealtimeMaxInputAudioBytes: 1000, RealtimeInputSampleRate: 24000, RealtimeOutputSampleRate: 24000, RealtimeTurnDetection: "server_vad", RealtimeReasoningEffort: "low"}}
+	p := NewProvider(cfg, nil)
+	for _, callID := range []string{"c1", "c2"} {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		if err := p.StartSession(ctx, ai.VoiceSessionConfig{CallID: callID}); err != nil {
+			cancel()
+			t.Fatal(err)
+		}
+		select {
+		case ev := <-p.Events():
+			if ev.CallID != callID {
+				cancel()
+				t.Fatalf("call_id=%s want %s", ev.CallID, callID)
+			}
+		case <-ctx.Done():
+			cancel()
+			t.Fatal("no event")
+		}
+		if err := p.Close(ctx); err != nil {
+			cancel()
+			t.Fatal(err)
+		}
+		for {
+			select {
+			case _, ok := <-p.Events():
+				if !ok {
+					cancel()
+					goto next
+				}
+			case <-ctx.Done():
+				cancel()
+				t.Fatal("events channel did not close")
+			}
+		}
+	next:
+	}
+}
+func TestCostGuard(t *testing.T) {
+	p := NewProvider(config.Config{OpenAI: config.OpenAIConfig{RealtimeMaxInputAudioBytes: 1}}, nil)
+	p.conn = &websocket.Conn{}
+	if err := p.SendAudio(context.Background(), media.AudioChunk{Data: []byte{0, 0}}); err == nil {
+		t.Fatal("want cost err")
+	}
+}

internal/ai/openai/realtime/url.go

@@ -0,0 +1,17 @@
+package realtime
+
+import (
+	"ai-operator/internal/config"
+	"net/url"
+)
+
+func BuildURL(cfg config.OpenAIConfig) (string, string, error) {
+	u, err := url.Parse(cfg.RealtimeURL)
+	if err != nil {
+		return "", "", err
+	}
+	q := u.Query()
+	q.Set("model", cfg.RealtimeModel)
+	u.RawQuery = q.Encode()
+	return u.String(), u.String(), nil
+}