sync: migrate ai-operator to Gitea (2026-08-10)
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFakeProviderDeterministic(t *testing.T) {
|
||||
p := NewFakeProvider(1536, "fake")
|
||||
a, err := p.Embed(context.Background(), []string{"hello", "world"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, _ := p.Embed(context.Background(), []string{"hello"})
|
||||
if len(a) != 2 || len(a[0]) != 1536 || len(b[0]) != 1536 {
|
||||
t.Fatalf("bad dims")
|
||||
}
|
||||
if a[0][0] != b[0][0] {
|
||||
t.Fatalf("not deterministic")
|
||||
}
|
||||
if a[0][0] == a[1][0] && a[0][1] == a[1][1] {
|
||||
t.Fatalf("different inputs too similar")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"math"
|
||||
)
|
||||
|
||||
type FakeProvider struct {
|
||||
dims int
|
||||
model string
|
||||
}
|
||||
|
||||
func NewFakeProvider(dims int, model string) *FakeProvider {
|
||||
if dims <= 0 {
|
||||
dims = 1536
|
||||
}
|
||||
if model == "" {
|
||||
model = "fake"
|
||||
}
|
||||
return &FakeProvider{dims: dims, model: model}
|
||||
}
|
||||
func (p *FakeProvider) Dimensions() int { return p.dims }
|
||||
func (p *FakeProvider) Model() string { return p.model }
|
||||
func (p *FakeProvider) ProviderName() string { return "fake" }
|
||||
func (p *FakeProvider) Embed(ctx context.Context, input []string) ([]Vector, error) {
|
||||
out := make([]Vector, 0, len(input))
|
||||
for _, text := range input {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
}
|
||||
v := make(Vector, p.dims)
|
||||
seed := sha256.Sum256([]byte(text))
|
||||
var norm float64
|
||||
for i := 0; i < p.dims; i++ {
|
||||
b := sha256.Sum256(append(seed[:], byte(i), byte(i>>8)))
|
||||
n := binary.BigEndian.Uint32(b[:4])
|
||||
val := (float64(n%2000000)/1000000.0 - 1.0)
|
||||
v[i] = float32(val)
|
||||
norm += val * val
|
||||
}
|
||||
if norm > 0 {
|
||||
scale := float32(1 / math.Sqrt(norm))
|
||||
for i := range v {
|
||||
v[i] *= scale
|
||||
}
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func VectorLiteral(v Vector) string {
|
||||
parts := make([]string, len(v))
|
||||
for i, x := range v {
|
||||
parts[i] = fmt.Sprintf("%.8f", x)
|
||||
}
|
||||
return "[" + strings.Join(parts, ",") + "]"
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
type OpenAIProvider struct {
|
||||
apiKey, model string
|
||||
dims int
|
||||
endpoint string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewOpenAIProvider(apiKey, model string, dims int) *OpenAIProvider {
|
||||
if model == "" {
|
||||
model = "text-embedding-3-small"
|
||||
}
|
||||
if dims <= 0 {
|
||||
dims = 1536
|
||||
}
|
||||
return &OpenAIProvider{apiKey: apiKey, model: model, dims: dims, endpoint: "https://api.openai.com/v1/embeddings", client: &http.Client{Timeout: 30 * time.Second}}
|
||||
}
|
||||
func (p *OpenAIProvider) Dimensions() int { return p.dims }
|
||||
func (p *OpenAIProvider) Model() string { return p.model }
|
||||
func (p *OpenAIProvider) ProviderName() string { return "openai" }
|
||||
func (p *OpenAIProvider) Embed(ctx context.Context, input []string) ([]Vector, error) {
|
||||
if p.apiKey == "" {
|
||||
return nil, fmt.Errorf("OPENAI_API_KEY is required")
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"model": p.model, "input": input, "dimensions": p.dims})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.endpoint, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+p.apiKey)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("openai embeddings request failed: status %d", resp.StatusCode)
|
||||
}
|
||||
var parsed struct {
|
||||
Data []struct {
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]Vector, 0, len(parsed.Data))
|
||||
for _, item := range parsed.Data {
|
||||
if len(item.Embedding) != p.dims {
|
||||
return nil, fmt.Errorf("embedding dimension mismatch")
|
||||
}
|
||||
out = append(out, Vector(item.Embedding))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package embedding
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"ai-operator/internal/config"
|
||||
)
|
||||
|
||||
type Vector []float32
|
||||
|
||||
type Provider interface {
|
||||
Embed(ctx context.Context, input []string) ([]Vector, error)
|
||||
Dimensions() int
|
||||
Model() string
|
||||
ProviderName() string
|
||||
}
|
||||
|
||||
func NewProvider(cfg config.Config, override string) (Provider, error) {
|
||||
name := cfg.Embedding.Provider
|
||||
if override != "" {
|
||||
name = override
|
||||
}
|
||||
switch name {
|
||||
case "fake":
|
||||
return NewFakeProvider(cfg.Embedding.Dimensions, cfg.Embedding.Model), nil
|
||||
case "openai":
|
||||
if cfg.OpenAI.APIKey == "" {
|
||||
return nil, fmt.Errorf("OPENAI_API_KEY is required for openai embedding provider")
|
||||
}
|
||||
return NewOpenAIProvider(cfg.OpenAI.APIKey, cfg.Embedding.Model, cfg.Embedding.Dimensions), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown embedding provider: %s", name)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user