Files
pipelines/internal/codex/models_test.go
T

195 lines
6.5 KiB
Go

package codex
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
"git.a-wan.net/alan/pipelines/internal/protocol"
"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
)
const fixtureThread = "01900000-0000-7000-8000-000000000001"
func modelFixture(t *testing.T, name string) json.RawMessage {
t.Helper()
raw, err := os.ReadFile("testdata/" + name + ".json")
if err != nil {
t.Fatal(err)
}
return raw
}
func modelTestChat(t *testing.T, handle func(string, json.RawMessage) any) *Chat {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := websocket.Accept(w, r, nil)
if err != nil {
return
}
defer conn.CloseNow()
for {
var req rpcMessage
if err := wsjson.Read(ctx, conn, &req); err != nil {
return
}
result := handle(req.Method, req.Params)
response := map[string]any{"id": req.ID, "result": result}
if failure, ok := result.(*rpcError); ok {
response = map[string]any{"id": req.ID, "error": failure}
}
if err := wsjson.Write(ctx, conn, response); err != nil {
return
}
}
}))
conn, _, err := websocket.Dial(ctx, "ws"+strings.TrimPrefix(srv.URL, "http"), nil)
if err != nil {
t.Fatal(err)
}
conn.SetReadLimit(16 << 20) // Match the real App Server adapter's frame bound.
c := &Chat{conn: conn, ctx: ctx, cancel: cancel, pending: map[string]chan rpcResponse{}, subs: map[string]map[chan protocol.ChatEventFrame]struct{}{}, forms: map[string]pendingForm{}, turns: map[string]string{}, emitted: map[string]bool{}}
go c.readLoop()
t.Cleanup(func() { c.Close(); srv.Close() })
return c
}
func TestModelCatalogAndSelectionNeverResume(t *testing.T) {
thread, catalog := modelFixture(t, "thread-read"), modelFixture(t, "model-list")
var mu sync.Mutex
methods := []string{}
chat := modelTestChat(t, func(method string, raw json.RawMessage) any {
mu.Lock()
methods = append(methods, method)
mu.Unlock()
var params map[string]any
if err := json.Unmarshal(raw, &params); err != nil {
t.Error(err)
}
switch method {
case "thread/read":
if params["threadId"] != fixtureThread || params["includeTurns"] != nil {
t.Errorf("thread read params = %s", raw)
}
return thread
case "model/list":
if params["includeHidden"] != false || params["limit"] != float64(100) {
t.Errorf("catalog params = %s", raw)
}
if params["cursor"] == nil {
return catalog
}
if params["cursor"] != "page-two" {
t.Errorf("cursor = %v", params["cursor"])
}
return json.RawMessage(`{"data":[{"id":"hidden","model":"fixture-hidden","displayName":"Hidden","hidden":true}],"nextCursor":null}`)
case "turn/start":
if params["model"] != "fixture-model" || params["threadId"] != fixtureThread || params["clientUserMessageId"] != "msg_fixture" {
t.Errorf("turn params = %s", raw)
}
if params["modelProvider"] != nil {
t.Error("turn/start does not accept a provider override")
}
return json.RawMessage(`{"turn":{"id":"turn-fixture","status":"inProgress","items":[]}}`)
default:
t.Errorf("unexpected/unsafe model operation: %s", method)
return map[string]any{}
}
})
ctx := context.Background()
session, found, err := chat.Session(ctx, fixtureThread)
if err != nil || !found || session.Model != "fixture-current" || session.ModelProvider != "fixture-provider" {
t.Fatalf("session = %+v, %v", session, err)
}
models, err := chat.Models(ctx, fixtureThread)
if err != nil || len(models) != 1 || models[0].ID != "fixture-model" || models[0].Provider != "fixture-provider" {
t.Fatalf("catalog = %+v, %v", models, err)
}
if err := chat.SwitchModel(ctx, fixtureThread, "fixture-provider", "fixture-model"); err != nil {
t.Fatal(err)
}
for _, pair := range [][2]string{{"wrong-provider", "fixture-model"}, {"fixture-provider", "opaque-picker-id"}, {"fixture-provider", "fixture-hidden"}} {
if err := chat.SwitchModel(ctx, fixtureThread, pair[0], pair[1]); err == nil {
t.Fatalf("invalid selection accepted: %v", pair)
}
}
mu.Lock()
for _, method := range methods {
if method != "thread/read" && method != "model/list" {
t.Errorf("selection invoked %s", method)
}
}
mu.Unlock()
if err := chat.PromptWithMessageIDModel(ctx, fixtureThread, "hello", "msg_fixture", "fixture-model"); err != nil {
t.Fatal(err)
}
}
func TestModelCatalogFailsClosed(t *testing.T) {
for _, name := range []string{"missing-provider", "invalid-data", "repeated-cursor"} {
t.Run(name, func(t *testing.T) {
thread := modelFixture(t, "thread-read")
chat := modelTestChat(t, func(method string, _ json.RawMessage) any {
switch method {
case "thread/read":
if name == "missing-provider" {
return json.RawMessage(`{"thread":{"id":"fixture-thread","model":null}}`)
}
return thread
case "model/list":
if name == "missing-provider" {
t.Error("catalog requested with unknown provider")
}
if name == "repeated-cursor" {
return json.RawMessage(`{"data":[],"nextCursor":"same"}`)
}
return json.RawMessage(`{"data":null,"nextCursor":null}`)
default:
t.Errorf("unsafe method %s", method)
return nil
}
})
if _, err := chat.Models(context.Background(), fixtureThread); err == nil {
t.Fatal("invalid catalog accepted")
}
})
}
}
func TestModelMetadataFromThreadStartAndList(t *testing.T) {
thread := modelFixture(t, "thread-read")
chat := modelTestChat(t, func(method string, _ json.RawMessage) any {
switch method {
case "thread/start":
return json.RawMessage(`{"thread":{"id":"fixture-thread","model":null,"modelProvider":""},"model":"fixture-start","modelProvider":"fixture-provider"}`)
case "thread/list":
var result struct {
Thread json.RawMessage `json:"thread"`
}
if err := json.Unmarshal(thread, &result); err != nil {
t.Error(err)
}
return map[string]any{"data": []json.RawMessage{result.Thread}, "nextCursor": nil}
default:
t.Errorf("unexpected method %s", method)
return nil
}
})
created, err := chat.Create(context.Background(), "", "/fixture")
if err != nil || created.Model != "fixture-start" || created.ModelProvider != "fixture-provider" {
t.Fatalf("create model = %+v, %v", created, err)
}
listed, err := chat.Sessions(context.Background())
if err != nil || len(listed) != 1 || listed[0].Model != "fixture-current" || listed[0].ModelProvider != "fixture-provider" {
t.Fatalf("list model = %+v, %v", listed, err)
}
}