Files
pipelines/internal/server/chat_efforts_test.go
T
alan 06001697c8 Add per-chat thinking levels for Codex, Claude Code and OpenCode
Codex: model/list supportedReasoningEfforts per model, turn/start effort,
thread reasoningEffort as the current level. Claude Code: initialize
supportedEffortLevels, apply_flag_settings effortLevel at the next turn
start. OpenCode v2: model variants via the session model endpoint.
chat_effort sets a level; rows and chat_models carry the levels; Codex and
Claude levels persist in operator_sessions.chat_effort and are re-applied
before every turn.
2026-11-09 20:07:41 +00:00

171 lines
7.4 KiB
Go

package server
import (
"context"
"errors"
"path/filepath"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// effortedChat reports levels per model and records the levels handed to it.
type effortedChat struct {
*selectedModelChat
levels map[string][]protocol.ChatEffort
defaults map[string]string
applied []string
}
func (c *effortedChat) Efforts(_ context.Context, session protocol.ChatSession) ([]protocol.ChatEffort, string, error) {
return c.levels[session.Model], c.defaults[session.Model], nil
}
func (c *effortedChat) SetEffort(_ context.Context, _, effort string) error {
if effort == "refused" {
return errors.New("harness refused")
}
c.applied = append(c.applied, effort)
return nil
}
func effortFixture(t *testing.T, path, backend string) (*pipeline.Store, *effortedChat, string) {
t.Helper()
ctx := context.Background()
store, err := pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
id := "01900000-0000-7000-8000-000000000003"
if err := store.RecordOperatorSession(ctx, id, "/fixture", "chat", protocol.ChatSessionTagOperator); err != nil {
t.Fatal(err)
}
if err := store.SetOperatorSessionBackend(ctx, id, backend); err != nil {
t.Fatal(err)
}
fake := &effortedChat{selectedModelChat: &selectedModelChat{fakeChat: scriptedChat()},
levels: map[string][]protocol.ChatEffort{"thinker": {{ID: "low", Name: "Low"}, {ID: "high", Name: "High"}}, "other": {{ID: "high", Name: "High"}, {ID: "max", Name: "Max"}}},
defaults: map[string]string{"thinker": "high", "other": "max"},
}
fake.models = []protocol.ChatModel{{ID: "thinker", Provider: "prov"}, {ID: "other", Provider: "prov"}, {ID: "plain", Provider: "prov"}}
fake.sessionRecord = protocol.ChatSession{ID: id, Backend: backend, Model: "thinker", ModelProvider: "prov"}
fake.sessionFound = true
fake.sessions = []protocol.ChatSession{fake.sessionRecord}
return store, fake, id
}
func TestChatEffortIsValidatedStoredAndReappliedAfterRestart(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "chat.db")
store, fake, id := effortFixture(t, path, "codex")
manager := &chatManager{codex: fake, registry: store, cache: map[string]string{}}
var reply protocol.Message
capture := func(msg protocol.Message) error { reply = msg; return nil }
session, _, err := manager.Session(ctx, id)
if err != nil || session.Effort != "" || session.DefaultEffort != "high" || len(session.Efforts) != 2 {
t.Fatalf("fresh row = %+v, %v", session, err)
}
for _, bad := range []string{"max", "refused"} {
handleChatEffort(ctx, protocol.Message{ID: "bad", EffortRequest: &protocol.ChatEffortRequest{SessionID: id, Effort: bad}}, manager, capture)
if reply.Type != protocol.Error {
t.Fatalf("level %q accepted: %+v", bad, reply)
}
}
handleChatEffort(ctx, protocol.Message{ID: "set", EffortRequest: &protocol.ChatEffortRequest{SessionID: id, Effort: "low"}}, manager, capture)
if reply.Type != protocol.ChatEffortSelect || reply.ID != "set" || reply.EffortRequest.Effort != "low" || len(reply.EffortRequest.Efforts) != 2 || reply.EffortRequest.DefaultEffort != "high" {
t.Fatalf("set reply = %+v", reply)
}
if len(fake.applied) != 1 || fake.applied[0] != "low" || fake.overrides != 0 {
t.Fatalf("applied %v, turns %d", fake.applied, fake.overrides)
}
handleChatModels(ctx, protocol.Message{ID: "models", ChatQuery: &protocol.ChatQuery{SessionID: id}}, manager, capture)
if reply.Type != protocol.ChatModels || reply.EffortRequest == nil || reply.EffortRequest.Effort != "low" {
t.Fatalf("models reply = %+v", reply)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
// A restarted server lists the stored level and re-applies it before turns.
store, err = pipeline.Open(path)
if err != nil {
t.Fatal(err)
}
defer store.Close()
fake.applied = nil
manager = &chatManager{codex: fake, registry: store, cache: map[string]string{}}
list, err := manager.Sessions(ctx)
if err != nil || len(list) != 1 || list[0].Effort != "low" || len(list[0].Efforts) != 2 {
t.Fatalf("restarted list = %+v, %v", list, err)
}
if err := manager.Prompt(ctx, id, "hello"); err != nil {
t.Fatal(err)
}
if err := manager.PromptWithMessageIDFiles(ctx, id, "again", "msg", nil); err != nil {
t.Fatal(err)
}
if len(fake.applied) != 2 || fake.applied[0] != "low" || fake.applied[1] != "low" {
t.Fatalf("stored level not re-applied before turns: %v", fake.applied)
}
// A model that keeps the level keeps it; one without it falls back to its
// default; one without levels clears it and offers no control.
for _, step := range []struct{ model, stored string }{{"thinker", "low"}, {"other", "max"}, {"plain", ""}} {
handleChatModelSwitch(ctx, protocol.Message{ID: "m", ModelRequest: &protocol.ChatModelRequest{SessionID: id, Provider: "prov", Model: step.model}}, manager, capture)
if reply.Type != protocol.ChatModelSelect {
t.Fatalf("switch reply = %+v", reply)
}
if stored, err := store.ChatEffort(ctx, id); err != nil || stored != step.stored {
t.Fatalf("after switching to %s stored %q, want %q (%v)", step.model, stored, step.stored, err)
}
}
session, _, err = manager.Session(ctx, id)
if err != nil || session.Effort != "" || len(session.Efforts) != 0 || session.DefaultEffort != "" {
t.Fatalf("level-less model row = %+v, %v", session, err)
}
handleChatEffort(ctx, protocol.Message{ID: "none", EffortRequest: &protocol.ChatEffortRequest{SessionID: id, Effort: "high"}}, manager, capture)
if reply.Type != protocol.Error || reply.Error != errChatEffortsNone.Error() {
t.Fatalf("level-less model accepted a level: %+v", reply)
}
}
func TestOpenCodeEffortStaysNativeAndUnsupportedHarnessOffersNone(t *testing.T) {
ctx := context.Background()
store, fake, id := effortFixture(t, filepath.Join(t.TempDir(), "chat.db"), "opencode")
defer store.Close()
fake.sessionRecord.Effort = "high"
fake.sessions = []protocol.ChatSession{fake.sessionRecord}
manager := &chatManager{opencode: fake, registry: store, cache: map[string]string{}}
session, _, err := manager.Session(ctx, id)
if err != nil || session.Effort != "high" || len(session.Efforts) != 2 {
t.Fatalf("native level row = %+v, %v", session, err)
}
state, err := manager.SetEffort(ctx, id, "low")
if err != nil || state.Effort != "low" || len(fake.applied) != 1 {
t.Fatalf("native set = %+v, %v (applied %v)", state, err, fake.applied)
}
if stored, _ := store.ChatEffort(ctx, id); stored != "" {
t.Fatalf("OpenCode level stored by Pipelines: %q", stored)
}
// The variant is OpenCode's own state; a turn never re-sends it.
if err := manager.Prompt(ctx, id, "hi"); err != nil || len(fake.applied) != 1 {
t.Fatalf("turn re-applied a native level: %v %v", fake.applied, err)
}
plain := scriptedChat()
plain.sessionRecord = protocol.ChatSession{ID: id, Backend: "opencode", Model: "thinker", ModelProvider: "prov", Effort: "high"}
plain.sessionFound = true
manager = &chatManager{opencode: plain, registry: store, cache: map[string]string{}}
session, _, err = manager.Session(ctx, id)
if err != nil || session.Effort != "" || len(session.Efforts) != 0 {
t.Fatalf("unsupported harness row = %+v, %v", session, err)
}
var reply protocol.Message
handleChatEffort(ctx, protocol.Message{ID: "x", EffortRequest: &protocol.ChatEffortRequest{SessionID: id, Effort: "high"}}, manager, func(msg protocol.Message) error { reply = msg; return nil })
if reply.Type != protocol.Error || reply.Error != errChatEffortsUnsupported.Error() {
t.Fatalf("unsupported harness reply = %+v", reply)
}
}