Files

194 lines
7.3 KiB
Go

package server
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// TestRunnerAppliesModelOverrides proves a run with exactly one overridden stage
// records that step's provider/model as the override and the other model stage as
// the pinned pair, that the same effective values reach the resolved ModelConfig
// (the single source the repair/formatter path reads), and that the run record
// and returned graph echo the overrides while the pinned source is unchanged.
func TestRunnerAppliesModelOverrides(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "overrides.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
source := strings.Replace(pipeline.Template([]string{"one", "two"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1)
if _, err := store.Submit(ctx, "overrides", source); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
pinned := map[string]stage.ModelConfig{}
for _, s := range definition.Graph.Stages {
pinned[s.Key] = stage.ModelConfig{Provider: s.Configs["provider"], Model: s.Configs["model"]}
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
resolved := map[string]stage.ModelConfig{}
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
spec, err := (stage.ModelStage{}).Spec(def)
if err != nil {
return stage.Result{}, err
}
resolved[def.Key] = spec.Config
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "one-override", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" || got.ReasoningEffort != "high" {
t.Fatalf("stage one resolved %+v, want the override", got)
}
if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model {
t.Fatalf("stage two resolved %+v, want the pinned %+v", got, pinned["two"])
}
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Model != "" || detail.StageModels["one"] != "acme/strong" {
t.Fatalf("run record did not echo the overrides: %+v", detail)
}
if len(detail.Steps) != 2 {
t.Fatalf("steps = %+v", detail.Steps)
}
if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" {
t.Fatalf("overridden step evidence = %+v", detail.Steps[0])
}
if detail.Steps[1].Provider != pinned["two"].Provider || detail.Steps[1].Model != pinned["two"].Model {
t.Fatalf("pinned step evidence = %+v, want %+v", detail.Steps[1], pinned["two"])
}
// The returned graph shows the effective pair for the overridden stage and
// leaves the other at its pinned configs; `show --version N` (the immutable
// source) is untouched.
overlaid, err := r.getRun(run.ID)
if err != nil {
t.Fatal(err)
}
configs := map[string]map[string]string{}
for _, s := range overlaid.Graph.Stages {
configs[s.Key] = s.Configs
}
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" || configs["one"]["reasoning_effort"] != "high" {
t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"])
}
if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model {
t.Fatalf("overlaid graph stage two = %+v, want the pinned pair", configs["two"])
}
loaded, err := store.Get(ctx, definition.ID, 1)
if err != nil {
t.Fatal(err)
}
if loaded.Source != definition.Source {
t.Fatal("the pinned immutable source was rewritten by an override")
}
}
// TestRunnerRefusesInvalidModelOverridesWithoutRunRow proves an unknown stage key// and a malformed provider/model both fail before admission and create no run row.
func TestRunnerRefusesInvalidModelOverridesWithoutRunRow(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "reject.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "reject", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
before, _ := store.ListRuns(ctx)
cases := []struct {
name string
key string
model string
stage map[string]string
want string
}{
{"unknown stage key", "unknown-key", "", map[string]string{"nope": "acme/strong"}, "unknown stage key"},
{"invalid whole-run pair", "invalid-model", "not-a-pair", nil, "provider/model"},
{"invalid stage pair", "invalid-stage", "", map[string]string{"one": "acme/"}, "provider/model"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: tc.key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, Model: tc.model, StageModels: tc.stage})
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("error = %v, want %q", err, tc.want)
}
if id, err := store.RunByKey(ctx, tc.key); err != nil || id != 0 {
t.Fatalf("rejected request created run %d: %v", id, err)
}
})
}
after, _ := store.ListRuns(ctx)
if len(after) != len(before) {
t.Fatalf("rejected requests created run rows: before %d after %d", len(before), len(after))
}
}
// TestRetryReusesModelOverrides proves a retry of a failed run reuses the
// original run's recorded overrides, so the new attempt is reproducible.
func TestRetryReusesModelOverrides(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-override.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "retry-override", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, errors.New("provider failed")
}
first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "first", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if detail, _ := store.GetRun(ctx, first.ID); detail.Status != "failed" {
t.Fatalf("original run did not fail: %+v", detail)
}
retried, err := r.retry(protocol.Message{RunID: first.ID, RequestKey: "retry"})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if retried.StageModels["one"] != "acme/strong" || retried.Model != "" {
t.Fatalf("retry did not reuse the original overrides: %+v", retried)
}
}