Files
pipelines/internal/server/overrides_test.go
T
alan fae1005962 Allow launching runs with whole-run and per-stage model overrides
Add `pipelines run --model PROVIDER/MODEL` and repeatable
`--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence
per-stage > whole-run > the model pinned in the pipeline version.

- A shared pure resolver (`pipeline.ParseModelOverride` /
  `pipeline.EffectiveModels`) owns the precedence and validation: every value
  must be a `provider/model` pair (`stage.ValidModel`), every stage key must
  exist in the pinned graph, an unknown key fails naming the valid keys, and a
  per-stage key that names a non-model stage is refused.
- The server validates against the pinned version before admission (no run row
  is ever inserted for an invalid request), persists `model`/`stage_models` on
  the run and `provider`/`model` on each step, applies the effective pair to a
  clone of the stage definition so the stage and its repair/formatter path share
  it, overlays the returned run graph with the effective pair (presentation
  only), and reuses the recorded overrides on retry.
- The run record echoes the overrides it was launched with and each step records
  the provider/model it actually ran with; the TUI Evidence pane renders that
  effective pair. The pinned pipeline version and `show --version N` are never
  modified.
- CLI help, `docs/cli.md` and the packaged authoring skill references document
  the flags and the precedence. A TUI override picker remains a documented
  follow-up; the CLI is the deliverable.

Tests: pure precedence/validation table tests, store round-trip and
request-key-mismatch plus migration, server admission/application/overlay/retry
with a controlled executor, CLI flag/help/malformed-value tests, and a TUI
evidence test.
2026-10-16 23:33:53 +01:00

193 lines
7.1 KiB
Go

package server
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
)
// TestRunnerAppliesModelOverrides proves a run with exactly one overridden stage
// records that step's provider/model as the override and the other model stage as
// the pinned pair, that the same effective values reach the resolved ModelConfig
// (the single source the repair/formatter path reads), and that the run record
// and returned graph echo the overrides while the pinned source is unchanged.
func TestRunnerAppliesModelOverrides(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "overrides.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "overrides", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
pinned := map[string]stage.ModelConfig{}
for _, s := range definition.Graph.Stages {
pinned[s.Key] = stage.ModelConfig{Provider: s.Configs["provider"], Model: s.Configs["model"]}
}
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
resolved := map[string]stage.ModelConfig{}
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
spec, err := (stage.ModelStage{}).Spec(def)
if err != nil {
return stage.Result{}, err
}
resolved[def.Key] = spec.Config
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "one-override", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" {
t.Fatalf("stage one resolved %+v, want the override", got)
}
if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model {
t.Fatalf("stage two resolved %+v, want the pinned %+v", got, pinned["two"])
}
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Model != "" || detail.StageModels["one"] != "acme/strong" {
t.Fatalf("run record did not echo the overrides: %+v", detail)
}
if len(detail.Steps) != 2 {
t.Fatalf("steps = %+v", detail.Steps)
}
if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" {
t.Fatalf("overridden step evidence = %+v", detail.Steps[0])
}
if detail.Steps[1].Provider != pinned["two"].Provider || detail.Steps[1].Model != pinned["two"].Model {
t.Fatalf("pinned step evidence = %+v, want %+v", detail.Steps[1], pinned["two"])
}
// The returned graph shows the effective pair for the overridden stage and
// leaves the other at its pinned configs; `show --version N` (the immutable
// source) is untouched.
overlaid, err := r.getRun(run.ID)
if err != nil {
t.Fatal(err)
}
configs := map[string]map[string]string{}
for _, s := range overlaid.Graph.Stages {
configs[s.Key] = s.Configs
}
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" {
t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"])
}
if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model {
t.Fatalf("overlaid graph stage two = %+v, want the pinned pair", configs["two"])
}
loaded, err := store.Get(ctx, definition.ID, 1)
if err != nil {
t.Fatal(err)
}
if loaded.Source != definition.Source {
t.Fatal("the pinned immutable source was rewritten by an override")
}
}
// TestRunnerRefusesInvalidModelOverridesWithoutRunRow proves an unknown stage key// and a malformed provider/model both fail before admission and create no run row.
func TestRunnerRefusesInvalidModelOverridesWithoutRunRow(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "reject.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "reject", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
before, _ := store.ListRuns(ctx)
cases := []struct {
name string
key string
model string
stage map[string]string
want string
}{
{"unknown stage key", "unknown-key", "", map[string]string{"nope": "acme/strong"}, "unknown stage key"},
{"invalid whole-run pair", "invalid-model", "not-a-pair", nil, "provider/model"},
{"invalid stage pair", "invalid-stage", "", map[string]string{"one": "acme/"}, "provider/model"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: tc.key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, Model: tc.model, StageModels: tc.stage})
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Fatalf("error = %v, want %q", err, tc.want)
}
if id, err := store.RunByKey(ctx, tc.key); err != nil || id != 0 {
t.Fatalf("rejected request created run %d: %v", id, err)
}
})
}
after, _ := store.ListRuns(ctx)
if len(after) != len(before) {
t.Fatalf("rejected requests created run rows: before %d after %d", len(before), len(after))
}
}
// TestRetryReusesModelOverrides proves a retry of a failed run reuses the
// original run's recorded overrides, so the new attempt is reproducible.
func TestRetryReusesModelOverrides(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-override.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "retry-override", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, errors.New("provider failed")
}
first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "first", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if detail, _ := store.GetRun(ctx, first.ID); detail.Status != "failed" {
t.Fatalf("original run did not fail: %+v", detail)
}
retried, err := r.retry(protocol.Message{RunID: first.ID, RequestKey: "retry"})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if retried.StageModels["one"] != "acme/strong" || retried.Model != "" {
t.Fatalf("retry did not reuse the original overrides: %+v", retried)
}
}