Files
pipelines/internal/pipeline/overrides_test.go
T
alan fae1005962 Allow launching runs with whole-run and per-stage model overrides
Add `pipelines run --model PROVIDER/MODEL` and repeatable
`--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence
per-stage > whole-run > the model pinned in the pipeline version.

- A shared pure resolver (`pipeline.ParseModelOverride` /
  `pipeline.EffectiveModels`) owns the precedence and validation: every value
  must be a `provider/model` pair (`stage.ValidModel`), every stage key must
  exist in the pinned graph, an unknown key fails naming the valid keys, and a
  per-stage key that names a non-model stage is refused.
- The server validates against the pinned version before admission (no run row
  is ever inserted for an invalid request), persists `model`/`stage_models` on
  the run and `provider`/`model` on each step, applies the effective pair to a
  clone of the stage definition so the stage and its repair/formatter path share
  it, overlays the returned run graph with the effective pair (presentation
  only), and reuses the recorded overrides on retry.
- The run record echoes the overrides it was launched with and each step records
  the provider/model it actually ran with; the TUI Evidence pane renders that
  effective pair. The pinned pipeline version and `show --version N` are never
  modified.
- CLI help, `docs/cli.md` and the packaged authoring skill references document
  the flags and the precedence. A TUI override picker remains a documented
  follow-up; the CLI is the deliverable.

Tests: pure precedence/validation table tests, store round-trip and
request-key-mismatch plus migration, server admission/application/overlay/retry
with a controlled executor, CLI flag/help/malformed-value tests, and a TUI
evidence test.
2026-10-16 23:33:53 +01:00

106 lines
3.8 KiB
Go

package pipeline
import (
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
func overrideGraph() protocol.Graph {
return protocol.Graph{Stages: []protocol.Stage{
{Key: "plan", Type: "model", Configs: map[string]string{"provider": "pinned", "model": "pinned-model"}},
{Key: "note", Type: "ntfy", Configs: map[string]string{}},
{Key: "review", Type: "model", Configs: map[string]string{"provider": "pinned2", "model": "pinned2-model"}},
}}
}
// TestEffectiveModelsPrecedence proves the one precedence rule: a per-stage
// override beats the whole-run override, which beats the model pinned in the
// pipeline version, and with no override the pinned pair is returned unchanged.
func TestEffectiveModelsPrecedence(t *testing.T) {
graph := overrideGraph()
cases := []struct {
name string
model string
stageModels map[string]string
want map[string]ModelOverride
}{
{
name: "no override keeps the pinned pair",
want: map[string]ModelOverride{
"plan": {Provider: "pinned", Model: "pinned-model"},
"review": {Provider: "pinned2", Model: "pinned2-model"},
},
},
{
name: "whole-run beats the pinned pair",
model: "acme/whole",
want: map[string]ModelOverride{
"plan": {Provider: "acme", Model: "whole"},
"review": {Provider: "acme", Model: "whole"},
},
},
{
name: "per-stage beats whole-run and leaves the other pinned",
model: "acme/whole",
stageModels: map[string]string{"plan": "acme/stage"},
want: map[string]ModelOverride{
"plan": {Provider: "acme", Model: "stage"},
"review": {Provider: "acme", Model: "whole"},
},
},
{
name: "per-stage alone leaves the other pinned",
stageModels: map[string]string{"plan": "acme/stage"},
want: map[string]ModelOverride{
"plan": {Provider: "acme", Model: "stage"},
"review": {Provider: "pinned2", Model: "pinned2-model"},
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := EffectiveModels(graph, tc.model, tc.stageModels)
if err != nil {
t.Fatal(err)
}
if len(got) != len(tc.want) {
t.Fatalf("effective = %+v, want %+v", got, tc.want)
}
for key, want := range tc.want {
if got[key] != want {
t.Fatalf("stage %s = %+v, want %+v", key, got[key], want)
}
}
})
}
}
// TestEffectiveModelsRejectsBadInput proves an unknown stage key fails naming the
// valid keys, a malformed provider/model fails, and a per-stage key for a
// non-model stage is refused as one that can never honour it.
func TestEffectiveModelsRejectsBadInput(t *testing.T) {
graph := overrideGraph()
if _, err := EffectiveModels(graph, "", map[string]string{"nope": "acme/x"}); err == nil || !strings.Contains(err.Error(), "unknown stage key") || !strings.Contains(err.Error(), "note, plan, review") {
t.Fatalf("unknown key error = %v, want the valid keys named", err)
}
for _, value := range []string{"not-a-pair", "/model", "provider/", "bad provider/model", "provider/bad model"} {
if _, err := EffectiveModels(graph, value, nil); err == nil {
t.Fatalf("whole-run value %q was accepted", value)
}
if _, err := EffectiveModels(graph, "", map[string]string{"plan": value}); err == nil {
t.Fatalf("per-stage value %q was accepted", value)
}
}
if _, err := EffectiveModels(graph, "", map[string]string{"note": "acme/x"}); err == nil || !strings.Contains(err.Error(), "cannot honour") {
t.Fatalf("non-model stage override error = %v, want a cannot-honour refusal", err)
}
// A whole-run override on a graph with no model stage is accepted and inert.
inert := protocol.Graph{Stages: []protocol.Stage{{Key: "note", Type: "ntfy", Configs: map[string]string{}}}}
got, err := EffectiveModels(inert, "acme/whole", nil)
if err != nil || len(got) != 0 {
t.Fatalf("whole-run override on a no-model graph = %+v, %v; want inert", got, err)
}
}