fae1005962
Add `pipelines run --model PROVIDER/MODEL` and repeatable `--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence per-stage > whole-run > the model pinned in the pipeline version. - A shared pure resolver (`pipeline.ParseModelOverride` / `pipeline.EffectiveModels`) owns the precedence and validation: every value must be a `provider/model` pair (`stage.ValidModel`), every stage key must exist in the pinned graph, an unknown key fails naming the valid keys, and a per-stage key that names a non-model stage is refused. - The server validates against the pinned version before admission (no run row is ever inserted for an invalid request), persists `model`/`stage_models` on the run and `provider`/`model` on each step, applies the effective pair to a clone of the stage definition so the stage and its repair/formatter path share it, overlays the returned run graph with the effective pair (presentation only), and reuses the recorded overrides on retry. - The run record echoes the overrides it was launched with and each step records the provider/model it actually ran with; the TUI Evidence pane renders that effective pair. The pinned pipeline version and `show --version N` are never modified. - CLI help, `docs/cli.md` and the packaged authoring skill references document the flags and the precedence. A TUI override picker remains a documented follow-up; the CLI is the deliverable. Tests: pure precedence/validation table tests, store round-trip and request-key-mismatch plus migration, server admission/application/overlay/retry with a controlled executor, CLI flag/help/malformed-value tests, and a TUI evidence test.
106 lines
3.8 KiB
Go
106 lines
3.8 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
func overrideGraph() protocol.Graph {
|
|
return protocol.Graph{Stages: []protocol.Stage{
|
|
{Key: "plan", Type: "model", Configs: map[string]string{"provider": "pinned", "model": "pinned-model"}},
|
|
{Key: "note", Type: "ntfy", Configs: map[string]string{}},
|
|
{Key: "review", Type: "model", Configs: map[string]string{"provider": "pinned2", "model": "pinned2-model"}},
|
|
}}
|
|
}
|
|
|
|
// TestEffectiveModelsPrecedence proves the one precedence rule: a per-stage
|
|
// override beats the whole-run override, which beats the model pinned in the
|
|
// pipeline version, and with no override the pinned pair is returned unchanged.
|
|
func TestEffectiveModelsPrecedence(t *testing.T) {
|
|
graph := overrideGraph()
|
|
cases := []struct {
|
|
name string
|
|
model string
|
|
stageModels map[string]string
|
|
want map[string]ModelOverride
|
|
}{
|
|
{
|
|
name: "no override keeps the pinned pair",
|
|
want: map[string]ModelOverride{
|
|
"plan": {Provider: "pinned", Model: "pinned-model"},
|
|
"review": {Provider: "pinned2", Model: "pinned2-model"},
|
|
},
|
|
},
|
|
{
|
|
name: "whole-run beats the pinned pair",
|
|
model: "acme/whole",
|
|
want: map[string]ModelOverride{
|
|
"plan": {Provider: "acme", Model: "whole"},
|
|
"review": {Provider: "acme", Model: "whole"},
|
|
},
|
|
},
|
|
{
|
|
name: "per-stage beats whole-run and leaves the other pinned",
|
|
model: "acme/whole",
|
|
stageModels: map[string]string{"plan": "acme/stage"},
|
|
want: map[string]ModelOverride{
|
|
"plan": {Provider: "acme", Model: "stage"},
|
|
"review": {Provider: "acme", Model: "whole"},
|
|
},
|
|
},
|
|
{
|
|
name: "per-stage alone leaves the other pinned",
|
|
stageModels: map[string]string{"plan": "acme/stage"},
|
|
want: map[string]ModelOverride{
|
|
"plan": {Provider: "acme", Model: "stage"},
|
|
"review": {Provider: "pinned2", Model: "pinned2-model"},
|
|
},
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got, err := EffectiveModels(graph, tc.model, tc.stageModels)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(got) != len(tc.want) {
|
|
t.Fatalf("effective = %+v, want %+v", got, tc.want)
|
|
}
|
|
for key, want := range tc.want {
|
|
if got[key] != want {
|
|
t.Fatalf("stage %s = %+v, want %+v", key, got[key], want)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestEffectiveModelsRejectsBadInput proves an unknown stage key fails naming the
|
|
// valid keys, a malformed provider/model fails, and a per-stage key for a
|
|
// non-model stage is refused as one that can never honour it.
|
|
func TestEffectiveModelsRejectsBadInput(t *testing.T) {
|
|
graph := overrideGraph()
|
|
if _, err := EffectiveModels(graph, "", map[string]string{"nope": "acme/x"}); err == nil || !strings.Contains(err.Error(), "unknown stage key") || !strings.Contains(err.Error(), "note, plan, review") {
|
|
t.Fatalf("unknown key error = %v, want the valid keys named", err)
|
|
}
|
|
for _, value := range []string{"not-a-pair", "/model", "provider/", "bad provider/model", "provider/bad model"} {
|
|
if _, err := EffectiveModels(graph, value, nil); err == nil {
|
|
t.Fatalf("whole-run value %q was accepted", value)
|
|
}
|
|
if _, err := EffectiveModels(graph, "", map[string]string{"plan": value}); err == nil {
|
|
t.Fatalf("per-stage value %q was accepted", value)
|
|
}
|
|
}
|
|
if _, err := EffectiveModels(graph, "", map[string]string{"note": "acme/x"}); err == nil || !strings.Contains(err.Error(), "cannot honour") {
|
|
t.Fatalf("non-model stage override error = %v, want a cannot-honour refusal", err)
|
|
}
|
|
// A whole-run override on a graph with no model stage is accepted and inert.
|
|
inert := protocol.Graph{Stages: []protocol.Stage{{Key: "note", Type: "ntfy", Configs: map[string]string{}}}}
|
|
got, err := EffectiveModels(inert, "acme/whole", nil)
|
|
if err != nil || len(got) != 0 {
|
|
t.Fatalf("whole-run override on a no-model graph = %+v, %v; want inert", got, err)
|
|
}
|
|
}
|