fae1005962
Add `pipelines run --model PROVIDER/MODEL` and repeatable `--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence per-stage > whole-run > the model pinned in the pipeline version. - A shared pure resolver (`pipeline.ParseModelOverride` / `pipeline.EffectiveModels`) owns the precedence and validation: every value must be a `provider/model` pair (`stage.ValidModel`), every stage key must exist in the pinned graph, an unknown key fails naming the valid keys, and a per-stage key that names a non-model stage is refused. - The server validates against the pinned version before admission (no run row is ever inserted for an invalid request), persists `model`/`stage_models` on the run and `provider`/`model` on each step, applies the effective pair to a clone of the stage definition so the stage and its repair/formatter path share it, overlays the returned run graph with the effective pair (presentation only), and reuses the recorded overrides on retry. - The run record echoes the overrides it was launched with and each step records the provider/model it actually ran with; the TUI Evidence pane renders that effective pair. The pinned pipeline version and `show --version N` are never modified. - CLI help, `docs/cli.md` and the packaged authoring skill references document the flags and the precedence. A TUI override picker remains a documented follow-up; the CLI is the deliverable. Tests: pure precedence/validation table tests, store round-trip and request-key-mismatch plus migration, server admission/application/overlay/retry with a controlled executor, CLI flag/help/malformed-value tests, and a TUI evidence test.
80 lines
3.1 KiB
Go
80 lines
3.1 KiB
Go
package cli
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// TestRunModelOverrideFlagsParse proves the repeatable --stage-model flag and the
|
|
// whole-run --model flag parse onto options, with a repeated key last-wins.
|
|
func TestRunModelOverrideFlagsParse(t *testing.T) {
|
|
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
|
f, o := app.flags("run")
|
|
if err := parse(f, []string{"--pipeline", "p", "--prompt", "task", "--model", "acme/whole", "--stage-model", "one=acme/first", "--stage-model", "two=acme/two", "--stage-model", "one=acme/last"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if o.model != "acme/whole" {
|
|
t.Fatalf("--model = %q", o.model)
|
|
}
|
|
if o.stageModels["one"] != "acme/last" {
|
|
t.Fatalf("repeated --stage-model key not last-wins: %+v", o.stageModels)
|
|
}
|
|
if o.stageModels["two"] != "acme/two" {
|
|
t.Fatalf("second --stage-model missing: %+v", o.stageModels)
|
|
}
|
|
}
|
|
|
|
// TestStageModelFlagRejectsMalformedValues proves a value that is not
|
|
// STAGE=PROVIDER/MODEL fails at flag parse, before any run is created.
|
|
func TestStageModelFlagRejectsMalformedValues(t *testing.T) {
|
|
for _, value := range []string{"nope", "one", "=acme/x", "one=", "one=not-a-pair", "one=acme/"} {
|
|
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
|
f, _ := app.flags("run")
|
|
if err := parse(f, []string{"--stage-model", value}); err == nil {
|
|
t.Fatalf("malformed --stage-model %q was accepted", value)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRunRejectsInvalidWholeRunModel proves the whole-run --model is form-checked
|
|
// with the shared parser before any server request.
|
|
func TestRunRejectsInvalidWholeRunModel(t *testing.T) {
|
|
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
|
called := false
|
|
call := func(protocol.Message) (protocol.Message, error) { called = true; return protocol.Message{}, nil }
|
|
resolve := func(string) (protocol.PipelineSummary, error) { return protocol.PipelineSummary{}, nil }
|
|
o := &options{json: true, yes: true, model: "not-a-pair", prompt: "task"}
|
|
err := app.run(context.Background(), []string{"pipe"}, o, call, resolve)
|
|
if err == nil || !strings.Contains(err.Error(), "provider/model") {
|
|
t.Fatalf("invalid --model error = %v", err)
|
|
}
|
|
if called {
|
|
t.Fatal("an invalid --model still sent a run request")
|
|
}
|
|
}
|
|
|
|
// TestHelpDocumentsModelOverrides proves the usage map, notes and flag help
|
|
// advertise --model and repeatable --stage-model and state the precedence.
|
|
func TestHelpDocumentsModelOverrides(t *testing.T) {
|
|
if !strings.Contains(usage["run"], "--model") || !strings.Contains(usage["run"], "--stage-model") {
|
|
t.Fatalf("run usage does not document the model overrides: %q", usage["run"])
|
|
}
|
|
note := notes["run"]
|
|
for _, want := range []string{"--model", "--stage-model", "per-stage", "pinned"} {
|
|
if !strings.Contains(note, want) {
|
|
t.Fatalf("run note does not document %q: %q", want, note)
|
|
}
|
|
}
|
|
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
|
f, _ := app.flags("run")
|
|
f.Usage()
|
|
help := app.Out.(*bytes.Buffer).String()
|
|
if !strings.Contains(help, "--model") || !strings.Contains(help, "--stage-model") {
|
|
t.Fatalf("run --help omits the overrides:\n%s", help)
|
|
}
|
|
}
|