Allow launching runs with whole-run and per-stage model overrides

Add `pipelines run --model PROVIDER/MODEL` and repeatable
`--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence
per-stage > whole-run > the model pinned in the pipeline version.

- A shared pure resolver (`pipeline.ParseModelOverride` /
  `pipeline.EffectiveModels`) owns the precedence and validation: every value
  must be a `provider/model` pair (`stage.ValidModel`), every stage key must
  exist in the pinned graph, an unknown key fails naming the valid keys, and a
  per-stage key that names a non-model stage is refused.
- The server validates against the pinned version before admission (no run row
  is ever inserted for an invalid request), persists `model`/`stage_models` on
  the run and `provider`/`model` on each step, applies the effective pair to a
  clone of the stage definition so the stage and its repair/formatter path share
  it, overlays the returned run graph with the effective pair (presentation
  only), and reuses the recorded overrides on retry.
- The run record echoes the overrides it was launched with and each step records
  the provider/model it actually ran with; the TUI Evidence pane renders that
  effective pair. The pinned pipeline version and `show --version N` are never
  modified.
- CLI help, `docs/cli.md` and the packaged authoring skill references document
  the flags and the precedence. A TUI override picker remains a documented
  follow-up; the CLI is the deliverable.

Tests: pure precedence/validation table tests, store round-trip and
request-key-mismatch plus migration, server admission/application/overlay/retry
with a controlled executor, CLI flag/help/malformed-value tests, and a TUI
evidence test.
This commit is contained in:
2026-10-16 23:33:53 +01:00
parent 80232a4ffe
commit fae1005962
19 changed files with 912 additions and 31 deletions
+76 -5
View File
@@ -10,6 +10,7 @@ import (
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
@@ -43,6 +44,41 @@ type options struct {
json, source, noOpen, yes, allow, createOnly, force, wait bool
version, expected, limit int
timeout, waitTimeout time.Duration
stageModels stageModelFlags
}
// stageModelFlags collects the repeatable --stage-model STAGE=PROVIDER/MODEL
// values. A repeated key is last-wins. It is a flag.Value so the repeatable flag
// needs no dependency (Go stdlib has no slice flag).
type stageModelFlags map[string]string
// String renders the collected overrides for flag's defaults display.
func (s stageModelFlags) String() string {
keys := make([]string, 0, len(s))
for key := range s {
keys = append(keys, key)
}
sort.Strings(keys)
parts := make([]string, 0, len(keys))
for _, key := range keys {
parts = append(parts, key+"="+s[key])
}
return strings.Join(parts, ",")
}
// Set parses one STAGE=PROVIDER/MODEL value and validates the provider/model with
// the shared parser, so a malformed value fails before any run is created.
func (s stageModelFlags) Set(value string) error {
key, model, ok := strings.Cut(value, "=")
key, model = strings.TrimSpace(key), strings.TrimSpace(model)
if !ok || key == "" || model == "" {
return fmt.Errorf("--stage-model requires STAGE=PROVIDER/MODEL")
}
if _, err := pipeline.ParseModelOverride(model); err != nil {
return fmt.Errorf("--stage-model %s: %w", key, err)
}
s[key] = model
return nil
}
func (a App) flags(command string) (*flag.FlagSet, *options) {
@@ -98,6 +134,9 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
f.StringVar(&o.paths, "write-paths", "", "Optional narrower edit scope; default is all files in the run directory")
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
f.IntVar(&o.version, "version", 0, "Version to run (default: latest)")
f.StringVar(&o.model, "model", "", "Override the model for every stage of this run (PROVIDER/MODEL)")
o.stageModels = stageModelFlags{}
f.Var(o.stageModels, "stage-model", "Override one stage's model; repeatable (STAGE=PROVIDER/MODEL); beats --model")
f.BoolVar(&o.wait, "wait", false, "In JSON mode, wait for a terminal run state")
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait after the run is admitted (0 waits until terminal); separate from per-request --timeout")
case "runs":
@@ -278,10 +317,11 @@ func (a App) Run(ctx context.Context, args []string) error {
if len(pos) != 0 {
return fmt.Errorf("provider-check takes no positional arguments")
}
provider, model, ok := strings.Cut(o.model, "/")
if !ok || !stage.ValidModel(provider, model) {
override, err := pipeline.ParseModelOverride(o.model)
if err != nil {
return fmt.Errorf("--model requires provider/model")
}
provider, model := override.Provider, override.Model
if err := a.confirmAction("Validate "+o.model+" with one small tool-free generation (provider charges may apply)?", o.yes); err != nil {
return err
}
@@ -1117,6 +1157,14 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
paths = append(paths, p)
}
}
// Pre-flight form validation of the whole-run override with the same parser
// the server uses; stage-key existence stays server-authoritative. A malformed
// value fails here, before any confirmation or run request.
if o.model != "" {
if _, err := pipeline.ParseModelOverride(o.model); err != nil {
return err
}
}
if o.yes && (o.name == "" || strings.TrimSpace(o.prompt) == "") {
return fmt.Errorf("--yes requires a pipeline and nonempty prompt")
}
@@ -1131,7 +1179,7 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
if o.wait || o.key != "" || o.version > 0 {
return fmt.Errorf("--wait, --request-key and --version require --json")
}
_, err := tea.NewProgram(tui.NewRun(ctx, o.url, tui.RunOptions{Pipeline: o.name, Prompt: o.prompt, Title: o.title, Workspace: workspace, Yes: o.yes, AllowTools: o.allow, WritePaths: paths}), tea.WithContext(ctx), tea.WithAltScreen()).Run()
_, err := tea.NewProgram(tui.NewRun(ctx, o.url, tui.RunOptions{Pipeline: o.name, Prompt: o.prompt, Title: o.title, Workspace: workspace, Yes: o.yes, AllowTools: o.allow, WritePaths: paths, Model: o.model, StageModels: o.stageModels}), tea.WithContext(ctx), tea.WithAltScreen()).Run()
return err
}
if o.name == "" || strings.TrimSpace(o.prompt) == "" {
@@ -1145,20 +1193,43 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
if version == 0 {
version = p.Version
}
if err := a.confirmAction(fmt.Sprintf("Run %s v%d in %s? This may edit files, execute commands and publish notifications.", p.Name, version, workspace), o.yes); err != nil {
confirmation := fmt.Sprintf("Run %s v%d in %s? This may edit files, execute commands and publish notifications.", p.Name, version, workspace) + o.modelSummary()
if err := a.confirmAction(confirmation, o.yes); err != nil {
return err
}
key := o.key
if key == "" {
key = fmt.Sprintf("cli-%d-%d", os.Getpid(), time.Now().UnixNano())
}
r, err := call(protocol.Message{Type: protocol.RunPipeline, PipelineID: p.ID, Version: version, RequestKey: key, Workspace: workspace, Title: o.title, Input: map[string]any{"prompt": o.prompt}, AllowTools: o.allow, WritePaths: paths})
r, err := call(protocol.Message{Type: protocol.RunPipeline, PipelineID: p.ID, Version: version, RequestKey: key, Workspace: workspace, Title: o.title, Input: map[string]any{"prompt": o.prompt}, AllowTools: o.allow, WritePaths: paths, Model: o.model, StageModels: o.stageModels})
if err != nil {
return err
}
return a.finishRun(ctx, r, o, call)
}
// modelSummary describes the launch-time model overrides for the action
// confirmation, so a model mutation is visible before approval. It is empty when
// no override was supplied.
func (o *options) modelSummary() string {
if o.model == "" && len(o.stageModels) == 0 {
return ""
}
parts := []string{}
if o.model != "" {
parts = append(parts, "all stages: "+o.model)
}
keys := make([]string, 0, len(o.stageModels))
for key := range o.stageModels {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
parts = append(parts, key+": "+o.stageModels[key])
}
return " Model overrides — " + strings.Join(parts, "; ") + "."
}
func (a App) finishRun(ctx context.Context, r protocol.Message, o *options, call callFunc) error {
var err error
if o.wait {
+2 -2
View File
@@ -16,7 +16,7 @@ var usage = map[string]string{
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
"templates": "templates [--json]",
"template": "template <key> [--source | --json]",
"run": "run [name|id] [prompt] [--title TITLE] [--json [--wait [--wait-timeout DURATION]]]",
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
"runs": "runs [--pipeline NAME] [--json]",
"inspect": "inspect <run-id> [--json]",
"cancel": "cancel <run-id> [--json]",
@@ -44,7 +44,7 @@ var notes = map[string]string{
"edit": "Omit the name to open fuzzy selection. Original immutable source is copied to a private draft. The configured harness edits that draft, never the live definition.",
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
"retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
+7
View File
@@ -61,6 +61,13 @@ Examples:
pipelines validate /path/to/draft.md --json
pipelines submit /path/to/draft.md
pipelines run "bug fix" "Fix the layout" --json
pipelines run "bug fix" "Fix the layout" --model openai/gpt-5 --stage-model review=openai/o4 --json
Launch-time model overrides: 'pipelines run --model P/M' covers every model stage;
'--stage-model STAGE=P/M' (repeatable; a repeated key is last-wins) overrides one
stage and wins over --model, which wins over the model pinned in the pipeline
version. Overrides apply to that run only, are recorded on the run, and are
validated before the run starts (an unknown stage key names the valid keys).
Draft-first: the configured harness (Pi/OpenCode; host by default, or Docker-linked
when a checkout is linked) edits a private draft; nvim remains a client editor.
+79
View File
@@ -0,0 +1,79 @@
package cli
import (
"bytes"
"context"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// TestRunModelOverrideFlagsParse proves the repeatable --stage-model flag and the
// whole-run --model flag parse onto options, with a repeated key last-wins.
func TestRunModelOverrideFlagsParse(t *testing.T) {
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
f, o := app.flags("run")
if err := parse(f, []string{"--pipeline", "p", "--prompt", "task", "--model", "acme/whole", "--stage-model", "one=acme/first", "--stage-model", "two=acme/two", "--stage-model", "one=acme/last"}); err != nil {
t.Fatal(err)
}
if o.model != "acme/whole" {
t.Fatalf("--model = %q", o.model)
}
if o.stageModels["one"] != "acme/last" {
t.Fatalf("repeated --stage-model key not last-wins: %+v", o.stageModels)
}
if o.stageModels["two"] != "acme/two" {
t.Fatalf("second --stage-model missing: %+v", o.stageModels)
}
}
// TestStageModelFlagRejectsMalformedValues proves a value that is not
// STAGE=PROVIDER/MODEL fails at flag parse, before any run is created.
func TestStageModelFlagRejectsMalformedValues(t *testing.T) {
for _, value := range []string{"nope", "one", "=acme/x", "one=", "one=not-a-pair", "one=acme/"} {
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
f, _ := app.flags("run")
if err := parse(f, []string{"--stage-model", value}); err == nil {
t.Fatalf("malformed --stage-model %q was accepted", value)
}
}
}
// TestRunRejectsInvalidWholeRunModel proves the whole-run --model is form-checked
// with the shared parser before any server request.
func TestRunRejectsInvalidWholeRunModel(t *testing.T) {
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
called := false
call := func(protocol.Message) (protocol.Message, error) { called = true; return protocol.Message{}, nil }
resolve := func(string) (protocol.PipelineSummary, error) { return protocol.PipelineSummary{}, nil }
o := &options{json: true, yes: true, model: "not-a-pair", prompt: "task"}
err := app.run(context.Background(), []string{"pipe"}, o, call, resolve)
if err == nil || !strings.Contains(err.Error(), "provider/model") {
t.Fatalf("invalid --model error = %v", err)
}
if called {
t.Fatal("an invalid --model still sent a run request")
}
}
// TestHelpDocumentsModelOverrides proves the usage map, notes and flag help
// advertise --model and repeatable --stage-model and state the precedence.
func TestHelpDocumentsModelOverrides(t *testing.T) {
if !strings.Contains(usage["run"], "--model") || !strings.Contains(usage["run"], "--stage-model") {
t.Fatalf("run usage does not document the model overrides: %q", usage["run"])
}
note := notes["run"]
for _, want := range []string{"--model", "--stage-model", "per-stage", "pinned"} {
if !strings.Contains(note, want) {
t.Fatalf("run note does not document %q: %q", want, note)
}
}
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
f, _ := app.flags("run")
f.Usage()
help := app.Out.(*bytes.Buffer).String()
if !strings.Contains(help, "--model") || !strings.Contains(help, "--stage-model") {
t.Fatalf("run --help omits the overrides:\n%s", help)
}
}