Allow launching runs with whole-run and per-stage model overrides
Add `pipelines run --model PROVIDER/MODEL` and repeatable `--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence per-stage > whole-run > the model pinned in the pipeline version. - A shared pure resolver (`pipeline.ParseModelOverride` / `pipeline.EffectiveModels`) owns the precedence and validation: every value must be a `provider/model` pair (`stage.ValidModel`), every stage key must exist in the pinned graph, an unknown key fails naming the valid keys, and a per-stage key that names a non-model stage is refused. - The server validates against the pinned version before admission (no run row is ever inserted for an invalid request), persists `model`/`stage_models` on the run and `provider`/`model` on each step, applies the effective pair to a clone of the stage definition so the stage and its repair/formatter path share it, overlays the returned run graph with the effective pair (presentation only), and reuses the recorded overrides on retry. - The run record echoes the overrides it was launched with and each step records the provider/model it actually ran with; the TUI Evidence pane renders that effective pair. The pinned pipeline version and `show --version N` are never modified. - CLI help, `docs/cli.md` and the packaged authoring skill references document the flags and the precedence. A TUI override picker remains a documented follow-up; the CLI is the deliverable. Tests: pure precedence/validation table tests, store round-trip and request-key-mismatch plus migration, server admission/application/overlay/retry with a controlled executor, CLI flag/help/malformed-value tests, and a TUI evidence test.
This commit is contained in:
+76
-5
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -43,6 +44,41 @@ type options struct {
|
||||
json, source, noOpen, yes, allow, createOnly, force, wait bool
|
||||
version, expected, limit int
|
||||
timeout, waitTimeout time.Duration
|
||||
stageModels stageModelFlags
|
||||
}
|
||||
|
||||
// stageModelFlags collects the repeatable --stage-model STAGE=PROVIDER/MODEL
|
||||
// values. A repeated key is last-wins. It is a flag.Value so the repeatable flag
|
||||
// needs no dependency (Go stdlib has no slice flag).
|
||||
type stageModelFlags map[string]string
|
||||
|
||||
// String renders the collected overrides for flag's defaults display.
|
||||
func (s stageModelFlags) String() string {
|
||||
keys := make([]string, 0, len(s))
|
||||
for key := range s {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
parts := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
parts = append(parts, key+"="+s[key])
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// Set parses one STAGE=PROVIDER/MODEL value and validates the provider/model with
|
||||
// the shared parser, so a malformed value fails before any run is created.
|
||||
func (s stageModelFlags) Set(value string) error {
|
||||
key, model, ok := strings.Cut(value, "=")
|
||||
key, model = strings.TrimSpace(key), strings.TrimSpace(model)
|
||||
if !ok || key == "" || model == "" {
|
||||
return fmt.Errorf("--stage-model requires STAGE=PROVIDER/MODEL")
|
||||
}
|
||||
if _, err := pipeline.ParseModelOverride(model); err != nil {
|
||||
return fmt.Errorf("--stage-model %s: %w", key, err)
|
||||
}
|
||||
s[key] = model
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
@@ -98,6 +134,9 @@ func (a App) flags(command string) (*flag.FlagSet, *options) {
|
||||
f.StringVar(&o.paths, "write-paths", "", "Optional narrower edit scope; default is all files in the run directory")
|
||||
f.StringVar(&o.key, "request-key", "", "Idempotency key (default: generated unique key)")
|
||||
f.IntVar(&o.version, "version", 0, "Version to run (default: latest)")
|
||||
f.StringVar(&o.model, "model", "", "Override the model for every stage of this run (PROVIDER/MODEL)")
|
||||
o.stageModels = stageModelFlags{}
|
||||
f.Var(o.stageModels, "stage-model", "Override one stage's model; repeatable (STAGE=PROVIDER/MODEL); beats --model")
|
||||
f.BoolVar(&o.wait, "wait", false, "In JSON mode, wait for a terminal run state")
|
||||
f.DurationVar(&o.waitTimeout, "wait-timeout", 0, "Client limit for --wait after the run is admitted (0 waits until terminal); separate from per-request --timeout")
|
||||
case "runs":
|
||||
@@ -278,10 +317,11 @@ func (a App) Run(ctx context.Context, args []string) error {
|
||||
if len(pos) != 0 {
|
||||
return fmt.Errorf("provider-check takes no positional arguments")
|
||||
}
|
||||
provider, model, ok := strings.Cut(o.model, "/")
|
||||
if !ok || !stage.ValidModel(provider, model) {
|
||||
override, err := pipeline.ParseModelOverride(o.model)
|
||||
if err != nil {
|
||||
return fmt.Errorf("--model requires provider/model")
|
||||
}
|
||||
provider, model := override.Provider, override.Model
|
||||
if err := a.confirmAction("Validate "+o.model+" with one small tool-free generation (provider charges may apply)?", o.yes); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1117,6 +1157,14 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
|
||||
paths = append(paths, p)
|
||||
}
|
||||
}
|
||||
// Pre-flight form validation of the whole-run override with the same parser
|
||||
// the server uses; stage-key existence stays server-authoritative. A malformed
|
||||
// value fails here, before any confirmation or run request.
|
||||
if o.model != "" {
|
||||
if _, err := pipeline.ParseModelOverride(o.model); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if o.yes && (o.name == "" || strings.TrimSpace(o.prompt) == "") {
|
||||
return fmt.Errorf("--yes requires a pipeline and nonempty prompt")
|
||||
}
|
||||
@@ -1131,7 +1179,7 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
|
||||
if o.wait || o.key != "" || o.version > 0 {
|
||||
return fmt.Errorf("--wait, --request-key and --version require --json")
|
||||
}
|
||||
_, err := tea.NewProgram(tui.NewRun(ctx, o.url, tui.RunOptions{Pipeline: o.name, Prompt: o.prompt, Title: o.title, Workspace: workspace, Yes: o.yes, AllowTools: o.allow, WritePaths: paths}), tea.WithContext(ctx), tea.WithAltScreen()).Run()
|
||||
_, err := tea.NewProgram(tui.NewRun(ctx, o.url, tui.RunOptions{Pipeline: o.name, Prompt: o.prompt, Title: o.title, Workspace: workspace, Yes: o.yes, AllowTools: o.allow, WritePaths: paths, Model: o.model, StageModels: o.stageModels}), tea.WithContext(ctx), tea.WithAltScreen()).Run()
|
||||
return err
|
||||
}
|
||||
if o.name == "" || strings.TrimSpace(o.prompt) == "" {
|
||||
@@ -1145,20 +1193,43 @@ func (a App) run(ctx context.Context, pos []string, o *options, call callFunc, r
|
||||
if version == 0 {
|
||||
version = p.Version
|
||||
}
|
||||
if err := a.confirmAction(fmt.Sprintf("Run %s v%d in %s? This may edit files, execute commands and publish notifications.", p.Name, version, workspace), o.yes); err != nil {
|
||||
confirmation := fmt.Sprintf("Run %s v%d in %s? This may edit files, execute commands and publish notifications.", p.Name, version, workspace) + o.modelSummary()
|
||||
if err := a.confirmAction(confirmation, o.yes); err != nil {
|
||||
return err
|
||||
}
|
||||
key := o.key
|
||||
if key == "" {
|
||||
key = fmt.Sprintf("cli-%d-%d", os.Getpid(), time.Now().UnixNano())
|
||||
}
|
||||
r, err := call(protocol.Message{Type: protocol.RunPipeline, PipelineID: p.ID, Version: version, RequestKey: key, Workspace: workspace, Title: o.title, Input: map[string]any{"prompt": o.prompt}, AllowTools: o.allow, WritePaths: paths})
|
||||
r, err := call(protocol.Message{Type: protocol.RunPipeline, PipelineID: p.ID, Version: version, RequestKey: key, Workspace: workspace, Title: o.title, Input: map[string]any{"prompt": o.prompt}, AllowTools: o.allow, WritePaths: paths, Model: o.model, StageModels: o.stageModels})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return a.finishRun(ctx, r, o, call)
|
||||
}
|
||||
|
||||
// modelSummary describes the launch-time model overrides for the action
|
||||
// confirmation, so a model mutation is visible before approval. It is empty when
|
||||
// no override was supplied.
|
||||
func (o *options) modelSummary() string {
|
||||
if o.model == "" && len(o.stageModels) == 0 {
|
||||
return ""
|
||||
}
|
||||
parts := []string{}
|
||||
if o.model != "" {
|
||||
parts = append(parts, "all stages: "+o.model)
|
||||
}
|
||||
keys := make([]string, 0, len(o.stageModels))
|
||||
for key := range o.stageModels {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
parts = append(parts, key+": "+o.stageModels[key])
|
||||
}
|
||||
return " Model overrides — " + strings.Join(parts, "; ") + "."
|
||||
}
|
||||
|
||||
func (a App) finishRun(ctx context.Context, r protocol.Message, o *options, call callFunc) error {
|
||||
var err error
|
||||
if o.wait {
|
||||
|
||||
@@ -16,7 +16,7 @@ var usage = map[string]string{
|
||||
"import": "import [FILE.zip] [--all] [--on-conflict ask|keep|both|import] [--yes] [--json]",
|
||||
"templates": "templates [--json]",
|
||||
"template": "template <key> [--source | --json]",
|
||||
"run": "run [name|id] [prompt] [--title TITLE] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"run": "run [name|id] [prompt] [--title TITLE] [--model PROVIDER/MODEL] [--stage-model STAGE=PROVIDER/MODEL ...] [--json [--wait [--wait-timeout DURATION]]]",
|
||||
"runs": "runs [--pipeline NAME] [--json]",
|
||||
"inspect": "inspect <run-id> [--json]",
|
||||
"cancel": "cancel <run-id> [--json]",
|
||||
@@ -44,7 +44,7 @@ var notes = map[string]string{
|
||||
"edit": "Omit the name to open fuzzy selection. Original immutable source is copied to a private draft. The configured harness edits that draft, never the live definition.",
|
||||
"validate": "Uses the same server parser as submission. Does not persist or activate schedules. Structural validity does not prove runtime/tool availability.",
|
||||
"submit": "Confirm publication at the prompt, or use --yes for explicit automation; schedules activate on acceptance. Managed drafts provide the name and base-version guard. --force never bypasses validation.",
|
||||
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
|
||||
"run": "Runs in your current directory after action confirmation (--yes skips it). Select a pipeline and compose a multiline task with Ctrl+S. Stage tools are enabled automatically; no file list or typed RUN confirmation. Cancellation does not undo edits. --title sets an explicit run title; without it the title is derived from the prompt. --model PROVIDER/MODEL overrides the model for every stage of that run; --stage-model STAGE=PROVIDER/MODEL overrides one stage and is repeatable, with a repeated key last-wins. Precedence is per-stage > whole-run > the model pinned in the pipeline version; overrides apply to that run only and never change the pinned definition. An unknown stage key or a malformed provider/model is refused before the run starts, naming the valid stage keys. In JSON mode --wait polls until terminal; --wait-timeout bounds only the client wait after admission and is independent of per-request --timeout. A reached wait limit leaves the run active: inspect read-only with `inspect <run-id> --json`, or reattach with --wait reusing the original --request-key (never an automatic replay or cancel).",
|
||||
"retry": "Start a new attempt of a failed run with the same initial input, directory, pipeline version, and edit scope. Original history is retained. Already-completed side effects are not undone. --wait/--wait-timeout behave as for run: a reached wait limit leaves the new run active; inspect read-only with `inspect <run-id> --json`, or reattach with --wait and the original --request-key instead of starting a duplicate retry.",
|
||||
"title": "Rename a run's operator-facing title after the fact without touching its input, steps or execution. The title is collapsed to a single bounded line and recorded as explicitly set. `pipelines runs` and `pipelines inspect` then show it.",
|
||||
"decide": "Choose an option for a run waiting on a human stage. The run must be `waiting`; the option must be one of that stage's non-error edges. The stage is inferred from the run's current stage unless named. The choice and optional --feedback are recorded as the step's evidence (visible in inspect and the TUI) and the run resumes along the chosen edge. An unknown option is refused with the valid list; a repeated decision for an already-decided visit does not double-apply. Works across a server restart because the wait is persisted.",
|
||||
|
||||
@@ -61,6 +61,13 @@ Examples:
|
||||
pipelines validate /path/to/draft.md --json
|
||||
pipelines submit /path/to/draft.md
|
||||
pipelines run "bug fix" "Fix the layout" --json
|
||||
pipelines run "bug fix" "Fix the layout" --model openai/gpt-5 --stage-model review=openai/o4 --json
|
||||
|
||||
Launch-time model overrides: 'pipelines run --model P/M' covers every model stage;
|
||||
'--stage-model STAGE=P/M' (repeatable; a repeated key is last-wins) overrides one
|
||||
stage and wins over --model, which wins over the model pinned in the pipeline
|
||||
version. Overrides apply to that run only, are recorded on the run, and are
|
||||
validated before the run starts (an unknown stage key names the valid keys).
|
||||
|
||||
Draft-first: the configured harness (Pi/OpenCode; host by default, or Docker-linked
|
||||
when a checkout is linked) edits a private draft; nvim remains a client editor.
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.a-wan.net/alan/pipelines/internal/protocol"
|
||||
)
|
||||
|
||||
// TestRunModelOverrideFlagsParse proves the repeatable --stage-model flag and the
|
||||
// whole-run --model flag parse onto options, with a repeated key last-wins.
|
||||
func TestRunModelOverrideFlagsParse(t *testing.T) {
|
||||
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
||||
f, o := app.flags("run")
|
||||
if err := parse(f, []string{"--pipeline", "p", "--prompt", "task", "--model", "acme/whole", "--stage-model", "one=acme/first", "--stage-model", "two=acme/two", "--stage-model", "one=acme/last"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if o.model != "acme/whole" {
|
||||
t.Fatalf("--model = %q", o.model)
|
||||
}
|
||||
if o.stageModels["one"] != "acme/last" {
|
||||
t.Fatalf("repeated --stage-model key not last-wins: %+v", o.stageModels)
|
||||
}
|
||||
if o.stageModels["two"] != "acme/two" {
|
||||
t.Fatalf("second --stage-model missing: %+v", o.stageModels)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStageModelFlagRejectsMalformedValues proves a value that is not
|
||||
// STAGE=PROVIDER/MODEL fails at flag parse, before any run is created.
|
||||
func TestStageModelFlagRejectsMalformedValues(t *testing.T) {
|
||||
for _, value := range []string{"nope", "one", "=acme/x", "one=", "one=not-a-pair", "one=acme/"} {
|
||||
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
||||
f, _ := app.flags("run")
|
||||
if err := parse(f, []string{"--stage-model", value}); err == nil {
|
||||
t.Fatalf("malformed --stage-model %q was accepted", value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestRunRejectsInvalidWholeRunModel proves the whole-run --model is form-checked
|
||||
// with the shared parser before any server request.
|
||||
func TestRunRejectsInvalidWholeRunModel(t *testing.T) {
|
||||
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
||||
called := false
|
||||
call := func(protocol.Message) (protocol.Message, error) { called = true; return protocol.Message{}, nil }
|
||||
resolve := func(string) (protocol.PipelineSummary, error) { return protocol.PipelineSummary{}, nil }
|
||||
o := &options{json: true, yes: true, model: "not-a-pair", prompt: "task"}
|
||||
err := app.run(context.Background(), []string{"pipe"}, o, call, resolve)
|
||||
if err == nil || !strings.Contains(err.Error(), "provider/model") {
|
||||
t.Fatalf("invalid --model error = %v", err)
|
||||
}
|
||||
if called {
|
||||
t.Fatal("an invalid --model still sent a run request")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHelpDocumentsModelOverrides proves the usage map, notes and flag help
|
||||
// advertise --model and repeatable --stage-model and state the precedence.
|
||||
func TestHelpDocumentsModelOverrides(t *testing.T) {
|
||||
if !strings.Contains(usage["run"], "--model") || !strings.Contains(usage["run"], "--stage-model") {
|
||||
t.Fatalf("run usage does not document the model overrides: %q", usage["run"])
|
||||
}
|
||||
note := notes["run"]
|
||||
for _, want := range []string{"--model", "--stage-model", "per-stage", "pinned"} {
|
||||
if !strings.Contains(note, want) {
|
||||
t.Fatalf("run note does not document %q: %q", want, note)
|
||||
}
|
||||
}
|
||||
app := App{Out: &bytes.Buffer{}, Err: &bytes.Buffer{}}
|
||||
f, _ := app.flags("run")
|
||||
f.Usage()
|
||||
help := app.Out.(*bytes.Buffer).String()
|
||||
if !strings.Contains(help, "--model") || !strings.Contains(help, "--stage-model") {
|
||||
t.Fatalf("run --help omits the overrides:\n%s", help)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user