fae1005962
Add `pipelines run --model PROVIDER/MODEL` and repeatable `--stage-model STAGE=PROVIDER/MODEL`, resolved with precedence per-stage > whole-run > the model pinned in the pipeline version. - A shared pure resolver (`pipeline.ParseModelOverride` / `pipeline.EffectiveModels`) owns the precedence and validation: every value must be a `provider/model` pair (`stage.ValidModel`), every stage key must exist in the pinned graph, an unknown key fails naming the valid keys, and a per-stage key that names a non-model stage is refused. - The server validates against the pinned version before admission (no run row is ever inserted for an invalid request), persists `model`/`stage_models` on the run and `provider`/`model` on each step, applies the effective pair to a clone of the stage definition so the stage and its repair/formatter path share it, overlays the returned run graph with the effective pair (presentation only), and reuses the recorded overrides on retry. - The run record echoes the overrides it was launched with and each step records the provider/model it actually ran with; the TUI Evidence pane renders that effective pair. The pinned pipeline version and `show --version N` are never modified. - CLI help, `docs/cli.md` and the packaged authoring skill references document the flags and the precedence. A TUI override picker remains a documented follow-up; the CLI is the deliverable. Tests: pure precedence/validation table tests, store round-trip and request-key-mismatch plus migration, server admission/application/overlay/retry with a controlled executor, CLI flag/help/malformed-value tests, and a TUI evidence test.
449 lines
16 KiB
Go
449 lines
16 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"path/filepath"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/summarise"
|
|
)
|
|
|
|
func TestRunListIncludesHistoryBeyondFiftyForSearchAndPipelineScope(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "runs.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "history", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 55; i++ {
|
|
if _, _, err := store.CreateRun(ctx, definition, fmt.Sprintf("history-%d", i), "workspace", false, nil, nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 55 || runs[len(runs)-1].ID != 1 {
|
|
t.Fatal("older searchable runs missing", len(runs), err)
|
|
}
|
|
}
|
|
|
|
// TestRunTitleStorageMigrationAndFallback proves the three title contracts: an
|
|
// explicit title is kept verbatim, an absent one is derived bounded and
|
|
// non-empty, and runs created before the title columns survive the migration and
|
|
// still list.
|
|
func TestRunTitleStorageMigrationAndFallback(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "titles.db")
|
|
store, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, "titled", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
|
|
explicit, _, err := store.CreateRunTitled(ctx, definition, "explicit", "workspace", false, nil,
|
|
map[string]any{"prompt": "a prompt that must not become the title"}, RunOverrides{},
|
|
summarise.Title{Text: "Fix the flaky login test", Source: summarise.SourceModel, Model: "openai/gpt"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := store.GetRun(ctx, explicit)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Title != "Fix the flaky login test" || got.TitleSource != summarise.SourceModel || got.TitleModel != "openai/gpt" {
|
|
t.Fatalf("explicit title not kept verbatim with its provenance: %+v", got)
|
|
}
|
|
|
|
derived, _, err := store.CreateRun(ctx, definition, "derived", "workspace", false, nil,
|
|
map[string]any{"prompt": strings.Repeat("é", 500) + "\nsecond line is ignored"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err = store.GetRun(ctx, derived)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Title == "" || got.TitleSource != summarise.SourceFallback || got.TitleModel != "" {
|
|
t.Fatalf("derived title = %+v, want a non-empty fallback with no model", got)
|
|
}
|
|
if len([]rune(got.Title)) > summarise.TitleCap {
|
|
t.Fatalf("derived title has %d runes, want <= %d", len([]rune(got.Title)), summarise.TitleCap)
|
|
}
|
|
|
|
// Simulate a database created before run titles by removing the columns, then
|
|
// reopen: existing rows must survive the re-add migration and still list.
|
|
for _, column := range []string{"title", "title_source", "title_model"} {
|
|
if _, err := store.db.ExecContext(ctx, "ALTER TABLE pipeline_runs DROP COLUMN "+column); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil {
|
|
t.Fatal("legacy runs did not survive the title migration", err)
|
|
}
|
|
ids := map[int64]bool{}
|
|
for _, run := range runs {
|
|
ids[run.ID] = true
|
|
}
|
|
if !ids[explicit] || !ids[derived] {
|
|
t.Fatalf("migrated run list lost history: %+v", runs)
|
|
}
|
|
migrated, err := store.GetRun(ctx, explicit)
|
|
if err != nil || migrated.Title != "" || migrated.TitleSource != "" || migrated.TitleModel != "" {
|
|
t.Fatalf("migrated row should keep a valid empty title: %+v %v", migrated, err)
|
|
}
|
|
}
|
|
|
|
// TestSetRunTitleRenamesWithoutTouchingInput proves an after-the-fact rename
|
|
// replaces the stored title and its provenance and leaves the run's input,
|
|
// status and steps untouched; an unknown run is reported, not silently ignored.
|
|
func TestSetRunTitleRenamesWithoutTouchingInput(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "rename.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "rename", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "rename-key", "workspace", false, nil, map[string]any{"prompt": "original derived prompt"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.SetRunTitle(ctx, id, summarise.Title{Text: "Investigate the regression", Source: summarise.SourceExplicit}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
after, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if after.Title != "Investigate the regression" || after.TitleSource != summarise.SourceExplicit || after.TitleModel != "" {
|
|
t.Fatalf("rename = %+v, want the explicit title and provenance", after)
|
|
}
|
|
if after.Status != before.Status || !reflect.DeepEqual(after.Input, before.Input) || len(after.Steps) != len(before.Steps) {
|
|
t.Fatalf("rename changed run state: before %+v after %+v", before, after)
|
|
}
|
|
if err := store.SetRunTitle(ctx, id+999, summarise.Title{Text: "ghost", Source: summarise.SourceExplicit}); err == nil {
|
|
t.Fatal("renaming an unknown run succeeded")
|
|
}
|
|
}
|
|
|
|
// TestRunModelOverrideRoundTripAndRequestKey proves the launch-time model
|
|
// overrides round-trip through the run record and the per-step provider/model,
|
|
// and that a request-key replay with different overrides is refused as a
|
|
// different run.
|
|
func TestRunModelOverrideRoundTripAndRequestKey(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "override.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "override", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
overrides := RunOverrides{Model: "acme/whole", StageModels: map[string]string{"one": "acme/strong"}}
|
|
id, created, err := store.CreateRunTitled(ctx, definition, "override-key", "workspace", false, nil,
|
|
map[string]any{"prompt": "task"}, overrides, summarise.Title{})
|
|
if err != nil || !created {
|
|
t.Fatalf("create override run = %d %v %v", id, created, err)
|
|
}
|
|
if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Output: map[string]any{}, Provider: "acme", Model: "strong"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Model != "acme/whole" || got.StageModels["one"] != "acme/strong" {
|
|
t.Fatalf("run record did not echo overrides: %+v", got)
|
|
}
|
|
if len(got.Steps) != 1 || got.Steps[0].Provider != "acme" || got.Steps[0].Model != "strong" {
|
|
t.Fatalf("step did not record the effective model: %+v", got.Steps)
|
|
}
|
|
list, err := store.ListRuns(ctx)
|
|
if err != nil || len(list) != 1 || list[0].Model != "acme/whole" || list[0].StageModels["one"] != "acme/strong" {
|
|
t.Fatalf("run list omitted the overrides: %+v %v", list, err)
|
|
}
|
|
// An idempotent replay with the same overrides returns the same run.
|
|
again, created, err := store.CreateRunTitled(ctx, definition, "override-key", "workspace", false, nil,
|
|
map[string]any{"prompt": "task"}, overrides, summarise.Title{})
|
|
if err != nil || created || again != id {
|
|
t.Fatalf("same-override replay = %d %v %v, want the stored run", again, created, err)
|
|
}
|
|
// A different override under the same key is a different run.
|
|
if _, _, err := store.CreateRunTitled(ctx, definition, "override-key", "workspace", false, nil,
|
|
map[string]any{"prompt": "task"}, RunOverrides{StageModels: map[string]string{"one": "acme/other"}}, summarise.Title{}); err == nil || !strings.Contains(err.Error(), "request key already used") {
|
|
t.Fatalf("differing-override replay was not refused: %v", err)
|
|
}
|
|
if _, _, err := store.CreateRunTitled(ctx, definition, "override-key", "workspace", false, nil,
|
|
map[string]any{"prompt": "task"}, RunOverrides{}, summarise.Title{}); err == nil || !strings.Contains(err.Error(), "request key already used") {
|
|
t.Fatalf("no-override replay was not refused: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestRunModelOverrideMigrationKeepsHistory proves a database created before the
|
|
// override columns existed gains them idempotently on Open and still lists and
|
|
// reads its historical rows with empty overrides.
|
|
func TestRunModelOverrideMigrationKeepsHistory(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "override-migration.db")
|
|
store, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, "legacy", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "legacy-key", "workspace", false, nil, map[string]any{"prompt": "task"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Simulate a pre-override database by dropping the columns, then reopen.
|
|
for _, column := range []string{"model", "stage_models"} {
|
|
if _, err := store.db.ExecContext(ctx, "ALTER TABLE pipeline_runs DROP COLUMN "+column); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
for _, column := range []string{"provider", "model"} {
|
|
if _, err := store.db.ExecContext(ctx, "ALTER TABLE run_steps DROP COLUMN "+column); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 1 {
|
|
t.Fatalf("legacy run did not survive the override migration: %+v %v", runs, err)
|
|
}
|
|
got, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got.Model != "" || len(got.StageModels) != 0 {
|
|
t.Fatalf("migrated row should keep empty overrides: %+v", got)
|
|
}
|
|
}
|
|
|
|
func TestRunRecoveryAndVersionPinning(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "audit.db")
|
|
store, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
source := Template([]string{"one"}, true)
|
|
if _, err := store.Submit(ctx, "audit", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "recover", "workspace", false, nil, map[string]any{"n": 123})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.BeginStep(ctx, id, 1, "one", map[string]any{"n": 123}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.RunSession(ctx, id, 1, "ses_recover", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
owners, err := store.ActiveSessionOwners(ctx)
|
|
if err != nil || len(owners) != 1 {
|
|
t.Fatal("lost active session ownership")
|
|
}
|
|
if err := store.RecoverRuns(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, "audit", Template([]string{"replacement", "another"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
run, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if run.Status != "interrupted" || run.Steps[0].Status != "interrupted" || run.Steps[0].AgentActive || run.ContentHash != definition.ContentHash || run.Version != 1 {
|
|
t.Fatal("recovery silently resumed or changed version", run)
|
|
}
|
|
if !reflect.DeepEqual(run.Graph, definition.Graph) {
|
|
t.Fatal("run graph did not retain its pinned definition", run.Graph)
|
|
}
|
|
loaded, _ := store.Get(ctx, definition.ID, 1)
|
|
if loaded.Source != source {
|
|
t.Fatal("run modified immutable source")
|
|
}
|
|
}
|
|
|
|
// TestActiveAgentSessionsAreScopedToRun proves the run-scoped query returns only
|
|
// the row's own active session: a second run's active session is never returned,
|
|
// and clearing the session clears the result.
|
|
func TestActiveAgentSessionsAreScopedToRun(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "scoped.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "scoped", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runA, _, err := store.CreateRun(ctx, definition, "a", "workspace-a", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
runB, _, err := store.CreateRun(ctx, definition, "b", "workspace-b", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, run := range []struct {
|
|
id int64
|
|
session string
|
|
}{{runA, "ses_a"}, {runB, "ses_b"}} {
|
|
if err := store.BeginStep(ctx, run.id, 1, "one", map[string]any{"prompt": "task"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.RunSession(ctx, run.id, 1, run.session, true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
sessions, err := store.ActiveAgentSessionsForRun(ctx, runA)
|
|
if err != nil || !reflect.DeepEqual(sessions, []string{"ses_a"}) {
|
|
t.Fatalf("run-scoped query leaked another run's session: %v %v", sessions, err)
|
|
}
|
|
if err := store.RunSession(ctx, runA, 1, "ses_a", false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sessions, err = store.ActiveAgentSessionsForRun(ctx, runA)
|
|
if err != nil || len(sessions) != 0 {
|
|
t.Fatalf("inactive session still reported active: %v %v", sessions, err)
|
|
}
|
|
if sessions, err = store.ActiveAgentSessionsForRun(ctx, runB); err != nil || !reflect.DeepEqual(sessions, []string{"ses_b"}) {
|
|
t.Fatalf("second run's own session was affected: %v %v", sessions, err)
|
|
}
|
|
}
|
|
|
|
// TestOwnedSessionsListsRecordedWithActiveFlag proves OwnedSessions returns every
|
|
// session the server ever recorded (active or not) in run_steps, deduped by
|
|
// session id in favour of the newest owning run, with that run's workspace
|
|
// directory and the active flag.
|
|
func TestOwnedSessionsListsRecordedWithActiveFlag(t *testing.T) {
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), "owned.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "owned", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
record := func(key, workspace, session string, active bool) {
|
|
t.Helper()
|
|
run, _, err := store.CreateRun(ctx, definition, key, workspace, false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.BeginStep(ctx, run, 1, "one", map[string]any{"prompt": "task"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.RunSession(ctx, run, 1, session, active, "v2", "/repo"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
record("owned-a", "/work/a", "ses_active", true)
|
|
record("owned-b", "/work/b", "ses_idle", false)
|
|
// The same session recorded by a newer run must dedupe to one entry, taking
|
|
// the newest run's workspace and active flag.
|
|
record("owned-c", "/work/c", "ses_idle", true)
|
|
|
|
owners, err := store.OwnedSessions(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(owners) != 2 {
|
|
t.Fatalf("owned sessions = %+v, want 2 deduped", owners)
|
|
}
|
|
byID := map[string]SessionOwner{}
|
|
for _, owner := range owners {
|
|
byID[owner.ID] = owner
|
|
}
|
|
if owner, ok := byID["ses_active"]; !ok || !owner.Active || owner.Directory != "/work/a" {
|
|
t.Fatalf("active owner = %+v", owner)
|
|
}
|
|
if owner, ok := byID["ses_idle"]; !ok || !owner.Active || owner.Directory != "/work/c" {
|
|
t.Fatalf("deduped owner should come from the newest run: %+v", owner)
|
|
}
|
|
if active, err := store.ActiveSessionOwners(ctx); err != nil || len(active) != 2 {
|
|
t.Fatalf("ActiveSessionOwners = %+v %v, want both now-active sessions", active, err)
|
|
}
|
|
}
|