package server import ( "context" "errors" "path/filepath" "strings" "testing" "git.a-wan.net/alan/pipelines/internal/pipeline" "git.a-wan.net/alan/pipelines/internal/protocol" "git.a-wan.net/alan/pipelines/internal/stage" ) // TestRunnerAppliesModelOverrides proves a run with exactly one overridden stage // records that step's provider/model as the override and the other model stage as // the pinned pair, that the same effective values reach the resolved ModelConfig // (the single source the repair/formatter path reads), and that the run record // and returned graph echo the overrides while the pinned source is unchanged. func TestRunnerAppliesModelOverrides(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "overrides.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "overrides", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } pinned := map[string]stage.ModelConfig{} for _, s := range definition.Graph.Stages { pinned[s.Key] = stage.ModelConfig{Provider: s.Configs["provider"], Model: s.Configs["model"]} } r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } resolved := map[string]stage.ModelConfig{} r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) { spec, err := (stage.ModelStage{}).Spec(def) if err != nil { return stage.Result{}, err } resolved[def.Key] = spec.Config return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "one-override", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}}) if err != nil { t.Fatal(err) } r.wg.Wait() if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" { t.Fatalf("stage one resolved %+v, want the override", got) } if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model { t.Fatalf("stage two resolved %+v, want the pinned %+v", got, pinned["two"]) } detail, err := store.GetRun(ctx, run.ID) if err != nil { t.Fatal(err) } if detail.Model != "" || detail.StageModels["one"] != "acme/strong" { t.Fatalf("run record did not echo the overrides: %+v", detail) } if len(detail.Steps) != 2 { t.Fatalf("steps = %+v", detail.Steps) } if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" { t.Fatalf("overridden step evidence = %+v", detail.Steps[0]) } if detail.Steps[1].Provider != pinned["two"].Provider || detail.Steps[1].Model != pinned["two"].Model { t.Fatalf("pinned step evidence = %+v, want %+v", detail.Steps[1], pinned["two"]) } // The returned graph shows the effective pair for the overridden stage and // leaves the other at its pinned configs; `show --version N` (the immutable // source) is untouched. overlaid, err := r.getRun(run.ID) if err != nil { t.Fatal(err) } configs := map[string]map[string]string{} for _, s := range overlaid.Graph.Stages { configs[s.Key] = s.Configs } if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" { t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"]) } if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model { t.Fatalf("overlaid graph stage two = %+v, want the pinned pair", configs["two"]) } loaded, err := store.Get(ctx, definition.ID, 1) if err != nil { t.Fatal(err) } if loaded.Source != definition.Source { t.Fatal("the pinned immutable source was rewritten by an override") } } // TestRunnerRefusesInvalidModelOverridesWithoutRunRow proves an unknown stage key// and a malformed provider/model both fail before admission and create no run row. func TestRunnerRefusesInvalidModelOverridesWithoutRunRow(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "reject.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "reject", pipeline.Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{Outcome: "success", Output: stage.Values{}}, nil } before, _ := store.ListRuns(ctx) cases := []struct { name string key string model string stage map[string]string want string }{ {"unknown stage key", "unknown-key", "", map[string]string{"nope": "acme/strong"}, "unknown stage key"}, {"invalid whole-run pair", "invalid-model", "not-a-pair", nil, "provider/model"}, {"invalid stage pair", "invalid-stage", "", map[string]string{"one": "acme/"}, "provider/model"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: tc.key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, Model: tc.model, StageModels: tc.stage}) if err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("error = %v, want %q", err, tc.want) } if id, err := store.RunByKey(ctx, tc.key); err != nil || id != 0 { t.Fatalf("rejected request created run %d: %v", id, err) } }) } after, _ := store.ListRuns(ctx) if len(after) != len(before) { t.Fatalf("rejected requests created run rows: before %d after %d", len(before), len(after)) } } // TestRetryReusesModelOverrides proves a retry of a failed run reuses the // original run's recorded overrides, so the new attempt is reproducible. func TestRetryReusesModelOverrides(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-override.db")) if err != nil { t.Fatal(err) } defer store.Close() if _, err := store.Submit(ctx, "retry-override", pipeline.Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) r, err := newRunner(ctx, store) if err != nil { t.Fatal(err) } r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) { return stage.Result{}, errors.New("provider failed") } first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "first", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}}) if err != nil { t.Fatal(err) } r.wg.Wait() if detail, _ := store.GetRun(ctx, first.ID); detail.Status != "failed" { t.Fatalf("original run did not fail: %+v", detail) } retried, err := r.retry(protocol.Message{RunID: first.ID, RequestKey: "retry"}) if err != nil { t.Fatal(err) } r.wg.Wait() if retried.StageModels["one"] != "acme/strong" || retried.Model != "" { t.Fatalf("retry did not reuse the original overrides: %+v", retried) } }