194 lines
7.3 KiB
Go
194 lines
7.3 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/pipeline"
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
"git.a-wan.net/alan/pipelines/internal/stage"
|
|
)
|
|
|
|
// TestRunnerAppliesModelOverrides proves a run with exactly one overridden stage
|
|
// records that step's provider/model as the override and the other model stage as
|
|
// the pinned pair, that the same effective values reach the resolved ModelConfig
|
|
// (the single source the repair/formatter path reads), and that the run record
|
|
// and returned graph echo the overrides while the pinned source is unchanged.
|
|
func TestRunnerAppliesModelOverrides(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "overrides.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
source := strings.Replace(pipeline.Template([]string{"one", "two"}, true), "retries=2", "reasoning_effort=high\nretries=2", 1)
|
|
if _, err := store.Submit(ctx, "overrides", source); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pinned := map[string]stage.ModelConfig{}
|
|
for _, s := range definition.Graph.Stages {
|
|
pinned[s.Key] = stage.ModelConfig{Provider: s.Configs["provider"], Model: s.Configs["model"]}
|
|
}
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
resolved := map[string]stage.ModelConfig{}
|
|
r.execute = func(_ context.Context, def stage.Definition, _ stage.Values) (stage.Result, error) {
|
|
spec, err := (stage.ModelStage{}).Spec(def)
|
|
if err != nil {
|
|
return stage.Result{}, err
|
|
}
|
|
resolved[def.Key] = spec.Config
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "one-override", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if got := resolved["one"]; got.Provider != "acme" || got.Model != "strong" || got.ReasoningEffort != "high" {
|
|
t.Fatalf("stage one resolved %+v, want the override", got)
|
|
}
|
|
if got := resolved["two"]; got.Provider != pinned["two"].Provider || got.Model != pinned["two"].Model {
|
|
t.Fatalf("stage two resolved %+v, want the pinned %+v", got, pinned["two"])
|
|
}
|
|
detail, err := store.GetRun(ctx, run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Model != "" || detail.StageModels["one"] != "acme/strong" {
|
|
t.Fatalf("run record did not echo the overrides: %+v", detail)
|
|
}
|
|
if len(detail.Steps) != 2 {
|
|
t.Fatalf("steps = %+v", detail.Steps)
|
|
}
|
|
if detail.Steps[0].Provider != "acme" || detail.Steps[0].Model != "strong" {
|
|
t.Fatalf("overridden step evidence = %+v", detail.Steps[0])
|
|
}
|
|
if detail.Steps[1].Provider != pinned["two"].Provider || detail.Steps[1].Model != pinned["two"].Model {
|
|
t.Fatalf("pinned step evidence = %+v, want %+v", detail.Steps[1], pinned["two"])
|
|
}
|
|
// The returned graph shows the effective pair for the overridden stage and
|
|
// leaves the other at its pinned configs; `show --version N` (the immutable
|
|
// source) is untouched.
|
|
overlaid, err := r.getRun(run.ID)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
configs := map[string]map[string]string{}
|
|
for _, s := range overlaid.Graph.Stages {
|
|
configs[s.Key] = s.Configs
|
|
}
|
|
if configs["one"]["provider"] != "acme" || configs["one"]["model"] != "strong" || configs["one"]["reasoning_effort"] != "high" {
|
|
t.Fatalf("overlaid graph stage one = %+v, want the override", configs["one"])
|
|
}
|
|
if configs["two"]["provider"] != pinned["two"].Provider || configs["two"]["model"] != pinned["two"].Model {
|
|
t.Fatalf("overlaid graph stage two = %+v, want the pinned pair", configs["two"])
|
|
}
|
|
loaded, err := store.Get(ctx, definition.ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if loaded.Source != definition.Source {
|
|
t.Fatal("the pinned immutable source was rewritten by an override")
|
|
}
|
|
}
|
|
|
|
// TestRunnerRefusesInvalidModelOverridesWithoutRunRow proves an unknown stage key// and a malformed provider/model both fail before admission and create no run row.
|
|
func TestRunnerRefusesInvalidModelOverridesWithoutRunRow(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "reject.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "reject", pipeline.Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
|
|
}
|
|
before, _ := store.ListRuns(ctx)
|
|
cases := []struct {
|
|
name string
|
|
key string
|
|
model string
|
|
stage map[string]string
|
|
want string
|
|
}{
|
|
{"unknown stage key", "unknown-key", "", map[string]string{"nope": "acme/strong"}, "unknown stage key"},
|
|
{"invalid whole-run pair", "invalid-model", "not-a-pair", nil, "provider/model"},
|
|
{"invalid stage pair", "invalid-stage", "", map[string]string{"one": "acme/"}, "provider/model"},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
_, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: tc.key, Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, Model: tc.model, StageModels: tc.stage})
|
|
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
|
t.Fatalf("error = %v, want %q", err, tc.want)
|
|
}
|
|
if id, err := store.RunByKey(ctx, tc.key); err != nil || id != 0 {
|
|
t.Fatalf("rejected request created run %d: %v", id, err)
|
|
}
|
|
})
|
|
}
|
|
after, _ := store.ListRuns(ctx)
|
|
if len(after) != len(before) {
|
|
t.Fatalf("rejected requests created run rows: before %d after %d", len(before), len(after))
|
|
}
|
|
}
|
|
|
|
// TestRetryReusesModelOverrides proves a retry of a failed run reuses the
|
|
// original run's recorded overrides, so the new attempt is reproducible.
|
|
func TestRetryReusesModelOverrides(t *testing.T) {
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
store, err := pipeline.Open(filepath.Join(t.TempDir(), "retry-override.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer store.Close()
|
|
if _, err := store.Submit(ctx, "retry-override", pipeline.Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
r, err := newRunner(ctx, store)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
|
|
return stage.Result{}, errors.New("provider failed")
|
|
}
|
|
first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "first", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}, StageModels: map[string]string{"one": "acme/strong"}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if detail, _ := store.GetRun(ctx, first.ID); detail.Status != "failed" {
|
|
t.Fatalf("original run did not fail: %+v", detail)
|
|
}
|
|
retried, err := r.retry(protocol.Message{RunID: first.ID, RequestKey: "retry"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
r.wg.Wait()
|
|
if retried.StageModels["one"] != "acme/strong" || retried.Model != "" {
|
|
t.Fatalf("retry did not reuse the original overrides: %+v", retried)
|
|
}
|
|
}
|