Files
pipelines/internal/server/runner_test.go
T
alan e79ff9585f Derive a plain-language verdict for every completed stage visit
Each completed step now exposes a verdict: the chosen option (outcome -> target),
a bounded one-line reason with model/fallback/absent provenance, and the recorded
artifacts the reason refers to. The store persists only the model-generated line
in a new step_verdicts table and derives the fallback at read time (no backfill);
the server generates the line once at completion, best-effort; the TUI renders it
in the stage progress summary. The packaged debugging reference documents it.
2026-10-16 13:54:48 +01:00

425 lines
15 KiB
Go

package server
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"testing"
"git.a-wan.net/alan/pipelines/internal/pipeline"
"git.a-wan.net/alan/pipelines/internal/protocol"
"git.a-wan.net/alan/pipelines/internal/stage"
"git.a-wan.net/alan/pipelines/internal/summarise"
)
// TestRunTitleGeneratedPersistedAndFallback proves the insert path stores a
// generated title with its provenance in list and detail responses, and falls
// back to a derived bounded title when no generator is wired, without failing
// run creation.
func TestRunTitleGeneratedPersistedAndFallback(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "titles.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "titled", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
r.title = func(context.Context, string) summarise.Title {
return summarise.Title{Text: "Fix the flaky login test", Source: summarise.SourceModel, Model: "openai/gpt"}
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "model-title", Workspace: t.TempDir(), Input: map[string]any{"prompt": "prompt body"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Title != "Fix the flaky login test" || detail.TitleSource != summarise.SourceModel || detail.TitleModel != "openai/gpt" {
t.Fatalf("model title not stored with provenance: %+v", detail)
}
list, err := store.ListRuns(ctx)
if err != nil || len(list) != 1 || list[0].Title != "Fix the flaky login test" {
t.Fatalf("run list omitted the title: %v %v", list, err)
}
// No generator: the fallback is derived from the prompt's first line and run
// creation still succeeds.
r.title = nil
fallback, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "fallback-title", Workspace: t.TempDir(), Input: map[string]any{"prompt": "Fix the widget\nand more detail"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err = store.GetRun(ctx, fallback.ID)
if err != nil {
t.Fatal(err)
}
if detail.Title != "Fix the widget" || detail.TitleSource != summarise.SourceFallback || detail.TitleModel != "" {
t.Fatalf("fallback title = %+v, want the prompt's first line", detail)
}
}
// TestRunTitleGeneratorSkippedForReplayAndRejection proves the bounded
// generation runs only for a request that starts a run: an idempotent replay and
// a request refused for an already-reserved workspace must not pay for it.
func TestRunTitleGeneratorSkippedForReplayAndRejection(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "skip.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "skip", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
release := make(chan struct{})
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
<-release
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
calls := 0
r.title = func(context.Context, string) summarise.Title {
calls++
return summarise.Title{Text: "Generated", Source: summarise.SourceModel}
}
workspace := t.TempDir()
first, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-key", Workspace: workspace, Input: map[string]any{"prompt": "first"}})
if err != nil {
t.Fatal(err)
}
if calls != 1 {
t.Fatalf("first start generated %d titles, want 1", calls)
}
// A replay while the first run is active reuses the stored title.
replay, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-key", Workspace: workspace, Input: map[string]any{"prompt": "first"}})
if err != nil || replay.ID != first.ID {
t.Fatalf("replay = %+v, %v; want run %d", replay, err, first.ID)
}
// A new key in the already-reserved workspace is refused.
if _, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "skip-other", Workspace: workspace, Input: map[string]any{"prompt": "second"}}); err == nil {
t.Fatal("workspace conflict was not refused")
}
if calls != 1 {
t.Fatalf("replay/rejection generated %d titles, want 1", calls)
}
close(release)
r.wg.Wait()
}
func TestHandlerWiresRunTitleGenerator(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
store, err := pipeline.Open(filepath.Join(t.TempDir(), "wire.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
configured := HandlerWithOptions(ctx, Options{RunTitles: summarise.GenerateTitle}, store).(*runHandler)
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
if configured.runner.title == nil {
t.Fatal("configured RunTitles was not applied to the runner")
}
if bare.runner.title != nil {
t.Fatal("unconfigured RunTitles must keep deterministic fallback title generation")
}
}
// TestRunnerExplicitTitleSkipsGeneratorAndRenames proves an explicit --title is
// stored verbatim and skips the model generator, and that setTitle later
// replaces it with an explicit provenance without touching the run's input.
func TestRunnerExplicitTitleSkipsGeneratorAndRenames(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "explicit.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "explicit", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
generated := 0
r.title = func(context.Context, string) summarise.Title {
generated++
return summarise.Title{Text: "Generated", Source: summarise.SourceModel}
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "explicit-key", Workspace: t.TempDir(), Title: "Operator chosen title", Input: map[string]any{"prompt": "prompt body"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if generated != 0 {
t.Fatalf("explicit title still called the generator %d times", generated)
}
if run.Title != "Operator chosen title" || run.TitleSource != summarise.SourceExplicit || run.TitleModel != "" {
t.Fatalf("explicit title = %+v", run)
}
renamed, err := r.setTitle(run.ID, " Renamed after the fact ")
if err != nil {
t.Fatal(err)
}
if renamed.Title != "Renamed after the fact" || renamed.TitleSource != summarise.SourceExplicit || renamed.Input["prompt"] != "prompt body" {
t.Fatalf("rename = %+v", renamed)
}
if _, err := r.setTitle(run.ID, " "); err == nil {
t.Fatal("blank rename was accepted")
}
if _, err := r.setTitle(run.ID+999, "ghost"); err == nil {
t.Fatal("renaming an unknown run was accepted")
}
}
func TestRunnerRoutingPersistenceAndIdempotency(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "runs.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "test", pipeline.Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
entered := make(chan struct{})
release := make(chan struct{})
r.execute = func(ctx context.Context, def stage.Definition, input stage.Values) (stage.Result, error) {
if def.Key == "one" {
if len(input) != 2 || input["prompt"] != "task" || input["pwd"] == "" {
t.Error("entry input must be the user prompt unchanged")
}
close(entered)
<-release
return stage.Result{Outcome: "success", Output: stage.Values{"handoff": "evidence"}}, nil
}
if len(input) != 1 || input["handoff"] != "evidence" {
t.Error("handoff not propagated")
}
return stage.Result{Outcome: "success", Output: stage.Values{"verified": true}}, nil
}
request := protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "same", Input: map[string]any{"prompt": "task"}}
run, err := r.start(request)
if err != nil {
t.Fatal(err)
}
<-entered
duplicate, err := r.start(request)
if err != nil || duplicate.ID != run.ID {
t.Fatal("active repeat was not idempotent", err)
}
other := request
other.RequestKey = "other"
if _, err := r.start(other); err == nil || !strings.Contains(err.Error(), fmt.Sprintf("run %d is already active", run.ID)) || !strings.Contains(err.Error(), "different workspace") {
t.Fatal("overlapping execution was not refused with the conflicting run", err)
}
close(release)
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" || len(detail.Steps) != 2 || detail.Steps[1].Input["handoff"] != "evidence" || detail.Output["verified"] != true {
t.Fatalf("invalid persisted run: %+v", detail)
}
duplicate, err = r.start(request)
if err != nil || duplicate.ID != run.ID {
t.Fatal("finished repeat not idempotent")
}
request.RequestKey = "failure"
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{}, errors.New("provider failed")
}
failed, err := r.start(request)
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, _ = store.GetRun(ctx, failed.ID)
if detail.Status != "failed" || detail.Steps[0].Outcome != "error" || detail.Steps[0].Target != protocol.FailureEnd {
t.Fatal("execution error did not follow mandatory error edge")
}
request.RequestKey = "cancel"
r.execute = func(ctx context.Context, _ stage.Definition, _ stage.Values) (stage.Result, error) {
<-ctx.Done()
return stage.Result{}, ctx.Err()
}
canceled, err := r.start(request)
if err != nil {
t.Fatal(err)
}
if err := r.cancel(canceled.ID); err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, _ = store.GetRun(ctx, canceled.ID)
if detail.Status != "canceled" {
t.Fatal("cancel not persisted")
}
request.RequestKey = "budget"
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{}}, nil
}
// A new version with a self-loop remains reachable to a terminal via error.
source := pipeline.Template([]string{"loop"}, true)
source = strings.Replace(source, "success -> #success", "success -> loop", 1)
if _, err := store.Submit(ctx, "loop", source); err != nil {
t.Fatal(err)
}
items, _ = store.List(ctx)
for _, p := range items {
if p.Name == "loop" {
request.PipelineID = p.ID
}
}
bounded, err := r.start(request)
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, _ = store.GetRun(ctx, bounded.ID)
if detail.Status != "failed" || len(detail.Steps) != 5 {
t.Fatalf("cycle not bounded: %+v", detail)
}
}
// TestStepVerdictGeneratedOnceAtCompletion proves the wired seam runs exactly
// once per completed stage visit, from the step's own output, and the generated
// line is stored with model provenance.
func TestStepVerdictGeneratedOnceAtCompletion(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "verdict", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "ran the checks"}}, nil
}
calls := 0
r.verdict = func(_ context.Context, output map[string]any) string {
calls++
if output["summary"] != "ran the checks" {
t.Errorf("verdict seam got output %v", output)
}
return "chosen because every check passed"
}
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "verdict-once", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
if calls != 1 {
t.Fatalf("verdict seam ran %d times, want exactly once for one step", calls)
}
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
verdict := detail.Steps[0].Verdict
if verdict == nil || verdict.Reason != "chosen because every check passed" || verdict.ReasonSource != protocol.VerdictReasonModel {
t.Fatalf("stored verdict = %+v", verdict)
}
}
// TestStepVerdictFailureNeverFailsStage proves a best-effort failure (the
// NoSummary sentinel) still lets the run reach its normal terminal status, with
// the verdict derived at read time from the step's output.
func TestStepVerdictFailureNeverFailsStage(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict-fail.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
if _, err := store.Submit(ctx, "verdict-fail", pipeline.Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
r, err := newRunner(ctx, store)
if err != nil {
t.Fatal(err)
}
r.execute = func(context.Context, stage.Definition, stage.Values) (stage.Result, error) {
return stage.Result{Outcome: "success", Output: stage.Values{"summary": "ran the checks"}}, nil
}
r.verdict = func(context.Context, map[string]any) string { return summarise.NoSummary }
run, err := r.start(protocol.Message{PipelineID: items[0].ID, Version: 1, RequestKey: "verdict-fail", Workspace: t.TempDir(), Input: map[string]any{"prompt": "task"}})
if err != nil {
t.Fatal(err)
}
r.wg.Wait()
detail, err := store.GetRun(ctx, run.ID)
if err != nil {
t.Fatal(err)
}
if detail.Status != "succeeded" {
t.Fatalf("summariser failure changed the run status: %+v", detail)
}
verdict := detail.Steps[0].Verdict
if verdict == nil || verdict.Reason != "ran the checks" || verdict.ReasonSource != protocol.VerdictReasonFallback {
t.Fatalf("read-time verdict after a summariser failure = %+v", verdict)
}
}
func TestHandlerWiresStepVerdictGenerator(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
store, err := pipeline.Open(filepath.Join(t.TempDir(), "verdict-wire.db"))
if err != nil {
t.Fatal(err)
}
defer store.Close()
configured := HandlerWithOptions(ctx, Options{StepVerdicts: summarise.GenerateVerdictReason}, store).(*runHandler)
bare := HandlerWithOptions(ctx, Options{}, store).(*runHandler)
defer func() { cancel(); WaitRuns(configured); WaitRuns(bare) }()
if configured.runner.verdict == nil {
t.Fatal("configured StepVerdicts was not applied to the runner")
}
if bare.runner.verdict != nil {
t.Fatal("unconfigured StepVerdicts must keep deterministic read-time derivation")
}
}