Files
pipelines/internal/pipeline/verdict_test.go
T
alan e79ff9585f Derive a plain-language verdict for every completed stage visit
Each completed step now exposes a verdict: the chosen option (outcome -> target),
a bounded one-line reason with model/fallback/absent provenance, and the recorded
artifacts the reason refers to. The store persists only the model-generated line
in a new step_verdicts table and derives the fallback at read time (no backfill);
the server generates the line once at completion, best-effort; the TUI renders it
in the stage progress summary. The packaged debugging reference documents it.
2026-10-16 13:54:48 +01:00

207 lines
7.4 KiB
Go

package pipeline
import (
"context"
"path/filepath"
"testing"
"git.a-wan.net/alan/pipelines/internal/protocol"
)
// storeWithRun submits one pipeline and creates one run for verdict tests.
func storeWithRun(t *testing.T, name string) (*Store, context.Context, int64) {
t.Helper()
ctx := context.Background()
store, err := Open(filepath.Join(t.TempDir(), name+".db"))
if err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, name, Template([]string{"one", "two"}, true)); err != nil {
t.Fatal(err)
}
items, err := store.List(ctx)
if err != nil {
t.Fatal(err)
}
definition, err := store.Get(ctx, items[0].ID, 1)
if err != nil {
t.Fatal(err)
}
id, _, err := store.CreateRun(ctx, definition, name, "workspace", false, nil, map[string]any{"prompt": "task"})
if err != nil {
t.Fatal(err)
}
return store, ctx, id
}
// TestStageVerdictStoredLineAndReadTimeFallback proves a model-generated line is
// stored with its provenance and read back verbatim, while a step finished
// without one gets a bounded reason derived at read time from its own output (the
// existing-run/no-backfill path).
func TestStageVerdictStoredLineAndReadTimeFallback(t *testing.T) {
store, ctx, id := storeWithRun(t, "verdict")
defer store.Close()
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two",
Output: map[string]any{"summary": "read-time text must lose to the stored line"},
Verdict: &protocol.StageVerdict{Reason: "the model line", ReasonSource: protocol.VerdictReasonModel},
}); err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{
Sequence: 2, Stage: "two", Status: "completed", Outcome: "success", Target: "#success",
Output: map[string]any{"summary": "derived at read time"},
}); err != nil {
t.Fatal(err)
}
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if len(detail.Steps) != 2 {
t.Fatalf("got %d steps", len(detail.Steps))
}
stored := detail.Steps[0].Verdict
if stored == nil || stored.Reason != "the model line" || stored.ReasonSource != protocol.VerdictReasonModel ||
stored.Option != "success → two" {
t.Fatalf("stored verdict = %+v", stored)
}
fallback := detail.Steps[1].Verdict
if fallback == nil || fallback.Reason != "derived at read time" ||
fallback.ReasonSource != protocol.VerdictReasonFallback || fallback.Option != "success → success" {
t.Fatalf("read-time verdict = %+v", fallback)
}
}
// TestStageVerdictMatchingUsesRecordedArtifacts proves the store derives the
// referenced artifacts from the artifacts it recorded for the same visit, by
// reason text and by screenshots, and never exposes an unrecorded name.
func TestStageVerdictMatchingUsesRecordedArtifacts(t *testing.T) {
store, ctx, id := storeWithRun(t, "verdict-artifacts")
defer store.Close()
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two",
Output: map[string]any{"summary": "reviewed captures/shot.png and verified it"},
Artifacts: []protocol.StageArtifact{{Name: "captures/shot.png"}, {Name: "unrelated.log"}},
}); err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 2, "two", nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{
Sequence: 2, Stage: "two", Status: "completed", Outcome: "success", Target: "#success",
Output: map[string]any{"summary": "no recorded name here", "screenshots": []any{"captures/shot.png"}},
Artifacts: []protocol.StageArtifact{{Name: "captures/shot.png"}},
}); err != nil {
t.Fatal(err)
}
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if got := detail.Steps[0].Verdict.Artifacts; len(got) != 1 || got[0] != "captures/shot.png" {
t.Fatalf("reason-referenced artifacts = %v, want only the recorded capture", got)
}
if got := detail.Steps[1].Verdict.Artifacts; len(got) != 1 || got[0] != "captures/shot.png" {
t.Fatalf("screenshot-referenced artifacts = %v, want only the recorded capture", got)
}
}
// TestStageVerdictAbsentAndNoArtifacts proves a running step has no verdict and a
// completed step with no recorded artifacts keeps Option and Reason but an empty
// artifact list.
func TestStageVerdictAbsentAndNoArtifacts(t *testing.T) {
store, ctx, id := storeWithRun(t, "verdict-absent")
defer store.Close()
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
// Still running: no outcome/target, so no verdict.
detail, err := store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
if detail.Steps[0].Verdict != nil {
t.Fatalf("running step verdict = %+v, want nil", detail.Steps[0].Verdict)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success",
Output: map[string]any{"summary": "done", "extra": "x"},
}); err != nil {
t.Fatal(err)
}
detail, err = store.GetRun(ctx, id)
if err != nil {
t.Fatal(err)
}
verdict := detail.Steps[0].Verdict
if verdict == nil || verdict.Reason != "done" || verdict.ReasonSource != protocol.VerdictReasonFallback || len(verdict.Artifacts) != 0 {
t.Fatalf("no-artifact verdict = %+v", verdict)
}
}
// TestStageVerdictStoreUpgrade proves a database created before step_verdicts
// exists opens cleanly, gains the table on init, and still lists and loads runs
// and steps.
func TestStageVerdictStoreUpgrade(t *testing.T) {
ctx := context.Background()
path := filepath.Join(t.TempDir(), "upgrade.db")
store, err := Open(path)
if err != nil {
t.Fatal(err)
}
if _, err := store.Submit(ctx, "upgrade", Template([]string{"one"}, true)); err != nil {
t.Fatal(err)
}
items, _ := store.List(ctx)
definition, _ := store.Get(ctx, items[0].ID, 1)
id, _, err := store.CreateRun(ctx, definition, "upgrade", "workspace", false, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
t.Fatal(err)
}
if err := store.FinishStep(ctx, id, protocol.RunStep{
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success",
Output: map[string]any{"summary": "legacy output"},
Verdict: &protocol.StageVerdict{Reason: "generated before the upgrade", ReasonSource: protocol.VerdictReasonModel},
}); err != nil {
t.Fatal(err)
}
// Simulate a database created before step_verdicts existed.
if _, err := store.db.ExecContext(ctx, "DROP TABLE step_verdicts"); err != nil {
t.Fatal(err)
}
if err := store.Close(); err != nil {
t.Fatal(err)
}
store, err = Open(path)
if err != nil {
t.Fatal("a pre-verdict database did not open cleanly", err)
}
defer store.Close()
runs, err := store.ListRuns(ctx)
if err != nil || len(runs) != 1 {
t.Fatalf("legacy run list lost history: %v %v", runs, err)
}
detail, err := store.GetRun(ctx, id)
if err != nil || len(detail.Steps) != 1 {
t.Fatalf("legacy run/step did not load: %v %v", detail, err)
}
// The stored line is gone, so the verdict is derived at read time.
if verdict := detail.Steps[0].Verdict; verdict == nil || verdict.Reason != "legacy output" || verdict.ReasonSource != protocol.VerdictReasonFallback {
t.Fatalf("post-upgrade read-time verdict = %+v", verdict)
}
}