e79ff9585f
Each completed step now exposes a verdict: the chosen option (outcome -> target), a bounded one-line reason with model/fallback/absent provenance, and the recorded artifacts the reason refers to. The store persists only the model-generated line in a new step_verdicts table and derives the fallback at read time (no backfill); the server generates the line once at completion, best-effort; the TUI renders it in the stage progress summary. The packaged debugging reference documents it.
207 lines
7.4 KiB
Go
207 lines
7.4 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"git.a-wan.net/alan/pipelines/internal/protocol"
|
|
)
|
|
|
|
// storeWithRun submits one pipeline and creates one run for verdict tests.
|
|
func storeWithRun(t *testing.T, name string) (*Store, context.Context, int64) {
|
|
t.Helper()
|
|
ctx := context.Background()
|
|
store, err := Open(filepath.Join(t.TempDir(), name+".db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, name, Template([]string{"one", "two"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, err := store.List(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
definition, err := store.Get(ctx, items[0].ID, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
id, _, err := store.CreateRun(ctx, definition, name, "workspace", false, nil, map[string]any{"prompt": "task"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return store, ctx, id
|
|
}
|
|
|
|
// TestStageVerdictStoredLineAndReadTimeFallback proves a model-generated line is
|
|
// stored with its provenance and read back verbatim, while a step finished
|
|
// without one gets a bounded reason derived at read time from its own output (the
|
|
// existing-run/no-backfill path).
|
|
func TestStageVerdictStoredLineAndReadTimeFallback(t *testing.T) {
|
|
store, ctx, id := storeWithRun(t, "verdict")
|
|
defer store.Close()
|
|
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{
|
|
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two",
|
|
Output: map[string]any{"summary": "read-time text must lose to the stored line"},
|
|
Verdict: &protocol.StageVerdict{Reason: "the model line", ReasonSource: protocol.VerdictReasonModel},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.BeginStep(ctx, id, 2, "two", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{
|
|
Sequence: 2, Stage: "two", Status: "completed", Outcome: "success", Target: "#success",
|
|
Output: map[string]any{"summary": "derived at read time"},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(detail.Steps) != 2 {
|
|
t.Fatalf("got %d steps", len(detail.Steps))
|
|
}
|
|
stored := detail.Steps[0].Verdict
|
|
if stored == nil || stored.Reason != "the model line" || stored.ReasonSource != protocol.VerdictReasonModel ||
|
|
stored.Option != "success → two" {
|
|
t.Fatalf("stored verdict = %+v", stored)
|
|
}
|
|
fallback := detail.Steps[1].Verdict
|
|
if fallback == nil || fallback.Reason != "derived at read time" ||
|
|
fallback.ReasonSource != protocol.VerdictReasonFallback || fallback.Option != "success → success" {
|
|
t.Fatalf("read-time verdict = %+v", fallback)
|
|
}
|
|
}
|
|
|
|
// TestStageVerdictMatchingUsesRecordedArtifacts proves the store derives the
|
|
// referenced artifacts from the artifacts it recorded for the same visit, by
|
|
// reason text and by screenshots, and never exposes an unrecorded name.
|
|
func TestStageVerdictMatchingUsesRecordedArtifacts(t *testing.T) {
|
|
store, ctx, id := storeWithRun(t, "verdict-artifacts")
|
|
defer store.Close()
|
|
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{
|
|
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two",
|
|
Output: map[string]any{"summary": "reviewed captures/shot.png and verified it"},
|
|
Artifacts: []protocol.StageArtifact{{Name: "captures/shot.png"}, {Name: "unrelated.log"}},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.BeginStep(ctx, id, 2, "two", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{
|
|
Sequence: 2, Stage: "two", Status: "completed", Outcome: "success", Target: "#success",
|
|
Output: map[string]any{"summary": "no recorded name here", "screenshots": []any{"captures/shot.png"}},
|
|
Artifacts: []protocol.StageArtifact{{Name: "captures/shot.png"}},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got := detail.Steps[0].Verdict.Artifacts; len(got) != 1 || got[0] != "captures/shot.png" {
|
|
t.Fatalf("reason-referenced artifacts = %v, want only the recorded capture", got)
|
|
}
|
|
if got := detail.Steps[1].Verdict.Artifacts; len(got) != 1 || got[0] != "captures/shot.png" {
|
|
t.Fatalf("screenshot-referenced artifacts = %v, want only the recorded capture", got)
|
|
}
|
|
}
|
|
|
|
// TestStageVerdictAbsentAndNoArtifacts proves a running step has no verdict and a
|
|
// completed step with no recorded artifacts keeps Option and Reason but an empty
|
|
// artifact list.
|
|
func TestStageVerdictAbsentAndNoArtifacts(t *testing.T) {
|
|
store, ctx, id := storeWithRun(t, "verdict-absent")
|
|
defer store.Close()
|
|
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Still running: no outcome/target, so no verdict.
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if detail.Steps[0].Verdict != nil {
|
|
t.Fatalf("running step verdict = %+v, want nil", detail.Steps[0].Verdict)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{
|
|
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success",
|
|
Output: map[string]any{"summary": "done", "extra": "x"},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
detail, err = store.GetRun(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
verdict := detail.Steps[0].Verdict
|
|
if verdict == nil || verdict.Reason != "done" || verdict.ReasonSource != protocol.VerdictReasonFallback || len(verdict.Artifacts) != 0 {
|
|
t.Fatalf("no-artifact verdict = %+v", verdict)
|
|
}
|
|
}
|
|
|
|
// TestStageVerdictStoreUpgrade proves a database created before step_verdicts
|
|
// exists opens cleanly, gains the table on init, and still lists and loads runs
|
|
// and steps.
|
|
func TestStageVerdictStoreUpgrade(t *testing.T) {
|
|
ctx := context.Background()
|
|
path := filepath.Join(t.TempDir(), "upgrade.db")
|
|
store, err := Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := store.Submit(ctx, "upgrade", Template([]string{"one"}, true)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
items, _ := store.List(ctx)
|
|
definition, _ := store.Get(ctx, items[0].ID, 1)
|
|
id, _, err := store.CreateRun(ctx, definition, "upgrade", "workspace", false, nil, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.FinishStep(ctx, id, protocol.RunStep{
|
|
Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success",
|
|
Output: map[string]any{"summary": "legacy output"},
|
|
Verdict: &protocol.StageVerdict{Reason: "generated before the upgrade", ReasonSource: protocol.VerdictReasonModel},
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Simulate a database created before step_verdicts existed.
|
|
if _, err := store.db.ExecContext(ctx, "DROP TABLE step_verdicts"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := store.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
store, err = Open(path)
|
|
if err != nil {
|
|
t.Fatal("a pre-verdict database did not open cleanly", err)
|
|
}
|
|
defer store.Close()
|
|
runs, err := store.ListRuns(ctx)
|
|
if err != nil || len(runs) != 1 {
|
|
t.Fatalf("legacy run list lost history: %v %v", runs, err)
|
|
}
|
|
detail, err := store.GetRun(ctx, id)
|
|
if err != nil || len(detail.Steps) != 1 {
|
|
t.Fatalf("legacy run/step did not load: %v %v", detail, err)
|
|
}
|
|
// The stored line is gone, so the verdict is derived at read time.
|
|
if verdict := detail.Steps[0].Verdict; verdict == nil || verdict.Reason != "legacy output" || verdict.ReasonSource != protocol.VerdictReasonFallback {
|
|
t.Fatalf("post-upgrade read-time verdict = %+v", verdict)
|
|
}
|
|
}
|