package pipeline import ( "context" "path/filepath" "testing" "git.a-wan.net/alan/pipelines/internal/protocol" ) // storeWithRun submits one pipeline and creates one run for verdict tests. func storeWithRun(t *testing.T, name string) (*Store, context.Context, int64) { t.Helper() ctx := context.Background() store, err := Open(filepath.Join(t.TempDir(), name+".db")) if err != nil { t.Fatal(err) } if _, err := store.Submit(ctx, name, Template([]string{"one", "two"}, true)); err != nil { t.Fatal(err) } items, err := store.List(ctx) if err != nil { t.Fatal(err) } definition, err := store.Get(ctx, items[0].ID, 1) if err != nil { t.Fatal(err) } id, _, err := store.CreateRun(ctx, definition, name, "workspace", false, nil, map[string]any{"prompt": "task"}) if err != nil { t.Fatal(err) } return store, ctx, id } // TestStageVerdictStoredLineAndReadTimeFallback proves a model-generated line is // stored with its provenance and read back verbatim, while a step finished // without one gets a bounded reason derived at read time from its own output (the // existing-run/no-backfill path). func TestStageVerdictStoredLineAndReadTimeFallback(t *testing.T) { store, ctx, id := storeWithRun(t, "verdict") defer store.Close() if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil { t.Fatal(err) } if err := store.FinishStep(ctx, id, protocol.RunStep{ Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"summary": "read-time text must lose to the stored line"}, Verdict: &protocol.StageVerdict{Reason: "the model line", ReasonSource: protocol.VerdictReasonModel}, }); err != nil { t.Fatal(err) } if err := store.BeginStep(ctx, id, 2, "two", nil); err != nil { t.Fatal(err) } if err := store.FinishStep(ctx, id, protocol.RunStep{ Sequence: 2, Stage: "two", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"summary": "derived at read time"}, }); err != nil { t.Fatal(err) } detail, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } if len(detail.Steps) != 2 { t.Fatalf("got %d steps", len(detail.Steps)) } stored := detail.Steps[0].Verdict if stored == nil || stored.Reason != "the model line" || stored.ReasonSource != protocol.VerdictReasonModel || stored.Option != "success → two" { t.Fatalf("stored verdict = %+v", stored) } fallback := detail.Steps[1].Verdict if fallback == nil || fallback.Reason != "derived at read time" || fallback.ReasonSource != protocol.VerdictReasonFallback || fallback.Option != "success → success" { t.Fatalf("read-time verdict = %+v", fallback) } } // TestStageVerdictMatchingUsesRecordedArtifacts proves the store derives the // referenced artifacts from the artifacts it recorded for the same visit, by // reason text and by screenshots, and never exposes an unrecorded name. func TestStageVerdictMatchingUsesRecordedArtifacts(t *testing.T) { store, ctx, id := storeWithRun(t, "verdict-artifacts") defer store.Close() if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil { t.Fatal(err) } if err := store.FinishStep(ctx, id, protocol.RunStep{ Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "two", Output: map[string]any{"summary": "reviewed captures/shot.png and verified it"}, Artifacts: []protocol.StageArtifact{{Name: "captures/shot.png"}, {Name: "unrelated.log"}}, }); err != nil { t.Fatal(err) } if err := store.BeginStep(ctx, id, 2, "two", nil); err != nil { t.Fatal(err) } if err := store.FinishStep(ctx, id, protocol.RunStep{ Sequence: 2, Stage: "two", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"summary": "no recorded name here", "screenshots": []any{"captures/shot.png"}}, Artifacts: []protocol.StageArtifact{{Name: "captures/shot.png"}}, }); err != nil { t.Fatal(err) } detail, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } if got := detail.Steps[0].Verdict.Artifacts; len(got) != 1 || got[0] != "captures/shot.png" { t.Fatalf("reason-referenced artifacts = %v, want only the recorded capture", got) } if got := detail.Steps[1].Verdict.Artifacts; len(got) != 1 || got[0] != "captures/shot.png" { t.Fatalf("screenshot-referenced artifacts = %v, want only the recorded capture", got) } } // TestStageVerdictAbsentAndNoArtifacts proves a running step has no verdict and a // completed step with no recorded artifacts keeps Option and Reason but an empty // artifact list. func TestStageVerdictAbsentAndNoArtifacts(t *testing.T) { store, ctx, id := storeWithRun(t, "verdict-absent") defer store.Close() if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil { t.Fatal(err) } // Still running: no outcome/target, so no verdict. detail, err := store.GetRun(ctx, id) if err != nil { t.Fatal(err) } if detail.Steps[0].Verdict != nil { t.Fatalf("running step verdict = %+v, want nil", detail.Steps[0].Verdict) } if err := store.FinishStep(ctx, id, protocol.RunStep{ Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"summary": "done", "extra": "x"}, }); err != nil { t.Fatal(err) } detail, err = store.GetRun(ctx, id) if err != nil { t.Fatal(err) } verdict := detail.Steps[0].Verdict if verdict == nil || verdict.Reason != "done" || verdict.ReasonSource != protocol.VerdictReasonFallback || len(verdict.Artifacts) != 0 { t.Fatalf("no-artifact verdict = %+v", verdict) } } // TestStageVerdictStoreUpgrade proves a database created before step_verdicts // exists opens cleanly, gains the table on init, and still lists and loads runs // and steps. func TestStageVerdictStoreUpgrade(t *testing.T) { ctx := context.Background() path := filepath.Join(t.TempDir(), "upgrade.db") store, err := Open(path) if err != nil { t.Fatal(err) } if _, err := store.Submit(ctx, "upgrade", Template([]string{"one"}, true)); err != nil { t.Fatal(err) } items, _ := store.List(ctx) definition, _ := store.Get(ctx, items[0].ID, 1) id, _, err := store.CreateRun(ctx, definition, "upgrade", "workspace", false, nil, nil) if err != nil { t.Fatal(err) } if err := store.BeginStep(ctx, id, 1, "one", nil); err != nil { t.Fatal(err) } if err := store.FinishStep(ctx, id, protocol.RunStep{ Sequence: 1, Stage: "one", Status: "completed", Outcome: "success", Target: "#success", Output: map[string]any{"summary": "legacy output"}, Verdict: &protocol.StageVerdict{Reason: "generated before the upgrade", ReasonSource: protocol.VerdictReasonModel}, }); err != nil { t.Fatal(err) } // Simulate a database created before step_verdicts existed. if _, err := store.db.ExecContext(ctx, "DROP TABLE step_verdicts"); err != nil { t.Fatal(err) } if err := store.Close(); err != nil { t.Fatal(err) } store, err = Open(path) if err != nil { t.Fatal("a pre-verdict database did not open cleanly", err) } defer store.Close() runs, err := store.ListRuns(ctx) if err != nil || len(runs) != 1 { t.Fatalf("legacy run list lost history: %v %v", runs, err) } detail, err := store.GetRun(ctx, id) if err != nil || len(detail.Steps) != 1 { t.Fatalf("legacy run/step did not load: %v %v", detail, err) } // The stored line is gone, so the verdict is derived at read time. if verdict := detail.Steps[0].Verdict; verdict == nil || verdict.Reason != "legacy output" || verdict.ReasonSource != protocol.VerdictReasonFallback { t.Fatalf("post-upgrade read-time verdict = %+v", verdict) } }