e79ff9585f
Each completed step now exposes a verdict: the chosen option (outcome -> target), a bounded one-line reason with model/fallback/absent provenance, and the recorded artifacts the reason refers to. The store persists only the model-generated line in a new step_verdicts table and derives the fallback at read time (no backfill); the server generates the line once at completion, best-effort; the TUI renders it in the stage progress summary. The packaged debugging reference documents it.
175 lines
7.1 KiB
Go
175 lines
7.1 KiB
Go
package protocol
|
|
|
|
import (
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
func TestRunStepVerdictRoundTrip(t *testing.T) {
|
|
step := RunStep{
|
|
Sequence: 2,
|
|
Stage: "review",
|
|
Outcome: "success",
|
|
Target: "deliver",
|
|
Verdict: &StageVerdict{
|
|
Option: "success → deliver",
|
|
Reason: "Tests passed and evidence is retained",
|
|
ReasonSource: VerdictReasonModel,
|
|
Artifacts: []string{"captures/shot.png"},
|
|
},
|
|
}
|
|
encoded, err := json.Marshal(step)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var decoded RunStep
|
|
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decoded.Verdict == nil {
|
|
t.Fatal("verdict lost in round trip")
|
|
}
|
|
if decoded.Verdict.Option != step.Verdict.Option || decoded.Verdict.Reason != step.Verdict.Reason ||
|
|
decoded.Verdict.ReasonSource != step.Verdict.ReasonSource ||
|
|
len(decoded.Verdict.Artifacts) != 1 || decoded.Verdict.Artifacts[0] != "captures/shot.png" {
|
|
t.Fatalf("verdict fields lost in round trip: %+v", decoded.Verdict)
|
|
}
|
|
}
|
|
|
|
// TestRunStepOmitsAbsentVerdict mirrors TestRunStepOmitsAbsentArtifacts: a nil
|
|
// verdict marshals without the key (absent, not {}) and an absent key decodes nil.
|
|
func TestRunStepOmitsAbsentVerdict(t *testing.T) {
|
|
encoded, err := json.Marshal(RunStep{Sequence: 1, Stage: "one"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(encoded), "verdict") {
|
|
t.Fatalf("nil verdict must be omitted, not serialised: %s", encoded)
|
|
}
|
|
var decoded RunStep
|
|
if err := json.Unmarshal([]byte(`{"sequence":1}`), &decoded); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if decoded.Verdict != nil {
|
|
t.Fatalf("absent verdict must decode nil: %+v", decoded.Verdict)
|
|
}
|
|
}
|
|
|
|
func TestDeriveVerdictReasonKeyOrder(t *testing.T) {
|
|
cases := []struct {
|
|
name string
|
|
output map[string]any
|
|
want string
|
|
}{
|
|
{"verdict wins", map[string]any{"summary": "s", "reason": "r", "verdict": "v"}, "v"},
|
|
{"reason before decision", map[string]any{"summary": "s", "decision": "d", "reason": "r"}, "r"},
|
|
{"decision before summary", map[string]any{"summary": "s", "decision": "d"}, "d"},
|
|
{"summary before implementation_summary", map[string]any{"implementation_summary": "i", "summary": "s"}, "s"},
|
|
{"implementation_summary alone", map[string]any{"implementation_summary": "i"}, "i"},
|
|
{"non-string is encoded", map[string]any{"summary": 42}, "42"},
|
|
{"empty values skipped to a later key", map[string]any{"reason": " ", "summary": "s"}, "s"},
|
|
{"none yields nothing", map[string]any{"other": "x", "screenshots": []any{"a.png"}}, ""},
|
|
{"nil output yields nothing", nil, ""},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
if got := DeriveVerdictReason(tc.output); got != tc.want {
|
|
t.Fatalf("DeriveVerdictReason(%v) = %q, want %q", tc.output, got, tc.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeriveVerdictReasonIsBoundedRuneSafe(t *testing.T) {
|
|
long := strings.Repeat("é", VerdictReasonCap+50)
|
|
got := DeriveVerdictReason(map[string]any{"summary": long})
|
|
if len([]rune(got)) != VerdictReasonCap {
|
|
t.Fatalf("got %d runes, want the %d cap", len([]rune(got)), VerdictReasonCap)
|
|
}
|
|
if !utf8.ValidString(got) {
|
|
t.Fatal("bounded reason is not valid UTF-8")
|
|
}
|
|
}
|
|
|
|
func TestNewStageVerdictBoundsStoredReasonRuneSafe(t *testing.T) {
|
|
long := strings.Repeat("é", VerdictReasonCap+50)
|
|
verdict := NewStageVerdict("success", "#success", nil, nil, long, VerdictReasonModel)
|
|
if verdict == nil || len([]rune(verdict.Reason)) != VerdictReasonCap || !utf8.ValidString(verdict.Reason) {
|
|
t.Fatalf("stored reason not bounded rune-safely: %+v", verdict)
|
|
}
|
|
}
|
|
|
|
func TestVerdictOptionRendersTerminals(t *testing.T) {
|
|
cases := []struct{ outcome, target, want string }{
|
|
{"success", "#success", "success → success"},
|
|
{"error", "#failure", "error → failure"},
|
|
{"success", "review", "success → review"},
|
|
{"error", "#", "error → failure"},
|
|
}
|
|
for _, tc := range cases {
|
|
if got := VerdictOption(tc.outcome, tc.target); got != tc.want {
|
|
t.Fatalf("VerdictOption(%q,%q) = %q, want %q", tc.outcome, tc.target, got, tc.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestReferencedVerdictArtifactsMatchesRecordedOnly(t *testing.T) {
|
|
artifacts := []StageArtifact{{Name: "captures/shot.png"}, {Name: "report.txt"}, {Name: "unrelated.log"}}
|
|
// A name and a basename in the reason both match a recorded artifact; a name
|
|
// that is not recorded is never invented into the list.
|
|
got := ReferencedVerdictArtifacts("see captures/shot.png and the shot.png capture, plus ghost.png", nil, artifacts)
|
|
want := []string{"captures/shot.png"}
|
|
if len(got) != 1 || got[0] != want[0] {
|
|
t.Fatalf("reason matches = %v, want %v (missing.png is not a recorded file)", got, want)
|
|
}
|
|
// The screenshots list matches independently of the reason text.
|
|
got = ReferencedVerdictArtifacts("no file names here", map[string]any{"screenshots": []any{"unrelated.log"}}, artifacts)
|
|
if len(got) != 1 || got[0] != "unrelated.log" {
|
|
t.Fatalf("screenshots matches = %v, want unrelated.log", got)
|
|
}
|
|
// A single-string screenshots value is accepted too.
|
|
got = ReferencedVerdictArtifacts("", map[string]any{"screenshots": "report.txt"}, artifacts)
|
|
if len(got) != 1 || got[0] != "report.txt" {
|
|
t.Fatalf("string screenshots matches = %v, want report.txt", got)
|
|
}
|
|
// Duplicates are collapsed, and matches keep the recorded order.
|
|
dupes := []StageArtifact{{Name: "report.txt"}, {Name: "report.txt"}, {Name: "captures/shot.png"}}
|
|
got = ReferencedVerdictArtifacts("report.txt and captures/shot.png", nil, dupes)
|
|
if len(got) != 2 || got[0] != "report.txt" || got[1] != "captures/shot.png" {
|
|
t.Fatalf("deduped ordered matches = %v", got)
|
|
}
|
|
// A screenshots entry with no recorded match is never exposed.
|
|
if got := ReferencedVerdictArtifacts("", map[string]any{"screenshots": []any{"ghost.png"}}, artifacts); len(got) != 0 {
|
|
t.Fatalf("unrecorded screenshots entry exposed: %v", got)
|
|
}
|
|
// No reason, no screenshots: no artifacts.
|
|
if got := ReferencedVerdictArtifacts("", nil, artifacts); len(got) != 0 {
|
|
t.Fatalf("artifacts exposed without a reference: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestNewStageVerdict(t *testing.T) {
|
|
// A running step carries neither outcome nor target and has no verdict.
|
|
if got := NewStageVerdict("", "", nil, nil, "", ""); got != nil {
|
|
t.Fatalf("running step verdict = %+v, want nil", got)
|
|
}
|
|
// A completed visit with no recorded artifacts still carries Option and Reason.
|
|
got := NewStageVerdict("success", "review", map[string]any{"summary": "did the work"}, nil, "", "")
|
|
if got == nil || got.Option != "success → review" || got.Reason != "did the work" ||
|
|
got.ReasonSource != VerdictReasonFallback || len(got.Artifacts) != 0 {
|
|
t.Fatalf("fallback verdict = %+v", got)
|
|
}
|
|
// A stored model line wins over the output and records its provenance.
|
|
got = NewStageVerdict("success", "review", map[string]any{"summary": "output text"}, nil, "the model line", VerdictReasonModel)
|
|
if got == nil || got.Reason != "the model line" || got.ReasonSource != VerdictReasonModel {
|
|
t.Fatalf("stored verdict = %+v", got)
|
|
}
|
|
// No usable reason key: honest absence with no invented text.
|
|
got = NewStageVerdict("success", "#success", map[string]any{"other": "x"}, nil, "", "")
|
|
if got == nil || got.Reason != "" || got.ReasonSource != VerdictReasonAbsent {
|
|
t.Fatalf("absent-reason verdict = %+v", got)
|
|
}
|
|
}
|