Files
pipelines/internal/summarise/summarise_test.go
T
alan e79ff9585f Derive a plain-language verdict for every completed stage visit
Each completed step now exposes a verdict: the chosen option (outcome -> target),
a bounded one-line reason with model/fallback/absent provenance, and the recorded
artifacts the reason refers to. The store persists only the model-generated line
in a new step_verdicts table and derives the fallback at read time (no backfill);
the server generates the line once at completion, best-effort; the TUI renders it
in the stage progress summary. The packaged debugging reference documents it.
2026-10-16 13:54:48 +01:00

416 lines
14 KiB
Go

package summarise
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/stage"
)
type generatorFunc func(context.Context, stage.ModelConfig, string) (string, error)
func (f generatorFunc) Generate(ctx context.Context, config stage.ModelConfig, prompt string) (string, error) {
return f(ctx, config, prompt)
}
func selection() *Selection {
return &Selection{Harness: "opencode", Executable: "opencode", Model: "p/open"}
}
func TestSummarisePromptIsBounded(t *testing.T) {
var prompt string
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(_ context.Context, _ stage.ModelConfig, p string) (string, error) {
prompt = p
return "ok", nil
}),
}
// Multi-byte source twice the byte cap: truncation must stay rune-safe.
source := strings.Repeat("é", InputCap)
if got := s.Summarise(context.Background(), source); got != "ok" {
t.Fatalf("got %q, want ok", got)
}
if !strings.Contains(prompt, instruction) {
t.Fatal("prompt is missing the fixed instruction")
}
if !utf8.ValidString(prompt) {
t.Fatal("prompt is not valid UTF-8")
}
body := strings.TrimPrefix(prompt, instruction)
if len(body) > InputCap {
t.Fatalf("source section is %d bytes, want <= InputCap %d", len(body), InputCap)
}
if len(prompt) > len(instruction)+InputCap {
t.Fatalf("prompt is %d bytes, want <= instruction+InputCap %d", len(prompt), len(instruction)+InputCap)
}
for _, want := range []string{"one line", "code fences", "untrusted"} {
if !strings.Contains(instruction, want) {
t.Fatalf("instruction does not mention %q", want)
}
}
}
func TestSummariseTrimsToOneLine(t *testing.T) {
cases := []struct {
name string
reply string
want string
}{
{"fenced json with quotes", "```json\n\"The quick brown fox jumps.\"\n```", "The quick brown fox jumps"},
{"plain fence", "```\nPlain summary\n```", "Plain summary"},
{"surrounding quotes", `"A tidy summary."`, "A tidy summary"},
{"multiline", "First line summary.\nSecond line is ignored.", "First line summary"},
{"trailing punctuation", "Done, but check it!!!", "Done, but check it"},
{"inline fence", "```Summary```", "Summary"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return tc.reply, nil
}),
}
got := s.Summarise(context.Background(), "source text")
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
if strings.ContainsAny(got, "\r\n") {
t.Fatalf("result %q is not a single line", got)
}
})
}
}
func TestSummariseNormalisesAwkwardReplies(t *testing.T) {
cases := []struct {
name string
reply string
want string
}{
{"newlines and tabs", "First\tline\r\nsecond line", "First line"},
{"non-breaking spaces", "Fixed\u00a0the\u202fflaky\u2007test.", "Fixed the flaky test"},
{"zero-width characters", "Fixed \u200bthe \u200dflaky\ufeff test.", "Fixed the flaky test"},
{"surrounding quotes and trailing punctuation", "\u201cFixed the flaky test!\u201d", "Fixed the flaky test"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return tc.reply, nil
}),
}
got := s.Summarise(context.Background(), "source text")
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, "\u00a0\u200b\u200d\ufeff") {
t.Fatalf("result %q still carries whitespace or zero-width runes", got)
}
})
}
}
func TestSummariseCapsLongReply(t *testing.T) {
reply := strings.Repeat("x", OutputCap+50)
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return reply, nil
}),
}
got := s.Summarise(context.Background(), "source")
if len([]rune(got)) != OutputCap {
t.Fatalf("got %d runes, want OutputCap %d", len([]rune(got)), OutputCap)
}
}
func TestSummariseUnusableReplies(t *testing.T) {
for _, reply := range []string{"", " ", "```\n```", "```json\n```", `""`, "``", "~~~\n~~~"} {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return reply, nil
}),
}
if got := s.Summarise(context.Background(), "source"); got != NoSummary {
t.Fatalf("reply %q produced %q, want %q", reply, got, NoSummary)
}
}
}
func TestSummariseEmptySourceSkipsGeneration(t *testing.T) {
calls := 0
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
calls++
return "should not run", nil
}),
}
for _, source := range []string{"", " ", "\n\t \r\n"} {
if got := s.Summarise(context.Background(), source); got != NoSummary {
t.Fatalf("source %q produced %q, want %q", source, got, NoSummary)
}
}
if calls != 0 {
t.Fatalf("generated %d times for empty/whitespace source", calls)
}
}
func TestSummariseGeneratorErrorReturnsNoSummary(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", errors.New("transport down")
}),
}
if got := s.Summarise(context.Background(), "source"); got != NoSummary {
t.Fatalf("got %q, want %q", got, NoSummary)
}
}
func TestSummariseTimeoutReturnsNoSummaryWithinBudget(t *testing.T) {
s := Summariser{
Selection: selection(),
Timeout: 200 * time.Millisecond,
Generator: generatorFunc(func(ctx context.Context, _ stage.ModelConfig, _ string) (string, error) {
<-ctx.Done()
return "", ctx.Err()
}),
}
start := time.Now()
if got := s.Summarise(context.Background(), "source"); got != NoSummary {
t.Fatalf("got %q, want %q", got, NoSummary)
}
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Fatalf("blocked for %v; the hard timeout did not bound the call", elapsed)
}
}
func TestSummariseResolvesConfiguredSelection(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "harnesses.json")
fixture := `{"default":"pi","profiles":{"pi":{"executable":"pi-fixture","model":"fixture/interactive","pipeline_model":"fixture/pipeline"}}}`
if err := os.WriteFile(path, []byte(fixture), 0600); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_HARNESS_CONFIG", path)
var captured Selection
var gotConfig stage.ModelConfig
s := Summariser{
NewGenerator: func(sel Selection) (stage.ModelGenerator, error) {
captured = sel
return generatorFunc(func(_ context.Context, config stage.ModelConfig, _ string) (string, error) {
gotConfig = config
return "configured", nil
}), nil
},
}
if got := s.Summarise(context.Background(), "source"); got != "configured" {
t.Fatalf("got %q, want configured", got)
}
want := Selection{Harness: "pi", Executable: "pi-fixture", Model: "fixture/interactive"}
if captured != want {
t.Fatalf("captured selection %+v, want %+v", captured, want)
}
if captured.Model == "fixture/pipeline" {
t.Fatal("resolved PipelineModel instead of the interactive Model")
}
if gotConfig.Harness != "pi" || gotConfig.Provider != "fixture" || gotConfig.Model != "interactive" {
t.Fatalf("generator config %+v does not match the fixture", gotConfig)
}
}
func TestSummariseDiagnostics(t *testing.T) {
var logs []string
operations := &stage.OperationLog{}
s := Summariser{
Selection: &Selection{Harness: "pi", Executable: "pi-fixture", Model: "p/m"},
Operations: operations,
Log: func(format string, args ...any) { logs = append(logs, fmt.Sprintf(format, args...)) },
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", errors.New("harness transport failure")
}),
}
const source = "confidential source text"
if got := s.Summarise(context.Background(), source); got != NoSummary {
t.Fatalf("got %q, want %q", got, NoSummary)
}
if len(logs) != 1 {
t.Fatalf("got %d log lines, want exactly 1: %v", len(logs), logs)
}
line := logs[0]
for _, want := range []string{"harness=pi", "model=p/m", "failed=true"} {
if !strings.Contains(line, want) {
t.Fatalf("log %q is missing %q", line, want)
}
}
if strings.Contains(line, source) {
t.Fatalf("log leaked the source text: %q", line)
}
if len(operations.Failures) != 1 {
t.Fatalf("got %d evidence records, want exactly 1", len(operations.Failures))
}
if e := operations.Failures[0]; e.Operation != "summarise" || e.Backend != "pi" {
t.Fatalf("evidence %+v does not name the summarise operation/harness", e)
}
}
func TestTitleUsesModelLineAndRecordsSource(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return `## "Fix the flaky login test."`, nil
}),
}
got := s.Title(context.Background(), "prompt text")
if got.Text != "Fix the flaky login test" {
t.Fatalf("got %q, want the cleaned model title", got.Text)
}
if got.Source != SourceModel || got.Model != "p/open" {
t.Fatalf("title provenance = %+v, want model p/open", got)
}
}
func TestTitleFallsBackToBoundedPromptLine(t *testing.T) {
long := strings.Repeat("é", TitleCap*2)
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", errors.New("transport down")
}),
}
for _, prompt := range []string{"First line\nthen ignored", " \n Fix the widget \nmore", "- bullet task here\nrest", long} {
got := s.Title(context.Background(), prompt)
if got.Text == "" || got.Text == NoSummary || got.Text == FallbackTitle {
t.Fatalf("prompt %q produced unusable title %+v", prompt, got)
}
if got.Source != SourceFallback || got.Model != "" {
t.Fatalf("prompt %q provenance = %+v, want fallback with no model", prompt, got)
}
if len([]rune(got.Text)) > TitleCap {
t.Fatalf("prompt %q title has %d runes, want <= %d", prompt, len([]rune(got.Text)), TitleCap)
}
if strings.ContainsAny(got.Text, "\r\n") {
t.Fatalf("prompt %q title %q is not a single line", prompt, got.Text)
}
}
if got := s.Title(context.Background(), "First line\nthen ignored"); got.Text != "First line" {
t.Fatalf("fallback title = %q, want the prompt's first line", got.Text)
}
if got := s.Title(context.Background(), "- bullet task here\nrest"); got.Text != "bullet task here" {
t.Fatalf("fallback stripped markdown = %q, want the leading marker removed", got.Text)
}
}
func TestTitleNeverEmptyForBlankPrompt(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", nil
}),
}
got := s.Title(context.Background(), " \n\t")
if got.Text != FallbackTitle || got.Source != SourceFallback {
t.Fatalf("blank prompt title = %+v, want %q fallback", got, FallbackTitle)
}
}
func TestExplicitTitleNormalisesAndBounds(t *testing.T) {
got, ok := ExplicitTitle(" Fix the\nflaky login test ")
if !ok || got.Text != "Fix the flaky login test" || got.Source != SourceExplicit || got.Model != "" {
t.Fatalf("explicit title = %+v, %t; want collapsed explicit text", got, ok)
}
// The operator's punctuation and wording are preserved, unlike a cleaned
// model line.
if got, ok := ExplicitTitle("Ship v2.0!"); !ok || got.Text != "Ship v2.0!" {
t.Fatalf("explicit title altered the operator's text: %+v %t", got, ok)
}
long := strings.Repeat("é", TitleCap*2)
if got, ok := ExplicitTitle(long); !ok || len([]rune(got.Text)) != TitleCap {
t.Fatalf("explicit title not capped to %d runes: %d", TitleCap, len([]rune(got.Text)))
}
for _, blank := range []string{"", " ", "\n\t", "\x00\x01"} {
if got, ok := ExplicitTitle(blank); ok {
t.Fatalf("blank explicit title %q accepted as %+v", blank, got)
}
}
}
func TestSummariseSuccessLogsAndRecordsNoFailure(t *testing.T) {
var logs []string
operations := &stage.OperationLog{}
s := Summariser{
Selection: selection(),
Operations: operations,
Log: func(format string, args ...any) { logs = append(logs, fmt.Sprintf(format, args...)) },
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "A summary.", nil
}),
}
if got := s.Summarise(context.Background(), "source"); got != "A summary" {
t.Fatalf("got %q, want %q", got, "A summary")
}
if len(logs) != 1 || !strings.Contains(logs[0], "failed=false") {
t.Fatalf("success log %v does not record failed=false", logs)
}
if len(operations.Failures) != 0 {
t.Fatalf("success recorded %d failure evidence records", len(operations.Failures))
}
}
func TestVerdictReasonUsesModelLineAndBounds(t *testing.T) {
var prompt string
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(_ context.Context, _ stage.ModelConfig, p string) (string, error) {
prompt = p
return strings.Repeat("x", OutputCap+50), nil
}),
}
got := s.VerdictReason(context.Background(), "source text")
if len([]rune(got)) != OutputCap {
t.Fatalf("got %d runes, want the OutputCap %d", len([]rune(got)), OutputCap)
}
if !strings.Contains(prompt, verdictInstruction) {
t.Fatal("verdict prompt is missing the fixed verdict instruction")
}
if !strings.Contains(verdictInstruction, "why the chosen option") || !strings.Contains(verdictInstruction, "untrusted") {
t.Fatal("verdict instruction does not ask for a reason over untrusted data")
}
if got := s.VerdictReason(context.Background(), " "); got != NoSummary {
t.Fatalf("blank source produced %q, want %q", got, NoSummary)
}
}
func TestVerdictReasonIsBestEffort(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", errors.New("transport down")
}),
}
if got := s.VerdictReason(context.Background(), "source"); got != NoSummary {
t.Fatalf("generator error produced %q, want %q", got, NoSummary)
}
}
func TestGenerateVerdictReasonSkipsEmptyOutput(t *testing.T) {
for _, output := range []map[string]any{nil, {}, {"input": "only excluded keys"}} {
if got := GenerateVerdictReason(context.Background(), output); got != NoSummary {
t.Fatalf("output %v produced %q, want %q with no model call", output, got, NoSummary)
}
}
}