Files
pipelines/internal/summarise/summarise_test.go
T
alan bfef465977 Fix build: define protocol.NormalizeText used by the summariser
internal/summarise called protocol.NormalizeText, which was never defined, so the tree did not compile. Add the shared normalisation helper: drop zero-width formatting runes, turn control characters into spaces and collapse whitespace runs (including the non-breaking space variants) into single spaces. Reuse it in OutputPreview instead of duplicating the mapping logic.

Cover the awkward inputs (newlines, tabs, non-breaking spaces, zero-width runes, surrounding quotes, trailing punctuation) in the protocol and summarise tests. Unrelated working-tree changes are left uncommitted.
2026-10-16 12:28:27 +01:00

291 lines
9.5 KiB
Go

package summarise
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"unicode/utf8"
"git.a-wan.net/alan/pipelines/internal/stage"
)
type generatorFunc func(context.Context, stage.ModelConfig, string) (string, error)
func (f generatorFunc) Generate(ctx context.Context, config stage.ModelConfig, prompt string) (string, error) {
return f(ctx, config, prompt)
}
func selection() *Selection {
return &Selection{Harness: "opencode", Executable: "opencode", Model: "p/open"}
}
func TestSummarisePromptIsBounded(t *testing.T) {
var prompt string
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(_ context.Context, _ stage.ModelConfig, p string) (string, error) {
prompt = p
return "ok", nil
}),
}
// Multi-byte source twice the byte cap: truncation must stay rune-safe.
source := strings.Repeat("é", InputCap)
if got := s.Summarise(context.Background(), source); got != "ok" {
t.Fatalf("got %q, want ok", got)
}
if !strings.Contains(prompt, instruction) {
t.Fatal("prompt is missing the fixed instruction")
}
if !utf8.ValidString(prompt) {
t.Fatal("prompt is not valid UTF-8")
}
body := strings.TrimPrefix(prompt, instruction)
if len(body) > InputCap {
t.Fatalf("source section is %d bytes, want <= InputCap %d", len(body), InputCap)
}
if len(prompt) > len(instruction)+InputCap {
t.Fatalf("prompt is %d bytes, want <= instruction+InputCap %d", len(prompt), len(instruction)+InputCap)
}
for _, want := range []string{"one line", "code fences", "untrusted"} {
if !strings.Contains(instruction, want) {
t.Fatalf("instruction does not mention %q", want)
}
}
}
func TestSummariseTrimsToOneLine(t *testing.T) {
cases := []struct {
name string
reply string
want string
}{
{"fenced json with quotes", "```json\n\"The quick brown fox jumps.\"\n```", "The quick brown fox jumps"},
{"plain fence", "```\nPlain summary\n```", "Plain summary"},
{"surrounding quotes", `"A tidy summary."`, "A tidy summary"},
{"multiline", "First line summary.\nSecond line is ignored.", "First line summary"},
{"trailing punctuation", "Done, but check it!!!", "Done, but check it"},
{"inline fence", "```Summary```", "Summary"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return tc.reply, nil
}),
}
got := s.Summarise(context.Background(), "source text")
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
if strings.ContainsAny(got, "\r\n") {
t.Fatalf("result %q is not a single line", got)
}
})
}
}
func TestSummariseNormalisesAwkwardReplies(t *testing.T) {
cases := []struct {
name string
reply string
want string
}{
{"newlines and tabs", "First\tline\r\nsecond line", "First line"},
{"non-breaking spaces", "Fixed\u00a0the\u202fflaky\u2007test.", "Fixed the flaky test"},
{"zero-width characters", "Fixed \u200bthe \u200dflaky\ufeff test.", "Fixed the flaky test"},
{"surrounding quotes and trailing punctuation", "\u201cFixed the flaky test!\u201d", "Fixed the flaky test"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return tc.reply, nil
}),
}
got := s.Summarise(context.Background(), "source text")
if got != tc.want {
t.Fatalf("got %q, want %q", got, tc.want)
}
if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, "\u00a0\u200b\u200d\ufeff") {
t.Fatalf("result %q still carries whitespace or zero-width runes", got)
}
})
}
}
func TestSummariseCapsLongReply(t *testing.T) {
reply := strings.Repeat("x", OutputCap+50)
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return reply, nil
}),
}
got := s.Summarise(context.Background(), "source")
if len([]rune(got)) != OutputCap {
t.Fatalf("got %d runes, want OutputCap %d", len([]rune(got)), OutputCap)
}
}
func TestSummariseUnusableReplies(t *testing.T) {
for _, reply := range []string{"", " ", "```\n```", "```json\n```", `""`, "``", "~~~\n~~~"} {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return reply, nil
}),
}
if got := s.Summarise(context.Background(), "source"); got != NoSummary {
t.Fatalf("reply %q produced %q, want %q", reply, got, NoSummary)
}
}
}
func TestSummariseEmptySourceSkipsGeneration(t *testing.T) {
calls := 0
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
calls++
return "should not run", nil
}),
}
for _, source := range []string{"", " ", "\n\t \r\n"} {
if got := s.Summarise(context.Background(), source); got != NoSummary {
t.Fatalf("source %q produced %q, want %q", source, got, NoSummary)
}
}
if calls != 0 {
t.Fatalf("generated %d times for empty/whitespace source", calls)
}
}
func TestSummariseGeneratorErrorReturnsNoSummary(t *testing.T) {
s := Summariser{
Selection: selection(),
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", errors.New("transport down")
}),
}
if got := s.Summarise(context.Background(), "source"); got != NoSummary {
t.Fatalf("got %q, want %q", got, NoSummary)
}
}
func TestSummariseTimeoutReturnsNoSummaryWithinBudget(t *testing.T) {
s := Summariser{
Selection: selection(),
Timeout: 200 * time.Millisecond,
Generator: generatorFunc(func(ctx context.Context, _ stage.ModelConfig, _ string) (string, error) {
<-ctx.Done()
return "", ctx.Err()
}),
}
start := time.Now()
if got := s.Summarise(context.Background(), "source"); got != NoSummary {
t.Fatalf("got %q, want %q", got, NoSummary)
}
if elapsed := time.Since(start); elapsed > 2*time.Second {
t.Fatalf("blocked for %v; the hard timeout did not bound the call", elapsed)
}
}
func TestSummariseResolvesConfiguredSelection(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "harnesses.json")
fixture := `{"default":"pi","profiles":{"pi":{"executable":"pi-fixture","model":"fixture/interactive","pipeline_model":"fixture/pipeline"}}}`
if err := os.WriteFile(path, []byte(fixture), 0600); err != nil {
t.Fatal(err)
}
t.Setenv("PIPELINES_HARNESS_CONFIG", path)
var captured Selection
var gotConfig stage.ModelConfig
s := Summariser{
NewGenerator: func(sel Selection) (stage.ModelGenerator, error) {
captured = sel
return generatorFunc(func(_ context.Context, config stage.ModelConfig, _ string) (string, error) {
gotConfig = config
return "configured", nil
}), nil
},
}
if got := s.Summarise(context.Background(), "source"); got != "configured" {
t.Fatalf("got %q, want configured", got)
}
want := Selection{Harness: "pi", Executable: "pi-fixture", Model: "fixture/interactive"}
if captured != want {
t.Fatalf("captured selection %+v, want %+v", captured, want)
}
if captured.Model == "fixture/pipeline" {
t.Fatal("resolved PipelineModel instead of the interactive Model")
}
if gotConfig.Harness != "pi" || gotConfig.Provider != "fixture" || gotConfig.Model != "interactive" {
t.Fatalf("generator config %+v does not match the fixture", gotConfig)
}
}
func TestSummariseDiagnostics(t *testing.T) {
var logs []string
operations := &stage.OperationLog{}
s := Summariser{
Selection: &Selection{Harness: "pi", Executable: "pi-fixture", Model: "p/m"},
Operations: operations,
Log: func(format string, args ...any) { logs = append(logs, fmt.Sprintf(format, args...)) },
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "", errors.New("harness transport failure")
}),
}
const source = "confidential source text"
if got := s.Summarise(context.Background(), source); got != NoSummary {
t.Fatalf("got %q, want %q", got, NoSummary)
}
if len(logs) != 1 {
t.Fatalf("got %d log lines, want exactly 1: %v", len(logs), logs)
}
line := logs[0]
for _, want := range []string{"harness=pi", "model=p/m", "failed=true"} {
if !strings.Contains(line, want) {
t.Fatalf("log %q is missing %q", line, want)
}
}
if strings.Contains(line, source) {
t.Fatalf("log leaked the source text: %q", line)
}
if len(operations.Failures) != 1 {
t.Fatalf("got %d evidence records, want exactly 1", len(operations.Failures))
}
if e := operations.Failures[0]; e.Operation != "summarise" || e.Backend != "pi" {
t.Fatalf("evidence %+v does not name the summarise operation/harness", e)
}
}
func TestSummariseSuccessLogsAndRecordsNoFailure(t *testing.T) {
var logs []string
operations := &stage.OperationLog{}
s := Summariser{
Selection: selection(),
Operations: operations,
Log: func(format string, args ...any) { logs = append(logs, fmt.Sprintf(format, args...)) },
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
return "A summary.", nil
}),
}
if got := s.Summarise(context.Background(), "source"); got != "A summary" {
t.Fatalf("got %q, want %q", got, "A summary")
}
if len(logs) != 1 || !strings.Contains(logs[0], "failed=false") {
t.Fatalf("success log %v does not record failed=false", logs)
}
if len(operations.Failures) != 0 {
t.Fatalf("success recorded %d failure evidence records", len(operations.Failures))
}
}