package summarise import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" "unicode/utf8" "git.a-wan.net/alan/pipelines/internal/stage" ) type generatorFunc func(context.Context, stage.ModelConfig, string) (string, error) func (f generatorFunc) Generate(ctx context.Context, config stage.ModelConfig, prompt string) (string, error) { return f(ctx, config, prompt) } func selection() *Selection { return &Selection{Harness: "opencode", Executable: "opencode", Model: "p/open"} } func TestSummarisePromptIsBounded(t *testing.T) { var prompt string s := Summariser{ Selection: selection(), Generator: generatorFunc(func(_ context.Context, _ stage.ModelConfig, p string) (string, error) { prompt = p return "ok", nil }), } // Multi-byte source twice the byte cap: truncation must stay rune-safe. source := strings.Repeat("é", InputCap) if got := s.Summarise(context.Background(), source); got != "ok" { t.Fatalf("got %q, want ok", got) } if !strings.Contains(prompt, instruction) { t.Fatal("prompt is missing the fixed instruction") } if !utf8.ValidString(prompt) { t.Fatal("prompt is not valid UTF-8") } body := strings.TrimPrefix(prompt, instruction) if len(body) > InputCap { t.Fatalf("source section is %d bytes, want <= InputCap %d", len(body), InputCap) } if len(prompt) > len(instruction)+InputCap { t.Fatalf("prompt is %d bytes, want <= instruction+InputCap %d", len(prompt), len(instruction)+InputCap) } for _, want := range []string{"one line", "code fences", "untrusted"} { if !strings.Contains(instruction, want) { t.Fatalf("instruction does not mention %q", want) } } } func TestSummariseTrimsToOneLine(t *testing.T) { cases := []struct { name string reply string want string }{ {"fenced json with quotes", "```json\n\"The quick brown fox jumps.\"\n```", "The quick brown fox jumps"}, {"plain fence", "```\nPlain summary\n```", "Plain summary"}, {"surrounding quotes", `"A tidy summary."`, "A tidy summary"}, {"multiline", "First line summary.\nSecond line is ignored.", "First line summary"}, {"trailing punctuation", "Done, but check it!!!", "Done, but check it"}, {"inline fence", "```Summary```", "Summary"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return tc.reply, nil }), } got := s.Summarise(context.Background(), "source text") if got != tc.want { t.Fatalf("got %q, want %q", got, tc.want) } if strings.ContainsAny(got, "\r\n") { t.Fatalf("result %q is not a single line", got) } }) } } func TestSummariseNormalisesAwkwardReplies(t *testing.T) { cases := []struct { name string reply string want string }{ {"newlines and tabs", "First\tline\r\nsecond line", "First line"}, {"non-breaking spaces", "Fixed\u00a0the\u202fflaky\u2007test.", "Fixed the flaky test"}, {"zero-width characters", "Fixed \u200bthe \u200dflaky\ufeff test.", "Fixed the flaky test"}, {"surrounding quotes and trailing punctuation", "\u201cFixed the flaky test!\u201d", "Fixed the flaky test"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return tc.reply, nil }), } got := s.Summarise(context.Background(), "source text") if got != tc.want { t.Fatalf("got %q, want %q", got, tc.want) } if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, "\u00a0\u200b\u200d\ufeff") { t.Fatalf("result %q still carries whitespace or zero-width runes", got) } }) } } func TestSummariseCapsLongReply(t *testing.T) { reply := strings.Repeat("x", OutputCap+50) s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return reply, nil }), } got := s.Summarise(context.Background(), "source") if len([]rune(got)) != OutputCap { t.Fatalf("got %d runes, want OutputCap %d", len([]rune(got)), OutputCap) } } func TestSummariseUnusableReplies(t *testing.T) { for _, reply := range []string{"", " ", "```\n```", "```json\n```", `""`, "``", "~~~\n~~~"} { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return reply, nil }), } if got := s.Summarise(context.Background(), "source"); got != NoSummary { t.Fatalf("reply %q produced %q, want %q", reply, got, NoSummary) } } } func TestSummariseEmptySourceSkipsGeneration(t *testing.T) { calls := 0 s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { calls++ return "should not run", nil }), } for _, source := range []string{"", " ", "\n\t \r\n"} { if got := s.Summarise(context.Background(), source); got != NoSummary { t.Fatalf("source %q produced %q, want %q", source, got, NoSummary) } } if calls != 0 { t.Fatalf("generated %d times for empty/whitespace source", calls) } } func TestSummariseGeneratorErrorReturnsNoSummary(t *testing.T) { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return "", errors.New("transport down") }), } if got := s.Summarise(context.Background(), "source"); got != NoSummary { t.Fatalf("got %q, want %q", got, NoSummary) } } func TestSummariseTimeoutReturnsNoSummaryWithinBudget(t *testing.T) { s := Summariser{ Selection: selection(), Timeout: 200 * time.Millisecond, Generator: generatorFunc(func(ctx context.Context, _ stage.ModelConfig, _ string) (string, error) { <-ctx.Done() return "", ctx.Err() }), } start := time.Now() if got := s.Summarise(context.Background(), "source"); got != NoSummary { t.Fatalf("got %q, want %q", got, NoSummary) } if elapsed := time.Since(start); elapsed > 2*time.Second { t.Fatalf("blocked for %v; the hard timeout did not bound the call", elapsed) } } func TestSummariseResolvesConfiguredSelection(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "harnesses.json") fixture := `{"default":"pi","profiles":{"pi":{"executable":"pi-fixture","model":"fixture/interactive","pipeline_model":"fixture/pipeline"}}}` if err := os.WriteFile(path, []byte(fixture), 0600); err != nil { t.Fatal(err) } t.Setenv("PIPELINES_HARNESS_CONFIG", path) var captured Selection var gotConfig stage.ModelConfig s := Summariser{ NewGenerator: func(sel Selection) (stage.ModelGenerator, error) { captured = sel return generatorFunc(func(_ context.Context, config stage.ModelConfig, _ string) (string, error) { gotConfig = config return "configured", nil }), nil }, } if got := s.Summarise(context.Background(), "source"); got != "configured" { t.Fatalf("got %q, want configured", got) } want := Selection{Harness: "pi", Executable: "pi-fixture", Model: "fixture/interactive"} if captured != want { t.Fatalf("captured selection %+v, want %+v", captured, want) } if captured.Model == "fixture/pipeline" { t.Fatal("resolved PipelineModel instead of the interactive Model") } if gotConfig.Harness != "pi" || gotConfig.Provider != "fixture" || gotConfig.Model != "interactive" { t.Fatalf("generator config %+v does not match the fixture", gotConfig) } } func TestSummariseDiagnostics(t *testing.T) { var logs []string operations := &stage.OperationLog{} s := Summariser{ Selection: &Selection{Harness: "pi", Executable: "pi-fixture", Model: "p/m"}, Operations: operations, Log: func(format string, args ...any) { logs = append(logs, fmt.Sprintf(format, args...)) }, Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return "", errors.New("harness transport failure") }), } const source = "confidential source text" if got := s.Summarise(context.Background(), source); got != NoSummary { t.Fatalf("got %q, want %q", got, NoSummary) } if len(logs) != 1 { t.Fatalf("got %d log lines, want exactly 1: %v", len(logs), logs) } line := logs[0] for _, want := range []string{"harness=pi", "model=p/m", "failed=true"} { if !strings.Contains(line, want) { t.Fatalf("log %q is missing %q", line, want) } } if strings.Contains(line, source) { t.Fatalf("log leaked the source text: %q", line) } if len(operations.Failures) != 1 { t.Fatalf("got %d evidence records, want exactly 1", len(operations.Failures)) } if e := operations.Failures[0]; e.Operation != "summarise" || e.Backend != "pi" { t.Fatalf("evidence %+v does not name the summarise operation/harness", e) } } func TestTitleUsesModelLineAndRecordsSource(t *testing.T) { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return `## "Fix the flaky login test."`, nil }), } got := s.Title(context.Background(), "prompt text") if got.Text != "Fix the flaky login test" { t.Fatalf("got %q, want the cleaned model title", got.Text) } if got.Source != SourceModel || got.Model != "p/open" { t.Fatalf("title provenance = %+v, want model p/open", got) } } func TestTitleFallsBackToBoundedPromptLine(t *testing.T) { long := strings.Repeat("é", TitleCap*2) s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return "", errors.New("transport down") }), } for _, prompt := range []string{"First line\nthen ignored", " \n Fix the widget \nmore", "- bullet task here\nrest", long} { got := s.Title(context.Background(), prompt) if got.Text == "" || got.Text == NoSummary || got.Text == FallbackTitle { t.Fatalf("prompt %q produced unusable title %+v", prompt, got) } if got.Source != SourceFallback || got.Model != "" { t.Fatalf("prompt %q provenance = %+v, want fallback with no model", prompt, got) } if len([]rune(got.Text)) > TitleCap { t.Fatalf("prompt %q title has %d runes, want <= %d", prompt, len([]rune(got.Text)), TitleCap) } if strings.ContainsAny(got.Text, "\r\n") { t.Fatalf("prompt %q title %q is not a single line", prompt, got.Text) } } if got := s.Title(context.Background(), "First line\nthen ignored"); got.Text != "First line" { t.Fatalf("fallback title = %q, want the prompt's first line", got.Text) } if got := s.Title(context.Background(), "- bullet task here\nrest"); got.Text != "bullet task here" { t.Fatalf("fallback stripped markdown = %q, want the leading marker removed", got.Text) } } func TestTitleNeverEmptyForBlankPrompt(t *testing.T) { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return "", nil }), } got := s.Title(context.Background(), " \n\t") if got.Text != FallbackTitle || got.Source != SourceFallback { t.Fatalf("blank prompt title = %+v, want %q fallback", got, FallbackTitle) } } func TestExplicitTitleNormalisesAndBounds(t *testing.T) { got, ok := ExplicitTitle(" Fix the\nflaky login test ") if !ok || got.Text != "Fix the flaky login test" || got.Source != SourceExplicit || got.Model != "" { t.Fatalf("explicit title = %+v, %t; want collapsed explicit text", got, ok) } // The operator's punctuation and wording are preserved, unlike a cleaned // model line. if got, ok := ExplicitTitle("Ship v2.0!"); !ok || got.Text != "Ship v2.0!" { t.Fatalf("explicit title altered the operator's text: %+v %t", got, ok) } long := strings.Repeat("é", TitleCap*2) if got, ok := ExplicitTitle(long); !ok || len([]rune(got.Text)) != TitleCap { t.Fatalf("explicit title not capped to %d runes: %d", TitleCap, len([]rune(got.Text))) } for _, blank := range []string{"", " ", "\n\t", "\x00\x01"} { if got, ok := ExplicitTitle(blank); ok { t.Fatalf("blank explicit title %q accepted as %+v", blank, got) } } } func TestSummariseSuccessLogsAndRecordsNoFailure(t *testing.T) { var logs []string operations := &stage.OperationLog{} s := Summariser{ Selection: selection(), Operations: operations, Log: func(format string, args ...any) { logs = append(logs, fmt.Sprintf(format, args...)) }, Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return "A summary.", nil }), } if got := s.Summarise(context.Background(), "source"); got != "A summary" { t.Fatalf("got %q, want %q", got, "A summary") } if len(logs) != 1 || !strings.Contains(logs[0], "failed=false") { t.Fatalf("success log %v does not record failed=false", logs) } if len(operations.Failures) != 0 { t.Fatalf("success recorded %d failure evidence records", len(operations.Failures)) } } func TestVerdictReasonUsesModelLineAndBounds(t *testing.T) { var prompt string s := Summariser{ Selection: selection(), Generator: generatorFunc(func(_ context.Context, _ stage.ModelConfig, p string) (string, error) { prompt = p return strings.Repeat("x", OutputCap+50), nil }), } got := s.VerdictReason(context.Background(), "source text") if len([]rune(got)) != OutputCap { t.Fatalf("got %d runes, want the OutputCap %d", len([]rune(got)), OutputCap) } if !strings.Contains(prompt, verdictInstruction) { t.Fatal("verdict prompt is missing the fixed verdict instruction") } if !strings.Contains(verdictInstruction, "why the chosen option") || !strings.Contains(verdictInstruction, "untrusted") { t.Fatal("verdict instruction does not ask for a reason over untrusted data") } if got := s.VerdictReason(context.Background(), " "); got != NoSummary { t.Fatalf("blank source produced %q, want %q", got, NoSummary) } } func TestVerdictReasonIsBestEffort(t *testing.T) { s := Summariser{ Selection: selection(), Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { return "", errors.New("transport down") }), } if got := s.VerdictReason(context.Background(), "source"); got != NoSummary { t.Fatalf("generator error produced %q, want %q", got, NoSummary) } } func TestGenerateVerdictReasonSkipsEmptyOutput(t *testing.T) { for _, output := range []map[string]any{nil, {}, {"input": "only excluded keys"}} { if got := GenerateVerdictReason(context.Background(), output); got != NoSummary { t.Fatalf("output %v produced %q, want %q with no model call", output, got, NoSummary) } } }