From bfef4659778fc972f9dce78e85afb9774dab2327 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 16 Oct 2026 12:28:27 +0100 Subject: [PATCH] Fix build: define protocol.NormalizeText used by the summariser internal/summarise called protocol.NormalizeText, which was never defined, so the tree did not compile. Add the shared normalisation helper: drop zero-width formatting runes, turn control characters into spaces and collapse whitespace runs (including the non-breaking space variants) into single spaces. Reuse it in OutputPreview instead of duplicating the mapping logic. Cover the awkward inputs (newlines, tabs, non-breaking spaces, zero-width runes, surrounding quotes, trailing punctuation) in the protocol and summarise tests. Unrelated working-tree changes are left uncommitted. --- internal/protocol/result.go | 41 +++++++++++++++++++++++----- internal/protocol/result_test.go | 26 ++++++++++++++++++ internal/summarise/summarise_test.go | 30 ++++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/internal/protocol/result.go b/internal/protocol/result.go index b720e1bf..472000b8 100644 --- a/internal/protocol/result.go +++ b/internal/protocol/result.go @@ -7,6 +7,39 @@ import ( "unicode" ) +// NormalizeText reduces text to a single clean line. Zero-width formatting +// runes (the zero-width space, the joiners, the word joiner, the byte-order +// mark, the Mongolian vowel separator and the soft hyphen) are dropped, control +// characters become spaces, and every run of whitespace - including the +// non-breaking space variants - is collapsed, so the result has no leading, +// trailing or repeated spaces. It is the shared primitive behind OutputPreview +// and search snippets, so a retained preview and a snippet stay single-line and +// safe to render in a terminal. +func NormalizeText(text string) string { + text = strings.Map(func(r rune) rune { + if isZeroWidth(r) { + return -1 + } + if unicode.IsControl(r) { + return ' ' + } + return r + }, text) + // strings.Fields splits on unicode.IsSpace, which covers the non-breaking + // space variants, and single spaces join the surviving fields. + return strings.Join(strings.Fields(text), " ") +} + +// isZeroWidth reports whether r is an invisible zero-width rune that must not +// survive into a one-line preview. +func isZeroWidth(r rune) bool { + switch r { + case '\u180e', '\u00ad', '\u200b', '\u200c', '\u200d', '\u2060', '\ufeff': + return true + } + return false +} + // OutputPreview quotes retained output, never infers accomplishments from a // successful process or invents a schema for technology-neutral workflows. func OutputPreview(output map[string]any) string { @@ -35,13 +68,7 @@ func OutputPreview(output map[string]any) string { } text = string(encoded) } - text = strings.Map(func(r rune) rune { - if unicode.IsControl(r) { - return ' ' - } - return r - }, text) - text = strings.Join(strings.Fields(text), " ") + text = NormalizeText(text) if text == "" || text == "[]" || text == "{}" { continue } diff --git a/internal/protocol/result_test.go b/internal/protocol/result_test.go index 59e5259a..830e9f57 100644 --- a/internal/protocol/result_test.go +++ b/internal/protocol/result_test.go @@ -26,3 +26,29 @@ func TestOutputPreviewUsesRetainedValuesWithoutAssumingSchema(t *testing.T) { t.Fatal("preview is not bounded by characters") } } + +func TestNormalizeTextCollapsesToSingleCleanLine(t *testing.T) { + for _, test := range []struct { + name string + in string + want string + }{ + {"whitespace and control characters", " a\t\t b\n\nc\r\nd ", "a b c d"}, + {"non-breaking spaces", "a\u00a0\u202fb\u2007c", "a b c"}, + {"zero-width joiners", "a\u200b\u200c\u200db", "ab"}, + {"word joiner and byte-order mark", "\ufeffa\u2060b", "ab"}, + {"soft hyphen and mongolian vowel separator", "so\u00adft\u180e-hyphen", "soft-hyphen"}, + {"only invisible runes", "\u00a0\t\u200b\r\n", ""}, + {"already clean", "one clean line", "one clean line"}, + } { + t.Run(test.name, func(t *testing.T) { + got := NormalizeText(test.in) + if got != test.want { + t.Fatalf("NormalizeText(%q) = %q, want %q", test.in, got, test.want) + } + if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, " ") { + t.Fatalf("NormalizeText(%q) is not single-spaced: %q", test.in, got) + } + }) + } +} diff --git a/internal/summarise/summarise_test.go b/internal/summarise/summarise_test.go index a22a4747..08c77ada 100644 --- a/internal/summarise/summarise_test.go +++ b/internal/summarise/summarise_test.go @@ -90,6 +90,36 @@ func TestSummariseTrimsToOneLine(t *testing.T) { } } +func TestSummariseNormalisesAwkwardReplies(t *testing.T) { + cases := []struct { + name string + reply string + want string + }{ + {"newlines and tabs", "First\tline\r\nsecond line", "First line"}, + {"non-breaking spaces", "Fixed\u00a0the\u202fflaky\u2007test.", "Fixed the flaky test"}, + {"zero-width characters", "Fixed \u200bthe \u200dflaky\ufeff test.", "Fixed the flaky test"}, + {"surrounding quotes and trailing punctuation", "\u201cFixed the flaky test!\u201d", "Fixed the flaky test"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s := Summariser{ + Selection: selection(), + Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) { + return tc.reply, nil + }), + } + got := s.Summarise(context.Background(), "source text") + if got != tc.want { + t.Fatalf("got %q, want %q", got, tc.want) + } + if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, "\u00a0\u200b\u200d\ufeff") { + t.Fatalf("result %q still carries whitespace or zero-width runes", got) + } + }) + } +} + func TestSummariseCapsLongReply(t *testing.T) { reply := strings.Repeat("x", OutputCap+50) s := Summariser{