Fix build: define protocol.NormalizeText used by the summariser
internal/summarise called protocol.NormalizeText, which was never defined, so the tree did not compile. Add the shared normalisation helper: drop zero-width formatting runes, turn control characters into spaces and collapse whitespace runs (including the non-breaking space variants) into single spaces. Reuse it in OutputPreview instead of duplicating the mapping logic. Cover the awkward inputs (newlines, tabs, non-breaking spaces, zero-width runes, surrounding quotes, trailing punctuation) in the protocol and summarise tests. Unrelated working-tree changes are left uncommitted.
This commit is contained in:
@@ -7,6 +7,39 @@ import (
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// NormalizeText reduces text to a single clean line. Zero-width formatting
|
||||
// runes (the zero-width space, the joiners, the word joiner, the byte-order
|
||||
// mark, the Mongolian vowel separator and the soft hyphen) are dropped, control
|
||||
// characters become spaces, and every run of whitespace - including the
|
||||
// non-breaking space variants - is collapsed, so the result has no leading,
|
||||
// trailing or repeated spaces. It is the shared primitive behind OutputPreview
|
||||
// and search snippets, so a retained preview and a snippet stay single-line and
|
||||
// safe to render in a terminal.
|
||||
func NormalizeText(text string) string {
|
||||
text = strings.Map(func(r rune) rune {
|
||||
if isZeroWidth(r) {
|
||||
return -1
|
||||
}
|
||||
if unicode.IsControl(r) {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, text)
|
||||
// strings.Fields splits on unicode.IsSpace, which covers the non-breaking
|
||||
// space variants, and single spaces join the surviving fields.
|
||||
return strings.Join(strings.Fields(text), " ")
|
||||
}
|
||||
|
||||
// isZeroWidth reports whether r is an invisible zero-width rune that must not
|
||||
// survive into a one-line preview.
|
||||
func isZeroWidth(r rune) bool {
|
||||
switch r {
|
||||
case '\u180e', '\u00ad', '\u200b', '\u200c', '\u200d', '\u2060', '\ufeff':
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// OutputPreview quotes retained output, never infers accomplishments from a
|
||||
// successful process or invents a schema for technology-neutral workflows.
|
||||
func OutputPreview(output map[string]any) string {
|
||||
@@ -35,13 +68,7 @@ func OutputPreview(output map[string]any) string {
|
||||
}
|
||||
text = string(encoded)
|
||||
}
|
||||
text = strings.Map(func(r rune) rune {
|
||||
if unicode.IsControl(r) {
|
||||
return ' '
|
||||
}
|
||||
return r
|
||||
}, text)
|
||||
text = strings.Join(strings.Fields(text), " ")
|
||||
text = NormalizeText(text)
|
||||
if text == "" || text == "[]" || text == "{}" {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -26,3 +26,29 @@ func TestOutputPreviewUsesRetainedValuesWithoutAssumingSchema(t *testing.T) {
|
||||
t.Fatal("preview is not bounded by characters")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeTextCollapsesToSingleCleanLine(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
in string
|
||||
want string
|
||||
}{
|
||||
{"whitespace and control characters", " a\t\t b\n\nc\r\nd ", "a b c d"},
|
||||
{"non-breaking spaces", "a\u00a0\u202fb\u2007c", "a b c"},
|
||||
{"zero-width joiners", "a\u200b\u200c\u200db", "ab"},
|
||||
{"word joiner and byte-order mark", "\ufeffa\u2060b", "ab"},
|
||||
{"soft hyphen and mongolian vowel separator", "so\u00adft\u180e-hyphen", "soft-hyphen"},
|
||||
{"only invisible runes", "\u00a0\t\u200b\r\n", ""},
|
||||
{"already clean", "one clean line", "one clean line"},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got := NormalizeText(test.in)
|
||||
if got != test.want {
|
||||
t.Fatalf("NormalizeText(%q) = %q, want %q", test.in, got, test.want)
|
||||
}
|
||||
if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, " ") {
|
||||
t.Fatalf("NormalizeText(%q) is not single-spaced: %q", test.in, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,36 @@ func TestSummariseTrimsToOneLine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummariseNormalisesAwkwardReplies(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
reply string
|
||||
want string
|
||||
}{
|
||||
{"newlines and tabs", "First\tline\r\nsecond line", "First line"},
|
||||
{"non-breaking spaces", "Fixed\u00a0the\u202fflaky\u2007test.", "Fixed the flaky test"},
|
||||
{"zero-width characters", "Fixed \u200bthe \u200dflaky\ufeff test.", "Fixed the flaky test"},
|
||||
{"surrounding quotes and trailing punctuation", "\u201cFixed the flaky test!\u201d", "Fixed the flaky test"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := Summariser{
|
||||
Selection: selection(),
|
||||
Generator: generatorFunc(func(context.Context, stage.ModelConfig, string) (string, error) {
|
||||
return tc.reply, nil
|
||||
}),
|
||||
}
|
||||
got := s.Summarise(context.Background(), "source text")
|
||||
if got != tc.want {
|
||||
t.Fatalf("got %q, want %q", got, tc.want)
|
||||
}
|
||||
if strings.ContainsAny(got, "\r\n\t") || strings.Contains(got, "\u00a0\u200b\u200d\ufeff") {
|
||||
t.Fatalf("result %q still carries whitespace or zero-width runes", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummariseCapsLongReply(t *testing.T) {
|
||||
reply := strings.Repeat("x", OutputCap+50)
|
||||
s := Summariser{
|
||||
|
||||
Reference in New Issue
Block a user