From df0614e2f78861be24117cbd98b31a4428255064 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 13 Nov 2026 09:41:02 +0000 Subject: [PATCH] Steady read-aloud highlight and a calmer default pace The spoken word is now painted behind unchanged text instead of restyling the paragraph, so words never reshape, shift or rewrap as the highlight moves. Revealing the word measures its own boxes against the visible area, so a reversed transcript no longer jumps for a word already in view. Sentence breaks the reading adds no longer map onto the next item's first letter. 1.0x now reads at 0.8 of the platform's normal rate. --- .../speech/flutter_tts_speech_engine.dart | 14 +- .../features/chat/speech/speech_document.dart | 14 +- .../ui/transcript/chat_markdown_speech.dart | 154 +++++++++++++----- .../ui/transcript/chat_markdown_view.dart | 40 +++-- .../speech/platform_speech_rate_test.dart | 14 ++ .../chat/speech/speech_document_test.dart | 13 ++ .../chat_markdown_read_aloud_test.dart | 139 ++++++++++++++-- 7 files changed, 310 insertions(+), 78 deletions(-) create mode 100644 app/test/features/chat/speech/platform_speech_rate_test.dart diff --git a/app/lib/features/chat/speech/flutter_tts_speech_engine.dart b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart index 2ab2700e..680cba83 100644 --- a/app/lib/features/chat/speech/flutter_tts_speech_engine.dart +++ b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart @@ -56,7 +56,7 @@ class FlutterTtsSpeechEngine extends SpeechEngine { @override Future setRate(double rate) async => - _tts.setSpeechRate(_appleScale ? (rate * 0.5).clamp(0, 1) : rate); + _tts.setSpeechRate(platformSpeechRate(rate, appleScale: _appleScale)); @override Future setVoice(SpeechVoice? voice) async { @@ -110,3 +110,15 @@ class FlutterTtsSpeechEngine extends SpeechEngine { ]; } } + +/// The app's 1.0× reads at a calm listening pace, a little under the +/// platform's own normal rate, which sounds rushed for long replies. +const readAloudPace = 0.8; + +/// The platform rate for the app's [rate] multiplier (1.0× = [readAloudPace] +/// of the platform's normal): AVSpeechSynthesizer's normal is 0.5 on a 0..1 +/// scale, Android's TextToSpeech and the Web Speech API use 1.0. +double platformSpeechRate(double rate, {required bool appleScale}) { + final paced = rate * readAloudPace; + return appleScale ? (paced * 0.5).clamp(0, 1) : paced; +} diff --git a/app/lib/features/chat/speech/speech_document.dart b/app/lib/features/chat/speech/speech_document.dart index 8623367e..4024da74 100644 --- a/app/lib/features/chat/speech/speech_document.dart +++ b/app/lib/features/chat/speech/speech_document.dart @@ -14,6 +14,7 @@ class SpeechDocument { required this.text, required this.parts, required this.sources, + this.added = const {}, }); final String text; @@ -22,6 +23,11 @@ class SpeechDocument { final List parts; final List sources; + /// Spoken indexes the reading added (a sentence break between blocks, the + /// "Code block." phrase): they come from no visible text, so a word that + /// reaches into them highlights only its own characters. + final Set added; + /// The index of the part whose markdown is [markdown], or -1. int partOf(String markdown) => parts.indexOf(markdown.trim()); @@ -33,7 +39,7 @@ class SpeechDocument { final last = range.end.clamp(0, sources.length); for (var index = range.start.clamp(0, last); index < last; index += 1) { final source = sources[index]; - if (source.part != part) continue; + if (source.part != part || added.contains(index)) continue; if (start == null || source.offset < start) start = source.offset; if (end == null || source.offset + 1 > end) end = source.offset + 1; } @@ -61,12 +67,14 @@ SpeechDocument speechDocumentFor(List markdownParts) { text: builder.spoken.toString(), parts: parts, sources: builder.sources, + added: builder.added, ); } class _SpeechBuilder { final spoken = StringBuffer(); final sources = []; + final added = {}; String _markdown = ''; var _part = 0; var _cursor = 0; @@ -103,11 +111,13 @@ class _SpeechBuilder { _write(' ', at); } + /// Writes text the reading adds, which no visible character stands for. void _write(String value, int at) { - spoken.write(value); for (var index = 0; index < value.length; index += 1) { + added.add(spoken.length + index); sources.add((part: _part, offset: at)); } + spoken.write(value); } void _phrase(String value, int at) { diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart index 72c1dbe7..b82404eb 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart @@ -28,34 +28,92 @@ SpeechRange? spokenSourceIn( return (start: range.start + lead, end: range.end + lead); } -/// [text] as spans whose characters with source [offsets] inside [range] are -/// filled with [fill]; plain text when nothing in it is spoken. -TextSpan spokenTextSpan( - String text, - List offsets, - SpeechRange? range, - Color fill, -) { - if (range == null || offsets.isEmpty) return TextSpan(text: text); - bool spoken(int index) => - offsets[index] >= range.start && offsets[index] < range.end; - final children = []; - var runStart = 0; - for (var index = 1; index <= text.length; index += 1) { - if (index < text.length && spoken(index) == spoken(runStart)) continue; - children.add( - TextSpan( - text: text.substring(runStart, index), - style: spoken(runStart) ? TextStyle(backgroundColor: fill) : null, - ), - ); - runStart = index; +/// Paints the spoken word's highlight behind the rendered text. +/// +/// The text itself is never restyled while speaking: splitting a paragraph +/// into differently styled runs changes how it is shaped, so words would +/// shift and lines rewrap as the highlight moves. Instead this painter, under +/// the text, looks up the spoken characters' boxes in the laid-out paragraphs +/// ([segments]) and fills them; each new word only repaints. +class SpokenWordPainter extends CustomPainter { + SpokenWordPainter({ + required this.anchor, + required this.segments, + required this.range, + required this.color, + super.repaint, + }); + + /// The key of the [CustomPaint] this painter draws in. + final GlobalKey anchor; + final List segments; + final SpeechRange? Function() range; + final Color color; + + @override + void paint(Canvas canvas, Size size) { + final spoken = range(); + final box = anchor.currentContext?.findRenderObject(); + if (spoken == null || box is! RenderBox || !box.attached) return; + final fill = Paint()..color = color; + for (final rect in spokenWordRects(segments, spoken, box)) { + canvas.drawRRect( + RRect.fromRectAndRadius(rect.inflate(1.5), const Radius.circular(4)), + fill, + ); + } } - return TextSpan(children: children); + + @override + bool shouldRepaint(SpokenWordPainter oldDelegate) => + oldDelegate.color != color || !identical(oldDelegate.segments, segments); +} + +/// The boxes, in [ancestor]'s coordinates, of the rendered characters whose +/// source offsets fall inside [range]. +List spokenWordRects( + List segments, + SpeechRange range, + RenderBox ancestor, +) { + final rects = []; + for (final segment in segments) { + if (segment.sourceStart >= range.end || segment.sourceEnd <= range.start) { + continue; + } + int? first; + var last = 0; + for (var index = 0; index < segment.offsets.length; index += 1) { + final offset = segment.offsets[index]; + if (offset >= range.start && offset < range.end) { + first ??= index; + last = index + 1; + } + } + if (first == null) continue; + final paragraph = _paragraphIn(segment.key.currentContext); + if (paragraph == null || !paragraph.attached || !paragraph.hasSize) { + continue; + } + final transform = paragraph.getTransformTo(ancestor); + for (final box in paragraph.getBoxesForSelection( + TextSelection( + baseOffset: segment.renderedStart + first, + extentOffset: segment.renderedStart + last, + ), + )) { + rects.add(MatrixUtils.transformRect(transform, box.toRect())); + } + } + return rects; } /// Scrolls [scroll] just enough to show the spoken [range] when it has left /// the viewport; a word already in view never moves the transcript. +/// +/// The word's own boxes are measured against the visible area directly, in +/// the viewport's coordinates, so a reversed or multi-sliver transcript +/// scrolls only by the distance the word is out of view. void revealSpokenWord( ScrollPosition scroll, List segments, @@ -69,23 +127,33 @@ void revealSpokenWord( ) .firstOrNull; final paragraph = _paragraphIn(segment?.key.currentContext); - if (segment == null || paragraph == null) return; - final boxes = paragraph.getBoxesForSelection( - TextSelection( - baseOffset: segment.renderedStart, - extentOffset: segment.renderedEnd, - ), - ); + if (paragraph == null || !paragraph.attached) return; final viewport = RenderAbstractViewport.maybeOf(paragraph); - if (boxes.isEmpty || viewport == null) return; - final rect = boxes.first.toRect().inflate(_revealMargin); - final leading = viewport.getOffsetToReveal(paragraph, 0, rect: rect).offset; - final trailing = viewport.getOffsetToReveal(paragraph, 1, rect: rect).offset; - final low = leading < trailing ? leading : trailing; - final high = leading < trailing ? trailing : leading; - final target = scroll.pixels - .clamp(low, high) - .clamp(scroll.minScrollExtent, scroll.maxScrollExtent); + if (viewport is! RenderBox || !(viewport as RenderBox).hasSize) return; + final box = viewport as RenderBox; + final rects = spokenWordRects(segments, range, box); + if (rects.isEmpty) return; + final word = rects.reduce((a, b) => a.expandToInclude(b)); + final vertical = scroll.axis == Axis.vertical; + final extent = vertical ? box.size.height : box.size.width; + final start = vertical ? word.top : word.left; + final end = vertical ? word.bottom : word.right; + // Positive when the word must move toward the start of the view. + final double delta; + if (start < _revealMargin) { + delta = start - _revealMargin; + } else if (end > extent - _revealMargin) { + delta = end - (extent - _revealMargin); + } else { + return; + } + final reversed = + scroll.axisDirection == AxisDirection.up || + scroll.axisDirection == AxisDirection.left; + final target = (scroll.pixels + (reversed ? -delta : delta)).clamp( + scroll.minScrollExtent, + scroll.maxScrollExtent, + ); if ((target - scroll.pixels).abs() < 1) return; if (reduceMotion) { scroll.jumpTo(target); @@ -170,12 +238,16 @@ class MarkdownSpokenSegment { this.sourceStart, this.sourceEnd, this.renderedStart, - this.renderedEnd, - ); + this.renderedEnd, [ + this.offsets = const [], + ]); final GlobalKey key; final int sourceStart; final int sourceEnd; final int renderedStart; final int renderedEnd; + + /// Each rendered character's offset in the source, in order. + final List offsets; } diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart index 065f8393..aeaee5d6 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart @@ -54,8 +54,6 @@ class _ChatMarkdownViewState extends State { ModalRoute? _route; var _sourceCursor = 0; - /// The source range of [ChatMarkdownView.text] being spoken, while building. - SpeechRange? _highlight; SpeechRange? _revealed; final _textKeys = []; var _textCount = 0; @@ -103,21 +101,33 @@ class _ChatMarkdownViewState extends State { final range = spokenSourceIn(speech, widget.messageId, widget.text); if (range == null) { _revealed = null; - return _rendered ?? _buildRendered(context, null); - } - if (range != _revealed) { + } else if (range != _revealed) { _revealed = range; WidgetsBinding.instance.addPostFrameCallback((_) => _reveal(range)); } - return _buildRendered(context, range); + // The text is built once and never restyled while speaking; the + // spoken word is painted behind it, so nothing shifts or rewraps. + final rendered = _rendered ??= _buildRendered(context); + return CustomPaint( + key: _paintKey, + painter: SpokenWordPainter( + anchor: _paintKey, + segments: _segments, + range: () => spokenSourceIn(speech, widget.messageId, widget.text), + color: Theme.of(context).colorScheme.primaryContainer, + repaint: Listenable.merge([speech, speech.word]), + ), + child: rendered, + ); }, ); } - Widget _buildRendered(BuildContext context, SpeechRange? range) { + final _paintKey = GlobalKey(); + + Widget _buildRendered(BuildContext context) { _sourceCursor = 0; _textCount = 0; - _highlight = range; _segments.clear(); final gap = chatParagraphGap(context); final blocks = chatMarkdownCache.blocksFor(widget.text); @@ -147,7 +157,7 @@ class _ChatMarkdownViewState extends State { ), ), ); - if (range == null) _rendered = rendered; + _rendered = rendered; return rendered; } @@ -394,19 +404,15 @@ class _ChatMarkdownViewState extends State { _segments.add( MarkdownSpokenSegment( source.key, - offsets.first, - offsets.last + 1, + offsets.reduce((a, b) => a < b ? a : b), + offsets.reduce((a, b) => a > b ? a : b) + 1, renderedStart, source.visibleOffset, + offsets, ), ); } - return spokenTextSpan( - text, - offsets, - _highlight, - Theme.of(context).colorScheme.primaryContainer, - ); + return TextSpan(text: text); } void _reveal(SpeechRange range) { diff --git a/app/test/features/chat/speech/platform_speech_rate_test.dart b/app/test/features/chat/speech/platform_speech_rate_test.dart new file mode 100644 index 00000000..b47dda5a --- /dev/null +++ b/app/test/features/chat/speech/platform_speech_rate_test.dart @@ -0,0 +1,14 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/flutter_tts_speech_engine.dart'; + +void main() { + test('1.0× reads a little under the platform normal on every scale', () { + expect(readAloudPace, lessThan(1)); + // Android and the Web Speech API: 1.0 is normal. + expect(platformSpeechRate(1, appleScale: false), closeTo(0.8, 1e-9)); + expect(platformSpeechRate(2, appleScale: false), closeTo(1.6, 1e-9)); + // AVSpeechSynthesizer: 0.5 is normal, on a 0..1 scale. + expect(platformSpeechRate(1, appleScale: true), closeTo(0.4, 1e-9)); + expect(platformSpeechRate(2, appleScale: true), closeTo(0.8, 1e-9)); + }); +} diff --git a/app/test/features/chat/speech/speech_document_test.dart b/app/test/features/chat/speech/speech_document_test.dart index 0bf900de..b5309e4b 100644 --- a/app/test/features/chat/speech/speech_document_test.dart +++ b/app/test/features/chat/speech/speech_document_test.dart @@ -53,4 +53,17 @@ void main() { )); expect(result.sourceRange(0, (start: two, end: two + 3)), isNull); }); + + test('a word read with its added sentence break stays on its own text', () { + const markdown = '- one\n- two'; + final document = markdownToSpeech(markdown); + final spoken = document.text; + // An engine may report the word with the period the reading added. + final start = spoken.indexOf('one'); + final range = document.sourceRange(0, (start: start, end: start + 5)); + expect(range, ( + start: markdown.indexOf('one'), + end: markdown.indexOf('one') + 3, + )); + }); } diff --git a/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart index 29f8205d..8604673e 100644 --- a/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart +++ b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart @@ -1,29 +1,45 @@ import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter/rendering.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_speech.dart'; import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart'; import '../../speech/fake_speech_engine.dart'; -/// The text of every span drawn with the spoken-word highlight. +/// The rendered text the spoken-word highlight is painted behind, per view. List _highlighted(WidgetTester tester, [Finder? within]) { - final fill = Theme.of( - tester.element(find.byType(ChatMarkdownView).first), - ).colorScheme.primaryContainer; - final spans = []; - final texts = within == null - ? find.byType(RichText) - : find.descendant(of: within, matching: find.byType(RichText)); - for (final rich in tester.widgetList(texts)) { - rich.text.visitChildren((span) { - if (span is TextSpan && - span.text != null && - span.style?.backgroundColor == fill) { - spans.add(span.text!); + final paints = within == null + ? find.byType(CustomPaint) + : find.descendant(of: within, matching: find.byType(CustomPaint)); + final words = []; + for (final paint in tester.widgetList(paints)) { + final painter = paint.painter; + if (painter is! SpokenWordPainter) continue; + final range = painter.range(); + if (range == null) continue; + final box = painter.anchor.currentContext!.findRenderObject()! as RenderBox; + if (spokenWordRects(painter.segments, range, box).isEmpty) continue; + for (final segment in painter.segments) { + final chars = StringBuffer(); + final paragraph = tester + .renderObjectList( + find.descendant( + of: find.byKey(segment.key), + matching: find.byType(RichText), + ), + ) + .first; + final plain = paragraph.text.toPlainText(); + for (var index = 0; index < segment.offsets.length; index += 1) { + final offset = segment.offsets[index]; + if (offset >= range.start && offset < range.end) { + chars.write(plain[segment.renderedStart + index]); + } } - return true; - }); + if (chars.isNotEmpty) words.add(chars.toString()); + } } - return spans; + return words; } void main() { @@ -73,6 +89,51 @@ void main() { expect(_highlighted(tester), isEmpty); }); + testWidgets('highlighting never restyles or moves the text', (tester) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + const text = + 'Alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu ' + 'nu xi omicron pi rho sigma tau upsilon phi chi psi omega.'; + await tester.pumpWidget( + const MaterialApp( + home: Scaffold( + body: SizedBox( + width: 300, + child: ChatMarkdownView(text: text, messageId: 'm'), + ), + ), + ), + ); + final paragraph = tester.renderObject( + find.byType(RichText).first, + ); + final span = paragraph.text; + final size = paragraph.size; + Rect wordRect(String word) { + final start = text.indexOf(word); + return paragraph + .getBoxesForSelection( + TextSelection(baseOffset: start, extentOffset: start + word.length), + ) + .first + .toRect(); + } + + final omega = wordRect('omega'); + await speech.read('m', [text]); + engine.onStart!(); + for (final word in ['beta', 'kappa', 'sigma', 'omega']) { + engine.speakWord(word); + await tester.pump(); + expect(_highlighted(tester), [word]); + // Same spans, same size, same word positions: nothing shifts. + expect(identical(paragraph.text, span), isTrue); + expect(paragraph.size, size); + expect(wordRect('omega'), omega); + } + }); + testWidgets('the transcript scrolls only when the word leaves the view', ( tester, ) async { @@ -117,4 +178,48 @@ void main() { await tester.pumpAndSettle(); expect(scroll.offset, followed); }); + + testWidgets('a reversed transcript never jumps for a word already in view', ( + tester, + ) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + final text = [ + for (var index = 0; index < 30; index += 1) 'Line $index words here.', + ].join('\n\n'); + final scroll = ScrollController(); + addTearDown(scroll.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SizedBox( + height: 400, + child: ListView( + controller: scroll, + reverse: true, + children: [ + const SizedBox(height: 50), + ChatMarkdownView(text: text, messageId: 'm'), + ], + ), + ), + ), + ), + ); + await speech.read('m', [text]); + engine.onStart!(); + // The newest lines sit at the bottom of a reversed transcript, in view. + for (final word in ['Line 28 ', 'Line 29 ', 'here.']) { + engine.speakWord(word, from: engine.spoken.last.indexOf('Line 28')); + await tester.pumpAndSettle(); + expect(scroll.offset, 0, reason: '"$word" is in view'); + } + // An early line above the view scrolls just into view, and only then. + engine.speakWord('Line 2 '); + await tester.pumpAndSettle(); + expect(scroll.offset, greaterThan(0)); + final viewport = tester.getRect(find.byType(ListView)); + final word = find.textContaining('Line 2 ', findRichText: true); + expect(viewport.overlaps(tester.getRect(word)), isTrue); + }); }