diff --git a/app/lib/features/chat/speech/flutter_tts_speech_engine.dart b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart index 2ab2700e..680cba83 100644 --- a/app/lib/features/chat/speech/flutter_tts_speech_engine.dart +++ b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart @@ -56,7 +56,7 @@ class FlutterTtsSpeechEngine extends SpeechEngine { @override Future setRate(double rate) async => - _tts.setSpeechRate(_appleScale ? (rate * 0.5).clamp(0, 1) : rate); + _tts.setSpeechRate(platformSpeechRate(rate, appleScale: _appleScale)); @override Future setVoice(SpeechVoice? voice) async { @@ -110,3 +110,15 @@ class FlutterTtsSpeechEngine extends SpeechEngine { ]; } } + +/// The app's 1.0× reads at a calm listening pace, a little under the +/// platform's own normal rate, which sounds rushed for long replies. +const readAloudPace = 0.8; + +/// The platform rate for the app's [rate] multiplier (1.0× = [readAloudPace] +/// of the platform's normal): AVSpeechSynthesizer's normal is 0.5 on a 0..1 +/// scale, Android's TextToSpeech and the Web Speech API use 1.0. +double platformSpeechRate(double rate, {required bool appleScale}) { + final paced = rate * readAloudPace; + return appleScale ? (paced * 0.5).clamp(0, 1) : paced; +} diff --git a/app/lib/features/chat/speech/speech_document.dart b/app/lib/features/chat/speech/speech_document.dart index 8623367e..4024da74 100644 --- a/app/lib/features/chat/speech/speech_document.dart +++ b/app/lib/features/chat/speech/speech_document.dart @@ -14,6 +14,7 @@ class SpeechDocument { required this.text, required this.parts, required this.sources, + this.added = const {}, }); final String text; @@ -22,6 +23,11 @@ class SpeechDocument { final List parts; final List sources; + /// Spoken indexes the reading added (a sentence break between blocks, the + /// "Code block." phrase): they come from no visible text, so a word that + /// reaches into them highlights only its own characters. + final Set added; + /// The index of the part whose markdown is [markdown], or -1. int partOf(String markdown) => parts.indexOf(markdown.trim()); @@ -33,7 +39,7 @@ class SpeechDocument { final last = range.end.clamp(0, sources.length); for (var index = range.start.clamp(0, last); index < last; index += 1) { final source = sources[index]; - if (source.part != part) continue; + if (source.part != part || added.contains(index)) continue; if (start == null || source.offset < start) start = source.offset; if (end == null || source.offset + 1 > end) end = source.offset + 1; } @@ -61,12 +67,14 @@ SpeechDocument speechDocumentFor(List markdownParts) { text: builder.spoken.toString(), parts: parts, sources: builder.sources, + added: builder.added, ); } class _SpeechBuilder { final spoken = StringBuffer(); final sources = []; + final added = {}; String _markdown = ''; var _part = 0; var _cursor = 0; @@ -103,11 +111,13 @@ class _SpeechBuilder { _write(' ', at); } + /// Writes text the reading adds, which no visible character stands for. void _write(String value, int at) { - spoken.write(value); for (var index = 0; index < value.length; index += 1) { + added.add(spoken.length + index); sources.add((part: _part, offset: at)); } + spoken.write(value); } void _phrase(String value, int at) { diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart index 72c1dbe7..b82404eb 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart @@ -28,34 +28,92 @@ SpeechRange? spokenSourceIn( return (start: range.start + lead, end: range.end + lead); } -/// [text] as spans whose characters with source [offsets] inside [range] are -/// filled with [fill]; plain text when nothing in it is spoken. -TextSpan spokenTextSpan( - String text, - List offsets, - SpeechRange? range, - Color fill, -) { - if (range == null || offsets.isEmpty) return TextSpan(text: text); - bool spoken(int index) => - offsets[index] >= range.start && offsets[index] < range.end; - final children = []; - var runStart = 0; - for (var index = 1; index <= text.length; index += 1) { - if (index < text.length && spoken(index) == spoken(runStart)) continue; - children.add( - TextSpan( - text: text.substring(runStart, index), - style: spoken(runStart) ? TextStyle(backgroundColor: fill) : null, - ), - ); - runStart = index; +/// Paints the spoken word's highlight behind the rendered text. +/// +/// The text itself is never restyled while speaking: splitting a paragraph +/// into differently styled runs changes how it is shaped, so words would +/// shift and lines rewrap as the highlight moves. Instead this painter, under +/// the text, looks up the spoken characters' boxes in the laid-out paragraphs +/// ([segments]) and fills them; each new word only repaints. +class SpokenWordPainter extends CustomPainter { + SpokenWordPainter({ + required this.anchor, + required this.segments, + required this.range, + required this.color, + super.repaint, + }); + + /// The key of the [CustomPaint] this painter draws in. + final GlobalKey anchor; + final List segments; + final SpeechRange? Function() range; + final Color color; + + @override + void paint(Canvas canvas, Size size) { + final spoken = range(); + final box = anchor.currentContext?.findRenderObject(); + if (spoken == null || box is! RenderBox || !box.attached) return; + final fill = Paint()..color = color; + for (final rect in spokenWordRects(segments, spoken, box)) { + canvas.drawRRect( + RRect.fromRectAndRadius(rect.inflate(1.5), const Radius.circular(4)), + fill, + ); + } } - return TextSpan(children: children); + + @override + bool shouldRepaint(SpokenWordPainter oldDelegate) => + oldDelegate.color != color || !identical(oldDelegate.segments, segments); +} + +/// The boxes, in [ancestor]'s coordinates, of the rendered characters whose +/// source offsets fall inside [range]. +List spokenWordRects( + List segments, + SpeechRange range, + RenderBox ancestor, +) { + final rects = []; + for (final segment in segments) { + if (segment.sourceStart >= range.end || segment.sourceEnd <= range.start) { + continue; + } + int? first; + var last = 0; + for (var index = 0; index < segment.offsets.length; index += 1) { + final offset = segment.offsets[index]; + if (offset >= range.start && offset < range.end) { + first ??= index; + last = index + 1; + } + } + if (first == null) continue; + final paragraph = _paragraphIn(segment.key.currentContext); + if (paragraph == null || !paragraph.attached || !paragraph.hasSize) { + continue; + } + final transform = paragraph.getTransformTo(ancestor); + for (final box in paragraph.getBoxesForSelection( + TextSelection( + baseOffset: segment.renderedStart + first, + extentOffset: segment.renderedStart + last, + ), + )) { + rects.add(MatrixUtils.transformRect(transform, box.toRect())); + } + } + return rects; } /// Scrolls [scroll] just enough to show the spoken [range] when it has left /// the viewport; a word already in view never moves the transcript. +/// +/// The word's own boxes are measured against the visible area directly, in +/// the viewport's coordinates, so a reversed or multi-sliver transcript +/// scrolls only by the distance the word is out of view. void revealSpokenWord( ScrollPosition scroll, List segments, @@ -69,23 +127,33 @@ void revealSpokenWord( ) .firstOrNull; final paragraph = _paragraphIn(segment?.key.currentContext); - if (segment == null || paragraph == null) return; - final boxes = paragraph.getBoxesForSelection( - TextSelection( - baseOffset: segment.renderedStart, - extentOffset: segment.renderedEnd, - ), - ); + if (paragraph == null || !paragraph.attached) return; final viewport = RenderAbstractViewport.maybeOf(paragraph); - if (boxes.isEmpty || viewport == null) return; - final rect = boxes.first.toRect().inflate(_revealMargin); - final leading = viewport.getOffsetToReveal(paragraph, 0, rect: rect).offset; - final trailing = viewport.getOffsetToReveal(paragraph, 1, rect: rect).offset; - final low = leading < trailing ? leading : trailing; - final high = leading < trailing ? trailing : leading; - final target = scroll.pixels - .clamp(low, high) - .clamp(scroll.minScrollExtent, scroll.maxScrollExtent); + if (viewport is! RenderBox || !(viewport as RenderBox).hasSize) return; + final box = viewport as RenderBox; + final rects = spokenWordRects(segments, range, box); + if (rects.isEmpty) return; + final word = rects.reduce((a, b) => a.expandToInclude(b)); + final vertical = scroll.axis == Axis.vertical; + final extent = vertical ? box.size.height : box.size.width; + final start = vertical ? word.top : word.left; + final end = vertical ? word.bottom : word.right; + // Positive when the word must move toward the start of the view. + final double delta; + if (start < _revealMargin) { + delta = start - _revealMargin; + } else if (end > extent - _revealMargin) { + delta = end - (extent - _revealMargin); + } else { + return; + } + final reversed = + scroll.axisDirection == AxisDirection.up || + scroll.axisDirection == AxisDirection.left; + final target = (scroll.pixels + (reversed ? -delta : delta)).clamp( + scroll.minScrollExtent, + scroll.maxScrollExtent, + ); if ((target - scroll.pixels).abs() < 1) return; if (reduceMotion) { scroll.jumpTo(target); @@ -170,12 +238,16 @@ class MarkdownSpokenSegment { this.sourceStart, this.sourceEnd, this.renderedStart, - this.renderedEnd, - ); + this.renderedEnd, [ + this.offsets = const [], + ]); final GlobalKey key; final int sourceStart; final int sourceEnd; final int renderedStart; final int renderedEnd; + + /// Each rendered character's offset in the source, in order. + final List offsets; } diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart index 065f8393..aeaee5d6 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart @@ -54,8 +54,6 @@ class _ChatMarkdownViewState extends State { ModalRoute? _route; var _sourceCursor = 0; - /// The source range of [ChatMarkdownView.text] being spoken, while building. - SpeechRange? _highlight; SpeechRange? _revealed; final _textKeys = []; var _textCount = 0; @@ -103,21 +101,33 @@ class _ChatMarkdownViewState extends State { final range = spokenSourceIn(speech, widget.messageId, widget.text); if (range == null) { _revealed = null; - return _rendered ?? _buildRendered(context, null); - } - if (range != _revealed) { + } else if (range != _revealed) { _revealed = range; WidgetsBinding.instance.addPostFrameCallback((_) => _reveal(range)); } - return _buildRendered(context, range); + // The text is built once and never restyled while speaking; the + // spoken word is painted behind it, so nothing shifts or rewraps. + final rendered = _rendered ??= _buildRendered(context); + return CustomPaint( + key: _paintKey, + painter: SpokenWordPainter( + anchor: _paintKey, + segments: _segments, + range: () => spokenSourceIn(speech, widget.messageId, widget.text), + color: Theme.of(context).colorScheme.primaryContainer, + repaint: Listenable.merge([speech, speech.word]), + ), + child: rendered, + ); }, ); } - Widget _buildRendered(BuildContext context, SpeechRange? range) { + final _paintKey = GlobalKey(); + + Widget _buildRendered(BuildContext context) { _sourceCursor = 0; _textCount = 0; - _highlight = range; _segments.clear(); final gap = chatParagraphGap(context); final blocks = chatMarkdownCache.blocksFor(widget.text); @@ -147,7 +157,7 @@ class _ChatMarkdownViewState extends State { ), ), ); - if (range == null) _rendered = rendered; + _rendered = rendered; return rendered; } @@ -394,19 +404,15 @@ class _ChatMarkdownViewState extends State { _segments.add( MarkdownSpokenSegment( source.key, - offsets.first, - offsets.last + 1, + offsets.reduce((a, b) => a < b ? a : b), + offsets.reduce((a, b) => a > b ? a : b) + 1, renderedStart, source.visibleOffset, + offsets, ), ); } - return spokenTextSpan( - text, - offsets, - _highlight, - Theme.of(context).colorScheme.primaryContainer, - ); + return TextSpan(text: text); } void _reveal(SpeechRange range) { diff --git a/app/test/features/chat/speech/platform_speech_rate_test.dart b/app/test/features/chat/speech/platform_speech_rate_test.dart new file mode 100644 index 00000000..b47dda5a --- /dev/null +++ b/app/test/features/chat/speech/platform_speech_rate_test.dart @@ -0,0 +1,14 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/flutter_tts_speech_engine.dart'; + +void main() { + test('1.0× reads a little under the platform normal on every scale', () { + expect(readAloudPace, lessThan(1)); + // Android and the Web Speech API: 1.0 is normal. + expect(platformSpeechRate(1, appleScale: false), closeTo(0.8, 1e-9)); + expect(platformSpeechRate(2, appleScale: false), closeTo(1.6, 1e-9)); + // AVSpeechSynthesizer: 0.5 is normal, on a 0..1 scale. + expect(platformSpeechRate(1, appleScale: true), closeTo(0.4, 1e-9)); + expect(platformSpeechRate(2, appleScale: true), closeTo(0.8, 1e-9)); + }); +} diff --git a/app/test/features/chat/speech/speech_document_test.dart b/app/test/features/chat/speech/speech_document_test.dart index 0bf900de..b5309e4b 100644 --- a/app/test/features/chat/speech/speech_document_test.dart +++ b/app/test/features/chat/speech/speech_document_test.dart @@ -53,4 +53,17 @@ void main() { )); expect(result.sourceRange(0, (start: two, end: two + 3)), isNull); }); + + test('a word read with its added sentence break stays on its own text', () { + const markdown = '- one\n- two'; + final document = markdownToSpeech(markdown); + final spoken = document.text; + // An engine may report the word with the period the reading added. + final start = spoken.indexOf('one'); + final range = document.sourceRange(0, (start: start, end: start + 5)); + expect(range, ( + start: markdown.indexOf('one'), + end: markdown.indexOf('one') + 3, + )); + }); } diff --git a/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart index 29f8205d..8604673e 100644 --- a/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart +++ b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart @@ -1,29 +1,45 @@ import 'package:flutter/material.dart'; import 'package:flutter_test/flutter_test.dart'; +import 'package:flutter/rendering.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_speech.dart'; import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart'; import '../../speech/fake_speech_engine.dart'; -/// The text of every span drawn with the spoken-word highlight. +/// The rendered text the spoken-word highlight is painted behind, per view. List _highlighted(WidgetTester tester, [Finder? within]) { - final fill = Theme.of( - tester.element(find.byType(ChatMarkdownView).first), - ).colorScheme.primaryContainer; - final spans = []; - final texts = within == null - ? find.byType(RichText) - : find.descendant(of: within, matching: find.byType(RichText)); - for (final rich in tester.widgetList(texts)) { - rich.text.visitChildren((span) { - if (span is TextSpan && - span.text != null && - span.style?.backgroundColor == fill) { - spans.add(span.text!); + final paints = within == null + ? find.byType(CustomPaint) + : find.descendant(of: within, matching: find.byType(CustomPaint)); + final words = []; + for (final paint in tester.widgetList(paints)) { + final painter = paint.painter; + if (painter is! SpokenWordPainter) continue; + final range = painter.range(); + if (range == null) continue; + final box = painter.anchor.currentContext!.findRenderObject()! as RenderBox; + if (spokenWordRects(painter.segments, range, box).isEmpty) continue; + for (final segment in painter.segments) { + final chars = StringBuffer(); + final paragraph = tester + .renderObjectList( + find.descendant( + of: find.byKey(segment.key), + matching: find.byType(RichText), + ), + ) + .first; + final plain = paragraph.text.toPlainText(); + for (var index = 0; index < segment.offsets.length; index += 1) { + final offset = segment.offsets[index]; + if (offset >= range.start && offset < range.end) { + chars.write(plain[segment.renderedStart + index]); + } } - return true; - }); + if (chars.isNotEmpty) words.add(chars.toString()); + } } - return spans; + return words; } void main() { @@ -73,6 +89,51 @@ void main() { expect(_highlighted(tester), isEmpty); }); + testWidgets('highlighting never restyles or moves the text', (tester) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + const text = + 'Alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu ' + 'nu xi omicron pi rho sigma tau upsilon phi chi psi omega.'; + await tester.pumpWidget( + const MaterialApp( + home: Scaffold( + body: SizedBox( + width: 300, + child: ChatMarkdownView(text: text, messageId: 'm'), + ), + ), + ), + ); + final paragraph = tester.renderObject( + find.byType(RichText).first, + ); + final span = paragraph.text; + final size = paragraph.size; + Rect wordRect(String word) { + final start = text.indexOf(word); + return paragraph + .getBoxesForSelection( + TextSelection(baseOffset: start, extentOffset: start + word.length), + ) + .first + .toRect(); + } + + final omega = wordRect('omega'); + await speech.read('m', [text]); + engine.onStart!(); + for (final word in ['beta', 'kappa', 'sigma', 'omega']) { + engine.speakWord(word); + await tester.pump(); + expect(_highlighted(tester), [word]); + // Same spans, same size, same word positions: nothing shifts. + expect(identical(paragraph.text, span), isTrue); + expect(paragraph.size, size); + expect(wordRect('omega'), omega); + } + }); + testWidgets('the transcript scrolls only when the word leaves the view', ( tester, ) async { @@ -117,4 +178,48 @@ void main() { await tester.pumpAndSettle(); expect(scroll.offset, followed); }); + + testWidgets('a reversed transcript never jumps for a word already in view', ( + tester, + ) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + final text = [ + for (var index = 0; index < 30; index += 1) 'Line $index words here.', + ].join('\n\n'); + final scroll = ScrollController(); + addTearDown(scroll.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SizedBox( + height: 400, + child: ListView( + controller: scroll, + reverse: true, + children: [ + const SizedBox(height: 50), + ChatMarkdownView(text: text, messageId: 'm'), + ], + ), + ), + ), + ), + ); + await speech.read('m', [text]); + engine.onStart!(); + // The newest lines sit at the bottom of a reversed transcript, in view. + for (final word in ['Line 28 ', 'Line 29 ', 'here.']) { + engine.speakWord(word, from: engine.spoken.last.indexOf('Line 28')); + await tester.pumpAndSettle(); + expect(scroll.offset, 0, reason: '"$word" is in view'); + } + // An early line above the view scrolls just into view, and only then. + engine.speakWord('Line 2 '); + await tester.pumpAndSettle(); + expect(scroll.offset, greaterThan(0)); + final viewport = tester.getRect(find.byType(ListView)); + final word = find.textContaining('Line 2 ', findRichText: true); + expect(viewport.overlaps(tester.getRect(word)), isTrue); + }); }