Steady read-aloud highlight and a calmer default pace

The spoken word is now painted behind unchanged text instead of
restyling the paragraph, so words never reshape, shift or rewrap as the
highlight moves. Revealing the word measures its own boxes against the
visible area, so a reversed transcript no longer jumps for a word already
in view. Sentence breaks the reading adds no longer map onto the next
item's first letter. 1.0x now reads at 0.8 of the platform's normal rate.
This commit is contained in:
2026-11-13 09:41:02 +00:00
parent b2736683e7
commit df0614e2f7
7 changed files with 310 additions and 78 deletions
@@ -56,7 +56,7 @@ class FlutterTtsSpeechEngine extends SpeechEngine {
@override
Future<void> setRate(double rate) async =>
_tts.setSpeechRate(_appleScale ? (rate * 0.5).clamp(0, 1) : rate);
_tts.setSpeechRate(platformSpeechRate(rate, appleScale: _appleScale));
@override
Future<void> setVoice(SpeechVoice? voice) async {
@@ -110,3 +110,15 @@ class FlutterTtsSpeechEngine extends SpeechEngine {
];
}
}
/// The app's 1.0× reads at a calm listening pace, a little under the
/// platform's own normal rate, which sounds rushed for long replies.
const readAloudPace = 0.8;
/// The platform rate for the app's [rate] multiplier (1.0× = [readAloudPace]
/// of the platform's normal): AVSpeechSynthesizer's normal is 0.5 on a 0..1
/// scale, Android's TextToSpeech and the Web Speech API use 1.0.
double platformSpeechRate(double rate, {required bool appleScale}) {
final paced = rate * readAloudPace;
return appleScale ? (paced * 0.5).clamp(0, 1) : paced;
}
@@ -14,6 +14,7 @@ class SpeechDocument {
required this.text,
required this.parts,
required this.sources,
this.added = const {},
});
final String text;
@@ -22,6 +23,11 @@ class SpeechDocument {
final List<String> parts;
final List<SpeechSource> sources;
/// Spoken indexes the reading added (a sentence break between blocks, the
/// "Code block." phrase): they come from no visible text, so a word that
/// reaches into them highlights only its own characters.
final Set<int> added;
/// The index of the part whose markdown is [markdown], or -1.
int partOf(String markdown) => parts.indexOf(markdown.trim());
@@ -33,7 +39,7 @@ class SpeechDocument {
final last = range.end.clamp(0, sources.length);
for (var index = range.start.clamp(0, last); index < last; index += 1) {
final source = sources[index];
if (source.part != part) continue;
if (source.part != part || added.contains(index)) continue;
if (start == null || source.offset < start) start = source.offset;
if (end == null || source.offset + 1 > end) end = source.offset + 1;
}
@@ -61,12 +67,14 @@ SpeechDocument speechDocumentFor(List<String> markdownParts) {
text: builder.spoken.toString(),
parts: parts,
sources: builder.sources,
added: builder.added,
);
}
class _SpeechBuilder {
final spoken = StringBuffer();
final sources = <SpeechSource>[];
final added = <int>{};
String _markdown = '';
var _part = 0;
var _cursor = 0;
@@ -103,11 +111,13 @@ class _SpeechBuilder {
_write(' ', at);
}
/// Writes text the reading adds, which no visible character stands for.
void _write(String value, int at) {
spoken.write(value);
for (var index = 0; index < value.length; index += 1) {
added.add(spoken.length + index);
sources.add((part: _part, offset: at));
}
spoken.write(value);
}
void _phrase(String value, int at) {
@@ -28,34 +28,92 @@ SpeechRange? spokenSourceIn(
return (start: range.start + lead, end: range.end + lead);
}
/// [text] as spans whose characters with source [offsets] inside [range] are
/// filled with [fill]; plain text when nothing in it is spoken.
TextSpan spokenTextSpan(
String text,
List<int> offsets,
SpeechRange? range,
Color fill,
) {
if (range == null || offsets.isEmpty) return TextSpan(text: text);
bool spoken(int index) =>
offsets[index] >= range.start && offsets[index] < range.end;
final children = <InlineSpan>[];
var runStart = 0;
for (var index = 1; index <= text.length; index += 1) {
if (index < text.length && spoken(index) == spoken(runStart)) continue;
children.add(
TextSpan(
text: text.substring(runStart, index),
style: spoken(runStart) ? TextStyle(backgroundColor: fill) : null,
),
);
runStart = index;
/// Paints the spoken word's highlight behind the rendered text.
///
/// The text itself is never restyled while speaking: splitting a paragraph
/// into differently styled runs changes how it is shaped, so words would
/// shift and lines rewrap as the highlight moves. Instead this painter, under
/// the text, looks up the spoken characters' boxes in the laid-out paragraphs
/// ([segments]) and fills them; each new word only repaints.
class SpokenWordPainter extends CustomPainter {
SpokenWordPainter({
required this.anchor,
required this.segments,
required this.range,
required this.color,
super.repaint,
});
/// The key of the [CustomPaint] this painter draws in.
final GlobalKey anchor;
final List<MarkdownSpokenSegment> segments;
final SpeechRange? Function() range;
final Color color;
@override
void paint(Canvas canvas, Size size) {
final spoken = range();
final box = anchor.currentContext?.findRenderObject();
if (spoken == null || box is! RenderBox || !box.attached) return;
final fill = Paint()..color = color;
for (final rect in spokenWordRects(segments, spoken, box)) {
canvas.drawRRect(
RRect.fromRectAndRadius(rect.inflate(1.5), const Radius.circular(4)),
fill,
);
}
}
return TextSpan(children: children);
@override
bool shouldRepaint(SpokenWordPainter oldDelegate) =>
oldDelegate.color != color || !identical(oldDelegate.segments, segments);
}
/// The boxes, in [ancestor]'s coordinates, of the rendered characters whose
/// source offsets fall inside [range].
List<Rect> spokenWordRects(
List<MarkdownSpokenSegment> segments,
SpeechRange range,
RenderBox ancestor,
) {
final rects = <Rect>[];
for (final segment in segments) {
if (segment.sourceStart >= range.end || segment.sourceEnd <= range.start) {
continue;
}
int? first;
var last = 0;
for (var index = 0; index < segment.offsets.length; index += 1) {
final offset = segment.offsets[index];
if (offset >= range.start && offset < range.end) {
first ??= index;
last = index + 1;
}
}
if (first == null) continue;
final paragraph = _paragraphIn(segment.key.currentContext);
if (paragraph == null || !paragraph.attached || !paragraph.hasSize) {
continue;
}
final transform = paragraph.getTransformTo(ancestor);
for (final box in paragraph.getBoxesForSelection(
TextSelection(
baseOffset: segment.renderedStart + first,
extentOffset: segment.renderedStart + last,
),
)) {
rects.add(MatrixUtils.transformRect(transform, box.toRect()));
}
}
return rects;
}
/// Scrolls [scroll] just enough to show the spoken [range] when it has left
/// the viewport; a word already in view never moves the transcript.
///
/// The word's own boxes are measured against the visible area directly, in
/// the viewport's coordinates, so a reversed or multi-sliver transcript
/// scrolls only by the distance the word is out of view.
void revealSpokenWord(
ScrollPosition scroll,
List<MarkdownSpokenSegment> segments,
@@ -69,23 +127,33 @@ void revealSpokenWord(
)
.firstOrNull;
final paragraph = _paragraphIn(segment?.key.currentContext);
if (segment == null || paragraph == null) return;
final boxes = paragraph.getBoxesForSelection(
TextSelection(
baseOffset: segment.renderedStart,
extentOffset: segment.renderedEnd,
),
);
if (paragraph == null || !paragraph.attached) return;
final viewport = RenderAbstractViewport.maybeOf(paragraph);
if (boxes.isEmpty || viewport == null) return;
final rect = boxes.first.toRect().inflate(_revealMargin);
final leading = viewport.getOffsetToReveal(paragraph, 0, rect: rect).offset;
final trailing = viewport.getOffsetToReveal(paragraph, 1, rect: rect).offset;
final low = leading < trailing ? leading : trailing;
final high = leading < trailing ? trailing : leading;
final target = scroll.pixels
.clamp(low, high)
.clamp(scroll.minScrollExtent, scroll.maxScrollExtent);
if (viewport is! RenderBox || !(viewport as RenderBox).hasSize) return;
final box = viewport as RenderBox;
final rects = spokenWordRects(segments, range, box);
if (rects.isEmpty) return;
final word = rects.reduce((a, b) => a.expandToInclude(b));
final vertical = scroll.axis == Axis.vertical;
final extent = vertical ? box.size.height : box.size.width;
final start = vertical ? word.top : word.left;
final end = vertical ? word.bottom : word.right;
// Positive when the word must move toward the start of the view.
final double delta;
if (start < _revealMargin) {
delta = start - _revealMargin;
} else if (end > extent - _revealMargin) {
delta = end - (extent - _revealMargin);
} else {
return;
}
final reversed =
scroll.axisDirection == AxisDirection.up ||
scroll.axisDirection == AxisDirection.left;
final target = (scroll.pixels + (reversed ? -delta : delta)).clamp(
scroll.minScrollExtent,
scroll.maxScrollExtent,
);
if ((target - scroll.pixels).abs() < 1) return;
if (reduceMotion) {
scroll.jumpTo(target);
@@ -170,12 +238,16 @@ class MarkdownSpokenSegment {
this.sourceStart,
this.sourceEnd,
this.renderedStart,
this.renderedEnd,
);
this.renderedEnd, [
this.offsets = const [],
]);
final GlobalKey key;
final int sourceStart;
final int sourceEnd;
final int renderedStart;
final int renderedEnd;
/// Each rendered character's offset in the source, in order.
final List<int> offsets;
}
@@ -54,8 +54,6 @@ class _ChatMarkdownViewState extends State<ChatMarkdownView> {
ModalRoute<dynamic>? _route;
var _sourceCursor = 0;
/// The source range of [ChatMarkdownView.text] being spoken, while building.
SpeechRange? _highlight;
SpeechRange? _revealed;
final _textKeys = <GlobalKey>[];
var _textCount = 0;
@@ -103,21 +101,33 @@ class _ChatMarkdownViewState extends State<ChatMarkdownView> {
final range = spokenSourceIn(speech, widget.messageId, widget.text);
if (range == null) {
_revealed = null;
return _rendered ?? _buildRendered(context, null);
}
if (range != _revealed) {
} else if (range != _revealed) {
_revealed = range;
WidgetsBinding.instance.addPostFrameCallback((_) => _reveal(range));
}
return _buildRendered(context, range);
// The text is built once and never restyled while speaking; the
// spoken word is painted behind it, so nothing shifts or rewraps.
final rendered = _rendered ??= _buildRendered(context);
return CustomPaint(
key: _paintKey,
painter: SpokenWordPainter(
anchor: _paintKey,
segments: _segments,
range: () => spokenSourceIn(speech, widget.messageId, widget.text),
color: Theme.of(context).colorScheme.primaryContainer,
repaint: Listenable.merge([speech, speech.word]),
),
child: rendered,
);
},
);
}
Widget _buildRendered(BuildContext context, SpeechRange? range) {
final _paintKey = GlobalKey();
Widget _buildRendered(BuildContext context) {
_sourceCursor = 0;
_textCount = 0;
_highlight = range;
_segments.clear();
final gap = chatParagraphGap(context);
final blocks = chatMarkdownCache.blocksFor(widget.text);
@@ -147,7 +157,7 @@ class _ChatMarkdownViewState extends State<ChatMarkdownView> {
),
),
);
if (range == null) _rendered = rendered;
_rendered = rendered;
return rendered;
}
@@ -394,19 +404,15 @@ class _ChatMarkdownViewState extends State<ChatMarkdownView> {
_segments.add(
MarkdownSpokenSegment(
source.key,
offsets.first,
offsets.last + 1,
offsets.reduce((a, b) => a < b ? a : b),
offsets.reduce((a, b) => a > b ? a : b) + 1,
renderedStart,
source.visibleOffset,
offsets,
),
);
}
return spokenTextSpan(
text,
offsets,
_highlight,
Theme.of(context).colorScheme.primaryContainer,
);
return TextSpan(text: text);
}
void _reveal(SpeechRange range) {
@@ -0,0 +1,14 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/chat/speech/flutter_tts_speech_engine.dart';
void main() {
test('1.0× reads a little under the platform normal on every scale', () {
expect(readAloudPace, lessThan(1));
// Android and the Web Speech API: 1.0 is normal.
expect(platformSpeechRate(1, appleScale: false), closeTo(0.8, 1e-9));
expect(platformSpeechRate(2, appleScale: false), closeTo(1.6, 1e-9));
// AVSpeechSynthesizer: 0.5 is normal, on a 0..1 scale.
expect(platformSpeechRate(1, appleScale: true), closeTo(0.4, 1e-9));
expect(platformSpeechRate(2, appleScale: true), closeTo(0.8, 1e-9));
});
}
@@ -53,4 +53,17 @@ void main() {
));
expect(result.sourceRange(0, (start: two, end: two + 3)), isNull);
});
test('a word read with its added sentence break stays on its own text', () {
const markdown = '- one\n- two';
final document = markdownToSpeech(markdown);
final spoken = document.text;
// An engine may report the word with the period the reading added.
final start = spoken.indexOf('one');
final range = document.sourceRange(0, (start: start, end: start + 5));
expect(range, (
start: markdown.indexOf('one'),
end: markdown.indexOf('one') + 3,
));
});
}
@@ -1,29 +1,45 @@
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter/rendering.dart';
import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_speech.dart';
import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart';
import '../../speech/fake_speech_engine.dart';
/// The text of every span drawn with the spoken-word highlight.
/// The rendered text the spoken-word highlight is painted behind, per view.
List<String> _highlighted(WidgetTester tester, [Finder? within]) {
final fill = Theme.of(
tester.element(find.byType(ChatMarkdownView).first),
).colorScheme.primaryContainer;
final spans = <String>[];
final texts = within == null
? find.byType(RichText)
: find.descendant(of: within, matching: find.byType(RichText));
for (final rich in tester.widgetList<RichText>(texts)) {
rich.text.visitChildren((span) {
if (span is TextSpan &&
span.text != null &&
span.style?.backgroundColor == fill) {
spans.add(span.text!);
final paints = within == null
? find.byType(CustomPaint)
: find.descendant(of: within, matching: find.byType(CustomPaint));
final words = <String>[];
for (final paint in tester.widgetList<CustomPaint>(paints)) {
final painter = paint.painter;
if (painter is! SpokenWordPainter) continue;
final range = painter.range();
if (range == null) continue;
final box = painter.anchor.currentContext!.findRenderObject()! as RenderBox;
if (spokenWordRects(painter.segments, range, box).isEmpty) continue;
for (final segment in painter.segments) {
final chars = StringBuffer();
final paragraph = tester
.renderObjectList<RenderParagraph>(
find.descendant(
of: find.byKey(segment.key),
matching: find.byType(RichText),
),
)
.first;
final plain = paragraph.text.toPlainText();
for (var index = 0; index < segment.offsets.length; index += 1) {
final offset = segment.offsets[index];
if (offset >= range.start && offset < range.end) {
chars.write(plain[segment.renderedStart + index]);
}
}
return true;
});
if (chars.isNotEmpty) words.add(chars.toString());
}
}
return spans;
return words;
}
void main() {
@@ -73,6 +89,51 @@ void main() {
expect(_highlighted(tester), isEmpty);
});
testWidgets('highlighting never restyles or moves the text', (tester) async {
final engine = FakeSpeechEngine();
final speech = installFakeSpeech(engine);
const text =
'Alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu '
'nu xi omicron pi rho sigma tau upsilon phi chi psi omega.';
await tester.pumpWidget(
const MaterialApp(
home: Scaffold(
body: SizedBox(
width: 300,
child: ChatMarkdownView(text: text, messageId: 'm'),
),
),
),
);
final paragraph = tester.renderObject<RenderParagraph>(
find.byType(RichText).first,
);
final span = paragraph.text;
final size = paragraph.size;
Rect wordRect(String word) {
final start = text.indexOf(word);
return paragraph
.getBoxesForSelection(
TextSelection(baseOffset: start, extentOffset: start + word.length),
)
.first
.toRect();
}
final omega = wordRect('omega');
await speech.read('m', [text]);
engine.onStart!();
for (final word in ['beta', 'kappa', 'sigma', 'omega']) {
engine.speakWord(word);
await tester.pump();
expect(_highlighted(tester), [word]);
// Same spans, same size, same word positions: nothing shifts.
expect(identical(paragraph.text, span), isTrue);
expect(paragraph.size, size);
expect(wordRect('omega'), omega);
}
});
testWidgets('the transcript scrolls only when the word leaves the view', (
tester,
) async {
@@ -117,4 +178,48 @@ void main() {
await tester.pumpAndSettle();
expect(scroll.offset, followed);
});
testWidgets('a reversed transcript never jumps for a word already in view', (
tester,
) async {
final engine = FakeSpeechEngine();
final speech = installFakeSpeech(engine);
final text = [
for (var index = 0; index < 30; index += 1) 'Line $index words here.',
].join('\n\n');
final scroll = ScrollController();
addTearDown(scroll.dispose);
await tester.pumpWidget(
MaterialApp(
home: Scaffold(
body: SizedBox(
height: 400,
child: ListView(
controller: scroll,
reverse: true,
children: [
const SizedBox(height: 50),
ChatMarkdownView(text: text, messageId: 'm'),
],
),
),
),
),
);
await speech.read('m', [text]);
engine.onStart!();
// The newest lines sit at the bottom of a reversed transcript, in view.
for (final word in ['Line 28 ', 'Line 29 ', 'here.']) {
engine.speakWord(word, from: engine.spoken.last.indexOf('Line 28'));
await tester.pumpAndSettle();
expect(scroll.offset, 0, reason: '"$word" is in view');
}
// An early line above the view scrolls just into view, and only then.
engine.speakWord('Line 2 ');
await tester.pumpAndSettle();
expect(scroll.offset, greaterThan(0));
final viewport = tester.getRect(find.byType(ListView));
final word = find.textContaining('Line 2 ', findRichText: true);
expect(viewport.overlaps(tester.getRect(word)), isTrue);
});
}