Read chat replies aloud with word highlighting

An assistant reply gets Read aloud beside Copy; while it reads it becomes
Pause/Resume and Stop, one reply at a time app-wide. The spoken word is
highlighted in the rendered reply (across multi-part replies) and the
transcript scrolls only when that word leaves the viewport. Reading stops
when its chat closes or is swapped, another reply starts, or the app goes
to the background. Markdown is spoken as sentences; code blocks are
announced as "Code block.".

Speech goes through a SpeechEngine seam with a flutter_tts implementation
(Android, macOS, web; hidden on Linux) and a fake for tests. Settings has
a Read aloud section: a voice picker grouped by language (device language
first, a sample per voice), a rate slider and reset, saved per device.

flutter_tts has no Swift package, so macOS/iOS gain the CocoaPods
integration (Podfiles with the Runner deployment targets); Android
declares the TTS_SERVICE query. The capture fixture gains ?speech=fake.
This commit is contained in:
2026-11-13 01:29:04 +00:00
parent 26e68140fd
commit 6367cce68a
40 changed files with 2643 additions and 29 deletions
@@ -0,0 +1,120 @@
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart';
import '../../speech/fake_speech_engine.dart';
/// The text of every span drawn with the spoken-word highlight.
List<String> _highlighted(WidgetTester tester, [Finder? within]) {
final fill = Theme.of(
tester.element(find.byType(ChatMarkdownView).first),
).colorScheme.primaryContainer;
final spans = <String>[];
final texts = within == null
? find.byType(RichText)
: find.descendant(of: within, matching: find.byType(RichText));
for (final rich in tester.widgetList<RichText>(texts)) {
rich.text.visitChildren((span) {
if (span is TextSpan &&
span.text != null &&
span.style?.backgroundColor == fill) {
spans.add(span.text!);
}
return true;
});
}
return spans;
}
void main() {
testWidgets('the highlight follows progress and clears on stop', (
tester,
) async {
final engine = FakeSpeechEngine();
final speech = installFakeSpeech(engine);
const first = 'Alpha **beta** gamma';
const second = '- delta beta';
await tester.pumpWidget(
const MaterialApp(
home: Scaffold(
body: Column(
children: [
ChatMarkdownView(text: first, messageId: 'm'),
ChatMarkdownView(text: second, messageId: 'm'),
ChatMarkdownView(text: first, messageId: 'other'),
],
),
),
),
);
await speech.read('m', [first, second]);
engine.onStart!();
await tester.pump();
expect(_highlighted(tester), isEmpty, reason: 'no word reported yet');
engine.speakWord('beta');
await tester.pump();
expect(_highlighted(tester), ['beta']);
engine.speakWord('gamma');
await tester.pump();
expect(_highlighted(tester), ['gamma']);
// The second part's "beta" lights in the second view only.
engine.speakWord('beta', from: engine.spoken.last.indexOf('delta'));
await tester.pump();
expect(_highlighted(tester), ['beta']);
final views = find.byType(ChatMarkdownView);
expect(_highlighted(tester, views.at(1)), ['beta']);
expect(_highlighted(tester, views.at(0)), isEmpty);
await speech.stop();
await tester.pump();
expect(_highlighted(tester), isEmpty);
});
testWidgets('the transcript scrolls only when the word leaves the view', (
tester,
) async {
final engine = FakeSpeechEngine();
final speech = installFakeSpeech(engine);
final text = [
for (var index = 0; index < 40; index += 1) 'Paragraph $index words.',
].join('\n\n');
final scroll = ScrollController();
addTearDown(scroll.dispose);
await tester.pumpWidget(
MaterialApp(
home: Scaffold(
body: SizedBox(
height: 300,
child: ListView(
controller: scroll,
children: [ChatMarkdownView(text: text, messageId: 'm')],
),
),
),
),
);
await speech.read('m', [text]);
engine.onStart!();
engine.speakWord('Paragraph 2 ');
await tester.pumpAndSettle();
expect(scroll.offset, 0, reason: 'the word is already in view');
engine.speakWord('Paragraph 30');
await tester.pumpAndSettle();
expect(scroll.offset, greaterThan(0));
final viewport = tester.getRect(find.byType(ListView));
final word = find.textContaining('Paragraph 30', findRichText: true);
expect(viewport.contains(tester.getRect(word).bottomLeft), isTrue);
final followed = scroll.offset;
// A word still inside the view does not move the transcript.
final spoken = engine.spoken.last;
engine.speakWord('words', from: spoken.indexOf('Paragraph 30'));
await tester.pumpAndSettle();
expect(scroll.offset, followed);
});
}
@@ -0,0 +1,136 @@
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/core/api_gateway.dart';
import 'package:pipelines_app/features/chat/domain/chat_transcript.dart';
import 'package:pipelines_app/features/chat/speech/speech.dart';
import 'package:pipelines_app/features/chat/ui/transcript/chat_copy_gutter.dart';
import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart';
import 'package:pipelines_app/features/chat/ui/transcript/chat_read_aloud_button.dart';
import '../../speech/fake_speech_engine.dart';
Widget _host(Widget child) => MaterialApp(
home: Scaffold(body: Center(child: child)),
);
JsonMap _answer(String id, String text) => transcriptRows([
{
'id': id,
'role': 'assistant',
'created_at': 1000,
'parts': [
{'type': 'text', 'text': text},
],
},
]).single;
final _read = find.byTooltip(chatReadAloudLabel);
final _pause = find.byTooltip(chatPauseReadingLabel);
final _resume = find.byTooltip(chatResumeReadingLabel);
final _stop = find.byTooltip(chatStopReadingLabel);
void main() {
testWidgets('Read aloud becomes Pause and Stop, then Resume', (tester) async {
final engine = FakeSpeechEngine();
final speech = installFakeSpeech(engine);
await tester.pumpWidget(
_host(const ChatReadAloudButton(messageId: 'a', parts: ['Hello'])),
);
expect(_read, findsOneWidget);
expect(_pause, findsNothing);
await tester.tap(_read);
await tester.pump();
expect(engine.spoken, ['Hello']);
expect(_read, findsNothing);
expect(_pause, findsOneWidget);
expect(_stop, findsOneWidget);
await tester.tap(_pause);
await tester.pump();
expect(speech.state, SpeechState.paused);
expect(_resume, findsOneWidget);
expect(_pause, findsNothing);
await tester.tap(_resume);
await tester.pump();
expect(speech.state, SpeechState.speaking);
expect(_pause, findsOneWidget);
await tester.tap(_stop);
await tester.pump();
expect(speech.state, SpeechState.idle);
expect(_read, findsOneWidget);
expect(_stop, findsNothing);
});
testWidgets('only one reply reads at a time', (tester) async {
final engine = FakeSpeechEngine();
installFakeSpeech(engine);
await tester.pumpWidget(
_host(
const Column(
mainAxisSize: MainAxisSize.min,
children: [
ChatReadAloudButton(messageId: 'a', parts: ['First']),
ChatReadAloudButton(messageId: 'b', parts: ['Second']),
],
),
),
);
await tester.tap(_read.first);
await tester.pump();
expect(_stop, findsOneWidget);
await tester.tap(_read);
await tester.pump();
expect(engine.spoken, ['First', 'Second']);
expect(engine.calls, ['speak', 'stop', 'speak']);
expect(_stop, findsOneWidget);
expect(_read, findsOneWidget);
});
testWidgets('hidden where the platform cannot speak', (tester) async {
installFakeSpeech(FakeSpeechEngine(supported: false));
await tester.pumpWidget(
_host(const ChatReadAloudButton(messageId: 'a', parts: ['Hello'])),
);
expect(_read, findsNothing);
});
testWidgets('an answer offers Read aloud beside Copy, footer and gutter', (
tester,
) async {
installFakeSpeech(FakeSpeechEngine());
Future<void> pump(double gutter) => tester.pumpWidget(
MaterialApp(
home: Scaffold(
body: ListView(
children: [
ChatCopyGutter(
width: gutter,
child: ChatMessageView(row: _answer('a1', 'The answer')),
),
],
),
),
),
);
final copy = find.byKey(const Key('chat-message-copy'));
await pump(0);
expect(_read, findsOneWidget);
expect(tester.getCenter(_read).dy, tester.getCenter(copy).dy);
expect(tester.getCenter(_read).dx, greaterThan(tester.getCenter(copy).dx));
// The 48 px gutter stacks the buttons; while reading, Pause and Stop sit
// above Copy and nothing overflows.
await pump(48);
expect(tester.getCenter(_read).dx, tester.getCenter(copy).dx);
expect(tester.getCenter(_read).dy, lessThan(tester.getCenter(copy).dy));
await tester.tap(_read);
await tester.pump();
expect(tester.takeException(), isNull);
expect(tester.getCenter(_stop).dy, lessThan(tester.getCenter(copy).dy));
expect(tester.getCenter(_pause).dy, lessThan(tester.getCenter(_stop).dy));
});
}