Read chat replies aloud with word highlighting

An assistant reply gets Read aloud beside Copy; while it reads it becomes
Pause/Resume and Stop, one reply at a time app-wide. The spoken word is
highlighted in the rendered reply (across multi-part replies) and the
transcript scrolls only when that word leaves the viewport. Reading stops
when its chat closes or is swapped, another reply starts, or the app goes
to the background. Markdown is spoken as sentences; code blocks are
announced as "Code block.".

Speech goes through a SpeechEngine seam with a flutter_tts implementation
(Android, macOS, web; hidden on Linux) and a fake for tests. Settings has
a Read aloud section: a voice picker grouped by language (device language
first, a sample per voice), a rate slider and reset, saved per device.

flutter_tts has no Swift package, so macOS/iOS gain the CocoaPods
integration (Podfiles with the Runner deployment targets); Android
declares the TTS_SERVICE query. The capture fixture gains ?speech=fake.
This commit is contained in:
2026-11-13 01:29:04 +00:00
parent 26e68140fd
commit 6367cce68a
40 changed files with 2643 additions and 29 deletions
@@ -0,0 +1,79 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/chat/speech/speech.dart';
/// A scripted [SpeechEngine]: it records calls and a test fires the platform's
/// events. [stop] reports a cancel like every real platform does.
class FakeSpeechEngine extends SpeechEngine {
FakeSpeechEngine({this.supported = true, this.available = const []});
@override
final bool supported;
final List<SpeechVoice> available;
final calls = <String>[];
final spoken = <String>[];
double? rate;
SpeechVoice? voice;
@override
Future<void> speak(String text) async {
calls.add('speak');
spoken.add(text);
}
@override
Future<void> pause() async => calls.add('pause');
@override
Future<void> stop() async {
calls.add('stop');
onCancel?.call();
}
@override
Future<void> setRate(double rate) async => this.rate = rate;
@override
Future<void> setVoice(SpeechVoice? voice) async => this.voice = voice;
@override
Future<List<SpeechVoice>> voices() async => available;
/// Reports [word] (its first occurrence at or after [from]) being spoken.
void speakWord(String word, {int from = 0}) {
final text = spoken.last;
final start = text.indexOf(word, from);
expect(start, greaterThanOrEqualTo(0), reason: '"$word" is not spoken');
onProgress?.call(text, start, start + word.length);
}
}
/// Saved read-aloud settings held in memory, surviving a "reload".
class MemorySpeechPreferences implements SpeechPreferencesStore {
SpeechPreferences? saved;
@override
Future<SpeechPreferences?> load() async => saved;
@override
Future<bool> save(SpeechPreferences preferences) async {
saved = preferences;
return true;
}
}
/// Installs a reader over [engine] as the app's own for one test.
SpeechController installFakeSpeech(
FakeSpeechEngine engine, {
SpeechPreferencesStore? store,
}) {
final controller = SpeechController(
engine: engine,
store: store ?? MemorySpeechPreferences(),
);
debugSpeechController = controller;
addTearDown(() {
debugSpeechController = null;
controller.dispose();
});
return controller;
}
@@ -0,0 +1,168 @@
import 'package:flutter/widgets.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/chat/speech/speech.dart';
import 'fake_speech_engine.dart';
void main() {
late FakeSpeechEngine engine;
late SpeechController speech;
setUp(() {
engine = FakeSpeechEngine();
});
SpeechController reader([SpeechPreferencesStore? store]) =>
speech = SpeechController(
engine: engine,
store: store ?? MemorySpeechPreferences(),
);
tearDown(() => speech.dispose());
testWidgets('read, pause, resume and stop move through the states', (
tester,
) async {
reader();
await speech.read('a', ['Hello **there**.']);
expect(speech.state, SpeechState.speaking);
expect(speech.messageId, 'a');
expect(engine.spoken.last, 'Hello there.');
engine.onStart!();
await speech.pause();
expect(speech.state, SpeechState.paused);
expect(engine.calls.last, 'pause');
engine.onPause!();
expect(speech.state, SpeechState.paused);
await speech.resume();
expect(speech.state, SpeechState.speaking);
// Every platform resumes when the paused text is spoken again.
expect(engine.spoken.last, 'Hello there.');
engine.onResume!();
await speech.stop();
expect(speech.state, SpeechState.idle);
expect(speech.messageId, isNull);
expect(speech.document, isNull);
});
testWidgets('starting another reply stops the first', (tester) async {
reader();
await speech.read('a', ['First reply']);
engine.onStart!();
engine.speakWord('reply');
await speech.read('b', ['Second reply']);
expect(engine.calls, ['speak', 'stop', 'speak']);
expect(speech.messageId, 'b');
expect(speech.word.value, isNull);
// A late cancel of the first reply does not end the second.
engine.onCancel!();
expect(speech.messageId, 'b');
expect(speech.state, SpeechState.speaking);
});
testWidgets('progress sets the word and maps back to the source', (
tester,
) async {
reader();
const first = 'Intro **bold** word';
const second = '- item word';
await speech.read('a', [first, second]);
engine.onStart!();
engine.speakWord('bold');
final document = speech.document!;
expect(speech.word.value, isNotNull);
expect(document.sourceRange(0, speech.word.value!), (
start: first.indexOf('bold'),
end: first.indexOf('bold') + 4,
));
expect(document.sourceRange(1, speech.word.value!), isNull);
final spoken = engine.spoken.last;
engine.speakWord('word', from: spoken.lastIndexOf('item'));
expect(document.sourceRange(0, speech.word.value!), isNull);
expect(document.sourceRange(1, speech.word.value!), (
start: second.indexOf('word'),
end: second.length,
));
});
testWidgets('progress after a resumed suffix keeps document offsets', (
tester,
) async {
reader();
await speech.read('a', ['one two three']);
engine.onStart!();
// Android speaks the remaining text after a resume.
engine.onProgress!('three', 0, 5);
expect(speech.word.value, (start: 8, end: 13));
});
testWidgets('completion and cancel clear the reply', (tester) async {
reader();
await speech.read('a', ['Done soon']);
engine.onStart!();
engine.speakWord('soon');
engine.onCompletion!();
expect(speech.state, SpeechState.idle);
expect(speech.messageId, isNull);
expect(speech.word.value, isNull);
await speech.read('b', ['Cancelled']);
engine.onStart!();
engine.onCancel!();
expect(speech.state, SpeechState.idle);
expect(speech.messageId, isNull);
});
testWidgets('going to the background stops reading', (tester) async {
reader();
await speech.read('a', ['Background']);
engine.onStart!();
tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.inactive);
await tester.pump();
expect(speech.state, SpeechState.speaking, reason: 'focus loss only');
tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.hidden);
await tester.pump();
expect(speech.state, SpeechState.idle);
expect(engine.calls.last, 'stop');
tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.resumed);
});
testWidgets('stopScope stops only replies of that conversation', (
tester,
) async {
reader();
await speech.read(speechMessageId('chat-1', 'm1'), ['Hello']);
engine.onStart!();
await speech.stopScope('chat-2');
expect(speech.state, SpeechState.speaking);
await speech.stopScope('chat-1');
expect(speech.state, SpeechState.idle);
});
testWidgets('voice and rate are applied and persist across a reload', (
tester,
) async {
final store = MemorySpeechPreferences();
reader(store);
const voice = SpeechVoice(name: 'Daniel', locale: 'en-GB');
await speech.setVoice(voice);
await speech.setRate(1.5);
await speech.read('a', ['Hello']);
expect(engine.voice, voice);
expect(engine.rate, 1.5);
speech.dispose();
reader(store);
await speech.restore();
expect(speech.voice, voice);
expect(speech.rate, 1.5);
await speech.reset();
expect(speech.voice, isNull);
expect(speech.rate, defaultSpeechRate);
expect(store.saved, (rate: defaultSpeechRate, voice: null));
});
}
@@ -0,0 +1,56 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:pipelines_app/features/chat/speech/speech_document.dart';
void main() {
test('markdown is spoken as sentences and code is announced', () {
final result = markdownToSpeech(
'# Welcome\n\n- **First**\n- [Docs](https://example.com)\n\n```dart\nprint(1);\n```\n\nThat is all.',
);
expect(result.text, 'Welcome. First. Docs. Code block. That is all.');
expect(result.sources.length, result.text.length);
});
test('tilde fences and tables are summarised', () {
final result = markdownToSpeech(
'~~~\nsecret();\n~~~\n\n| A | B |\n|---|---|\n| 1 | 2 |',
);
expect(result.text, 'Code block. Table. A. B. 1. 2');
expect(result.text, isNot(contains('secret')));
});
test('repeated words map to their own occurrence', () {
const markdown = 'First repeat\n\nSecond repeat';
final result = markdownToSpeech(markdown);
final first = result.text.indexOf('repeat');
final second = result.text.lastIndexOf('repeat');
expect(result.sourceRange(0, (start: first, end: first + 6)), (
start: 6,
end: 12,
));
expect(result.sourceRange(0, (start: second, end: second + 6)), (
start: markdown.lastIndexOf('repeat'),
end: markdown.length,
));
});
test('text after a code block maps after the skipped code', () {
const markdown = 'Before repeat\n\n```dart\nrepeat();\n```\n\nAfter repeat';
final result = markdownToSpeech(markdown);
final spoken = result.text.lastIndexOf('repeat');
final range = result.sourceRange(0, (start: spoken, end: spoken + 6))!;
expect(range.start, markdown.lastIndexOf('repeat'));
});
test('parts are trimmed and keep their own sources', () {
final result = speechDocumentFor([' One part\n', '', 'Two part']);
expect(result.parts, ['One part', 'Two part']);
expect(result.text, 'One part. Two part');
expect(result.partOf('\nTwo part '), 1);
final two = result.text.indexOf('Two');
expect(result.sourceRange(1, (start: two, end: two + 3)), (
start: 0,
end: 3,
));
expect(result.sourceRange(0, (start: two, end: two + 3)), isNull);
});
}