Read chat replies aloud with word highlighting
An assistant reply gets Read aloud beside Copy; while it reads it becomes Pause/Resume and Stop, one reply at a time app-wide. The spoken word is highlighted in the rendered reply (across multi-part replies) and the transcript scrolls only when that word leaves the viewport. Reading stops when its chat closes or is swapped, another reply starts, or the app goes to the background. Markdown is spoken as sentences; code blocks are announced as "Code block.". Speech goes through a SpeechEngine seam with a flutter_tts implementation (Android, macOS, web; hidden on Linux) and a fake for tests. Settings has a Read aloud section: a voice picker grouped by language (device language first, a sample per voice), a rate slider and reset, saved per device. flutter_tts has no Swift package, so macOS/iOS gain the CocoaPods integration (Podfiles with the Runner deployment targets); Android declares the TTS_SERVICE query. The capture fixture gains ?speech=fake.
This commit is contained in:
@@ -0,0 +1,79 @@
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:pipelines_app/features/chat/speech/speech.dart';
|
||||
|
||||
/// A scripted [SpeechEngine]: it records calls and a test fires the platform's
|
||||
/// events. [stop] reports a cancel like every real platform does.
|
||||
class FakeSpeechEngine extends SpeechEngine {
|
||||
FakeSpeechEngine({this.supported = true, this.available = const []});
|
||||
|
||||
@override
|
||||
final bool supported;
|
||||
final List<SpeechVoice> available;
|
||||
final calls = <String>[];
|
||||
final spoken = <String>[];
|
||||
double? rate;
|
||||
SpeechVoice? voice;
|
||||
|
||||
@override
|
||||
Future<void> speak(String text) async {
|
||||
calls.add('speak');
|
||||
spoken.add(text);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> pause() async => calls.add('pause');
|
||||
|
||||
@override
|
||||
Future<void> stop() async {
|
||||
calls.add('stop');
|
||||
onCancel?.call();
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> setRate(double rate) async => this.rate = rate;
|
||||
|
||||
@override
|
||||
Future<void> setVoice(SpeechVoice? voice) async => this.voice = voice;
|
||||
|
||||
@override
|
||||
Future<List<SpeechVoice>> voices() async => available;
|
||||
|
||||
/// Reports [word] (its first occurrence at or after [from]) being spoken.
|
||||
void speakWord(String word, {int from = 0}) {
|
||||
final text = spoken.last;
|
||||
final start = text.indexOf(word, from);
|
||||
expect(start, greaterThanOrEqualTo(0), reason: '"$word" is not spoken');
|
||||
onProgress?.call(text, start, start + word.length);
|
||||
}
|
||||
}
|
||||
|
||||
/// Saved read-aloud settings held in memory, surviving a "reload".
|
||||
class MemorySpeechPreferences implements SpeechPreferencesStore {
|
||||
SpeechPreferences? saved;
|
||||
|
||||
@override
|
||||
Future<SpeechPreferences?> load() async => saved;
|
||||
|
||||
@override
|
||||
Future<bool> save(SpeechPreferences preferences) async {
|
||||
saved = preferences;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Installs a reader over [engine] as the app's own for one test.
|
||||
SpeechController installFakeSpeech(
|
||||
FakeSpeechEngine engine, {
|
||||
SpeechPreferencesStore? store,
|
||||
}) {
|
||||
final controller = SpeechController(
|
||||
engine: engine,
|
||||
store: store ?? MemorySpeechPreferences(),
|
||||
);
|
||||
debugSpeechController = controller;
|
||||
addTearDown(() {
|
||||
debugSpeechController = null;
|
||||
controller.dispose();
|
||||
});
|
||||
return controller;
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
import 'package:flutter/widgets.dart';
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:pipelines_app/features/chat/speech/speech.dart';
|
||||
|
||||
import 'fake_speech_engine.dart';
|
||||
|
||||
void main() {
|
||||
late FakeSpeechEngine engine;
|
||||
late SpeechController speech;
|
||||
|
||||
setUp(() {
|
||||
engine = FakeSpeechEngine();
|
||||
});
|
||||
|
||||
SpeechController reader([SpeechPreferencesStore? store]) =>
|
||||
speech = SpeechController(
|
||||
engine: engine,
|
||||
store: store ?? MemorySpeechPreferences(),
|
||||
);
|
||||
|
||||
tearDown(() => speech.dispose());
|
||||
|
||||
testWidgets('read, pause, resume and stop move through the states', (
|
||||
tester,
|
||||
) async {
|
||||
reader();
|
||||
await speech.read('a', ['Hello **there**.']);
|
||||
expect(speech.state, SpeechState.speaking);
|
||||
expect(speech.messageId, 'a');
|
||||
expect(engine.spoken.last, 'Hello there.');
|
||||
engine.onStart!();
|
||||
|
||||
await speech.pause();
|
||||
expect(speech.state, SpeechState.paused);
|
||||
expect(engine.calls.last, 'pause');
|
||||
engine.onPause!();
|
||||
expect(speech.state, SpeechState.paused);
|
||||
|
||||
await speech.resume();
|
||||
expect(speech.state, SpeechState.speaking);
|
||||
// Every platform resumes when the paused text is spoken again.
|
||||
expect(engine.spoken.last, 'Hello there.');
|
||||
engine.onResume!();
|
||||
|
||||
await speech.stop();
|
||||
expect(speech.state, SpeechState.idle);
|
||||
expect(speech.messageId, isNull);
|
||||
expect(speech.document, isNull);
|
||||
});
|
||||
|
||||
testWidgets('starting another reply stops the first', (tester) async {
|
||||
reader();
|
||||
await speech.read('a', ['First reply']);
|
||||
engine.onStart!();
|
||||
engine.speakWord('reply');
|
||||
await speech.read('b', ['Second reply']);
|
||||
expect(engine.calls, ['speak', 'stop', 'speak']);
|
||||
expect(speech.messageId, 'b');
|
||||
expect(speech.word.value, isNull);
|
||||
// A late cancel of the first reply does not end the second.
|
||||
engine.onCancel!();
|
||||
expect(speech.messageId, 'b');
|
||||
expect(speech.state, SpeechState.speaking);
|
||||
});
|
||||
|
||||
testWidgets('progress sets the word and maps back to the source', (
|
||||
tester,
|
||||
) async {
|
||||
reader();
|
||||
const first = 'Intro **bold** word';
|
||||
const second = '- item word';
|
||||
await speech.read('a', [first, second]);
|
||||
engine.onStart!();
|
||||
engine.speakWord('bold');
|
||||
final document = speech.document!;
|
||||
expect(speech.word.value, isNotNull);
|
||||
expect(document.sourceRange(0, speech.word.value!), (
|
||||
start: first.indexOf('bold'),
|
||||
end: first.indexOf('bold') + 4,
|
||||
));
|
||||
expect(document.sourceRange(1, speech.word.value!), isNull);
|
||||
|
||||
final spoken = engine.spoken.last;
|
||||
engine.speakWord('word', from: spoken.lastIndexOf('item'));
|
||||
expect(document.sourceRange(0, speech.word.value!), isNull);
|
||||
expect(document.sourceRange(1, speech.word.value!), (
|
||||
start: second.indexOf('word'),
|
||||
end: second.length,
|
||||
));
|
||||
});
|
||||
|
||||
testWidgets('progress after a resumed suffix keeps document offsets', (
|
||||
tester,
|
||||
) async {
|
||||
reader();
|
||||
await speech.read('a', ['one two three']);
|
||||
engine.onStart!();
|
||||
// Android speaks the remaining text after a resume.
|
||||
engine.onProgress!('three', 0, 5);
|
||||
expect(speech.word.value, (start: 8, end: 13));
|
||||
});
|
||||
|
||||
testWidgets('completion and cancel clear the reply', (tester) async {
|
||||
reader();
|
||||
await speech.read('a', ['Done soon']);
|
||||
engine.onStart!();
|
||||
engine.speakWord('soon');
|
||||
engine.onCompletion!();
|
||||
expect(speech.state, SpeechState.idle);
|
||||
expect(speech.messageId, isNull);
|
||||
expect(speech.word.value, isNull);
|
||||
|
||||
await speech.read('b', ['Cancelled']);
|
||||
engine.onStart!();
|
||||
engine.onCancel!();
|
||||
expect(speech.state, SpeechState.idle);
|
||||
expect(speech.messageId, isNull);
|
||||
});
|
||||
|
||||
testWidgets('going to the background stops reading', (tester) async {
|
||||
reader();
|
||||
await speech.read('a', ['Background']);
|
||||
engine.onStart!();
|
||||
tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.inactive);
|
||||
await tester.pump();
|
||||
expect(speech.state, SpeechState.speaking, reason: 'focus loss only');
|
||||
tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.hidden);
|
||||
await tester.pump();
|
||||
expect(speech.state, SpeechState.idle);
|
||||
expect(engine.calls.last, 'stop');
|
||||
tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.resumed);
|
||||
});
|
||||
|
||||
testWidgets('stopScope stops only replies of that conversation', (
|
||||
tester,
|
||||
) async {
|
||||
reader();
|
||||
await speech.read(speechMessageId('chat-1', 'm1'), ['Hello']);
|
||||
engine.onStart!();
|
||||
await speech.stopScope('chat-2');
|
||||
expect(speech.state, SpeechState.speaking);
|
||||
await speech.stopScope('chat-1');
|
||||
expect(speech.state, SpeechState.idle);
|
||||
});
|
||||
|
||||
testWidgets('voice and rate are applied and persist across a reload', (
|
||||
tester,
|
||||
) async {
|
||||
final store = MemorySpeechPreferences();
|
||||
reader(store);
|
||||
const voice = SpeechVoice(name: 'Daniel', locale: 'en-GB');
|
||||
await speech.setVoice(voice);
|
||||
await speech.setRate(1.5);
|
||||
await speech.read('a', ['Hello']);
|
||||
expect(engine.voice, voice);
|
||||
expect(engine.rate, 1.5);
|
||||
speech.dispose();
|
||||
|
||||
reader(store);
|
||||
await speech.restore();
|
||||
expect(speech.voice, voice);
|
||||
expect(speech.rate, 1.5);
|
||||
await speech.reset();
|
||||
expect(speech.voice, isNull);
|
||||
expect(speech.rate, defaultSpeechRate);
|
||||
expect(store.saved, (rate: defaultSpeechRate, voice: null));
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:pipelines_app/features/chat/speech/speech_document.dart';
|
||||
|
||||
void main() {
|
||||
test('markdown is spoken as sentences and code is announced', () {
|
||||
final result = markdownToSpeech(
|
||||
'# Welcome\n\n- **First**\n- [Docs](https://example.com)\n\n```dart\nprint(1);\n```\n\nThat is all.',
|
||||
);
|
||||
expect(result.text, 'Welcome. First. Docs. Code block. That is all.');
|
||||
expect(result.sources.length, result.text.length);
|
||||
});
|
||||
|
||||
test('tilde fences and tables are summarised', () {
|
||||
final result = markdownToSpeech(
|
||||
'~~~\nsecret();\n~~~\n\n| A | B |\n|---|---|\n| 1 | 2 |',
|
||||
);
|
||||
expect(result.text, 'Code block. Table. A. B. 1. 2');
|
||||
expect(result.text, isNot(contains('secret')));
|
||||
});
|
||||
|
||||
test('repeated words map to their own occurrence', () {
|
||||
const markdown = 'First repeat\n\nSecond repeat';
|
||||
final result = markdownToSpeech(markdown);
|
||||
final first = result.text.indexOf('repeat');
|
||||
final second = result.text.lastIndexOf('repeat');
|
||||
expect(result.sourceRange(0, (start: first, end: first + 6)), (
|
||||
start: 6,
|
||||
end: 12,
|
||||
));
|
||||
expect(result.sourceRange(0, (start: second, end: second + 6)), (
|
||||
start: markdown.lastIndexOf('repeat'),
|
||||
end: markdown.length,
|
||||
));
|
||||
});
|
||||
|
||||
test('text after a code block maps after the skipped code', () {
|
||||
const markdown = 'Before repeat\n\n```dart\nrepeat();\n```\n\nAfter repeat';
|
||||
final result = markdownToSpeech(markdown);
|
||||
final spoken = result.text.lastIndexOf('repeat');
|
||||
final range = result.sourceRange(0, (start: spoken, end: spoken + 6))!;
|
||||
expect(range.start, markdown.lastIndexOf('repeat'));
|
||||
});
|
||||
|
||||
test('parts are trimmed and keep their own sources', () {
|
||||
final result = speechDocumentFor([' One part\n', '', 'Two part']);
|
||||
expect(result.parts, ['One part', 'Two part']);
|
||||
expect(result.text, 'One part. Two part');
|
||||
expect(result.partOf('\nTwo part '), 1);
|
||||
final two = result.text.indexOf('Two');
|
||||
expect(result.sourceRange(1, (start: two, end: two + 3)), (
|
||||
start: 0,
|
||||
end: 3,
|
||||
));
|
||||
expect(result.sourceRange(0, (start: two, end: two + 3)), isNull);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user