From 6367cce68a8613ca68024f9ffabc47ba277498c2 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 13 Nov 2026 01:29:04 +0000 Subject: [PATCH] Read chat replies aloud with word highlighting An assistant reply gets Read aloud beside Copy; while it reads it becomes Pause/Resume and Stop, one reply at a time app-wide. The spoken word is highlighted in the rendered reply (across multi-part replies) and the transcript scrolls only when that word leaves the viewport. Reading stops when its chat closes or is swapped, another reply starts, or the app goes to the background. Markdown is spoken as sentences; code blocks are announced as "Code block.". Speech goes through a SpeechEngine seam with a flutter_tts implementation (Android, macOS, web; hidden on Linux) and a fake for tests. Settings has a Read aloud section: a voice picker grouped by language (device language first, a sample per voice), a rate slider and reset, saved per device. flutter_tts has no Swift package, so macOS/iOS gain the CocoaPods integration (Podfiles with the Runner deployment targets); Android declares the TTS_SERVICE query. The capture fixture gains ?speech=fake. --- app/android/app/src/main/AndroidManifest.xml | 4 + app/ios/Flutter/Debug.xcconfig | 1 + app/ios/Flutter/Release.xcconfig | 1 + app/ios/Podfile | 43 +++ .../chat/conversation/chat_thread_view.dart | 21 +- .../speech/flutter_tts_speech_engine.dart | 112 ++++++++ app/lib/features/chat/speech/speech.dart | 4 + .../chat/speech/speech_controller.dart | 265 ++++++++++++++++++ .../features/chat/speech/speech_document.dart | 171 +++++++++++ .../features/chat/speech/speech_engine.dart | 50 ++++ .../chat/speech/speech_preferences.dart | 62 ++++ .../chat/ui/transcript/chat_copy_gutter.dart | 8 +- .../ui/transcript/chat_markdown_speech.dart | 181 ++++++++++++ .../ui/transcript/chat_markdown_view.dart | 155 +++++++++- .../chat/ui/transcript/chat_message_view.dart | 53 +++- .../ui/transcript/chat_read_aloud_button.dart | 69 +++++ .../settings/settings_page_sections.dart | 10 + .../settings/settings_read_aloud_section.dart | 134 +++++++++ .../features/settings/settings_screen.dart | 2 + .../settings/settings_voice_picker.dart | 203 ++++++++++++++ app/macos/Flutter/Flutter-Debug.xcconfig | 1 + app/macos/Flutter/Flutter-Release.xcconfig | 1 + .../Flutter/GeneratedPluginRegistrant.swift | 2 + app/macos/Podfile | 47 ++++ app/macos/Podfile.lock | 22 ++ app/macos/Runner.xcodeproj/project.pbxproj | 102 ++++++- .../contents.xcworkspacedata | 3 + app/notes/read-aloud.md | 55 ++++ app/pubspec.lock | 8 + app/pubspec.yaml | 1 + .../chat_thread_read_aloud_test.dart | 74 +++++ .../chat/speech/fake_speech_engine.dart | 79 ++++++ .../chat/speech/speech_controller_test.dart | 168 +++++++++++ .../chat/speech/speech_document_test.dart | 56 ++++ .../chat_markdown_read_aloud_test.dart | 120 ++++++++ .../chat_read_aloud_button_test.dart | 136 +++++++++ .../settings_read_aloud_section_test.dart | 148 ++++++++++ app/tool/capture/README.md | 4 + app/tool/probe_speech_fixture.dart | 92 ++++++ app/tool/search_selection_probe_entry.dart | 4 + 40 files changed, 2643 insertions(+), 29 deletions(-) create mode 100644 app/ios/Podfile create mode 100644 app/lib/features/chat/speech/flutter_tts_speech_engine.dart create mode 100644 app/lib/features/chat/speech/speech.dart create mode 100644 app/lib/features/chat/speech/speech_controller.dart create mode 100644 app/lib/features/chat/speech/speech_document.dart create mode 100644 app/lib/features/chat/speech/speech_engine.dart create mode 100644 app/lib/features/chat/speech/speech_preferences.dart create mode 100644 app/lib/features/chat/ui/transcript/chat_markdown_speech.dart create mode 100644 app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart create mode 100644 app/lib/features/settings/settings_read_aloud_section.dart create mode 100644 app/lib/features/settings/settings_voice_picker.dart create mode 100644 app/macos/Podfile create mode 100644 app/macos/Podfile.lock create mode 100644 app/notes/read-aloud.md create mode 100644 app/test/features/chat/conversation/chat_thread_read_aloud_test.dart create mode 100644 app/test/features/chat/speech/fake_speech_engine.dart create mode 100644 app/test/features/chat/speech/speech_controller_test.dart create mode 100644 app/test/features/chat/speech/speech_document_test.dart create mode 100644 app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart create mode 100644 app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart create mode 100644 app/test/features/settings/settings_read_aloud_section_test.dart create mode 100644 app/tool/probe_speech_fixture.dart diff --git a/app/android/app/src/main/AndroidManifest.xml b/app/android/app/src/main/AndroidManifest.xml index 2e2bdf08..844805a2 100644 --- a/app/android/app/src/main/AndroidManifest.xml +++ b/app/android/app/src/main/AndroidManifest.xml @@ -115,5 +115,9 @@ + + + + diff --git a/app/ios/Flutter/Debug.xcconfig b/app/ios/Flutter/Debug.xcconfig index 592ceee8..ec97fc6f 100644 --- a/app/ios/Flutter/Debug.xcconfig +++ b/app/ios/Flutter/Debug.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" #include "Generated.xcconfig" diff --git a/app/ios/Flutter/Release.xcconfig b/app/ios/Flutter/Release.xcconfig index 592ceee8..c4855bfe 100644 --- a/app/ios/Flutter/Release.xcconfig +++ b/app/ios/Flutter/Release.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" #include "Generated.xcconfig" diff --git a/app/ios/Podfile b/app/ios/Podfile new file mode 100644 index 00000000..3f878a59 --- /dev/null +++ b/app/ios/Podfile @@ -0,0 +1,43 @@ +# Match the Runner's IPHONEOS_DEPLOYMENT_TARGET. +platform :ios, '15.0' + +# CocoaPods analytics sends network stats synchronously affecting flutter build latency. +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure flutter pub get is executed first" + end + + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Generated.xcconfig, then run flutter pub get" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_ios_podfile_setup + +target 'Runner' do + use_frameworks! + + flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_ios_build_settings(target) + end +end diff --git a/app/lib/features/chat/conversation/chat_thread_view.dart b/app/lib/features/chat/conversation/chat_thread_view.dart index f1c248e8..63cbe05e 100644 --- a/app/lib/features/chat/conversation/chat_thread_view.dart +++ b/app/lib/features/chat/conversation/chat_thread_view.dart @@ -63,6 +63,7 @@ import '../../../ui/layout/app_sheet_header.dart'; import '../../../ui/layout/wheel_scroll_forwarding.dart'; import 'chat_reading_width.dart'; import '../ui/transcript/chat_copy_gutter.dart'; +import '../speech/speech.dart'; import 'chat_pane_presentation.dart'; import 'chat_thread_header.dart'; import 'chat_transcript_anchor.dart'; @@ -655,6 +656,12 @@ class _ChatThreadViewState extends State { @override void didUpdateWidget(covariant ChatThreadView oldWidget) { super.didUpdateWidget(oldWidget); + if (oldWidget.sessionId != widget.sessionId || + oldWidget.gateway != widget.gateway) { + // A pane reused for another conversation stops reading the previous + // one's reply, so neither its audio nor its highlight carries over. + unawaited(speechController.stopScope(oldWidget.sessionId)); + } if (!identical(oldWidget.paneCommands, widget.paneCommands)) { oldWidget.paneCommands?.removeListener(_takeCommands); oldWidget.paneCommands?.setFinished(oldWidget.sessionId, false); @@ -751,6 +758,9 @@ class _ChatThreadViewState extends State { @override void dispose() { + // Leaving or closing the conversation stops its reply reading aloud; a + // reply another pane is reading carries on. + unawaited(speechController.stopScope(widget.sessionId)); _persistPosition(); _saveTranscript(widget.transcripts, widget.sessionId); widget.paneCommands?.removeListener(_takeCommands); @@ -1715,7 +1725,16 @@ class _ChatThreadViewState extends State { [delivery, onRemove != null, _images, _controller, _composer, _shellRuns], () => ChatMessageView( row: row, - textBuilder: chatMarkdownText, + speechScope: widget.sessionId, + // The same reader id as the reply's Read aloud button, so the spoken + // word is highlighted in this reply's text. + textBuilder: (context, text) => ChatMarkdownView( + text: text, + messageId: speechMessageId( + widget.sessionId, + '${row['key'] ?? row['id'] ?? ''}', + ), + ), footer: [ ...chips, if (delivery != null) diff --git a/app/lib/features/chat/speech/flutter_tts_speech_engine.dart b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart new file mode 100644 index 00000000..2ab2700e --- /dev/null +++ b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart @@ -0,0 +1,112 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_tts/flutter_tts.dart'; + +import 'speech_engine.dart'; + +/// [SpeechEngine] over the `flutter_tts` plugin (Android, Apple, web). +/// The plugin is created on first use so merely building the UI opens no +/// platform channel. +class FlutterTtsSpeechEngine extends SpeechEngine { + FlutterTts? _plugin; + SpeechVoice? _applied; + + @override + bool get supported => + kIsWeb || + switch (defaultTargetPlatform) { + TargetPlatform.android || + TargetPlatform.iOS || + TargetPlatform.macOS || + TargetPlatform.windows => true, + TargetPlatform.linux || TargetPlatform.fuchsia => false, + }; + + /// AVSpeechSynthesizer's normal rate is 0.5 on a 0..1 scale; Android's + /// TextToSpeech and the Web Speech API use 1.0 as normal. + bool get _appleScale => + !kIsWeb && + (defaultTargetPlatform == TargetPlatform.iOS || + defaultTargetPlatform == TargetPlatform.macOS); + + FlutterTts get _tts => _plugin ??= _create(); + + FlutterTts _create() { + final tts = FlutterTts(); + tts.setStartHandler(() => onStart?.call()); + tts.setPauseHandler(() => onPause?.call()); + tts.setContinueHandler(() => onResume?.call()); + tts.setCompletionHandler(() => onCompletion?.call()); + tts.setCancelHandler(() => onCancel?.call()); + tts.setErrorHandler((message) => onError?.call('$message')); + tts.setProgressHandler( + (utterance, start, end, _) => onProgress?.call(utterance, start, end), + ); + return tts; + } + + @override + Future speak(String text) async => _tts.speak(text); + + @override + Future pause() async => _tts.pause(); + + @override + Future stop() async => _tts.stop(); + + @override + Future setRate(double rate) async => + _tts.setSpeechRate(_appleScale ? (rate * 0.5).clamp(0, 1) : rate); + + @override + Future setVoice(SpeechVoice? voice) async { + if (voice == _applied) return; + if (voice != null) { + await _tts.setVoice({'name': voice.name, 'locale': voice.locale}); + _applied = voice; + return; + } + if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) { + await _tts.clearVoice(); + _applied = null; + return; + } + // Apple and web plugins cannot clear a chosen voice. After one was chosen + // in this run (a sample, or a setting that was reset) the device + // language's first voice stands in for the system default. + final device = PlatformDispatcher.instance.locale.languageCode; + final fallback = (await voices()) + .where((item) => item.language == device.toLowerCase()) + .firstOrNull; + if (fallback == null) return; + await _tts.setVoice({'name': fallback.name, 'locale': fallback.locale}); + _applied = null; + } + + @override + Future> voices() async { + var voices = await _readVoices(); + // Browsers load their voice list asynchronously after the first request. + for (var attempt = 0; kIsWeb && voices.isEmpty && attempt < 4; attempt++) { + await Future.delayed(const Duration(milliseconds: 250)); + voices = await _readVoices(); + } + return voices; + } + + Future> _readVoices() async { + Object? raw; + try { + raw = await _tts.getVoices; + } on PlatformException { + return const []; + } on MissingPluginException { + return const []; + } + return [ + for (final item in raw is List ? raw : const []) + if (item is Map && item['name'] != null && item['locale'] != null) + SpeechVoice(name: '${item['name']}', locale: '${item['locale']}'), + ]; + } +} diff --git a/app/lib/features/chat/speech/speech.dart b/app/lib/features/chat/speech/speech.dart new file mode 100644 index 00000000..7b73f4c7 --- /dev/null +++ b/app/lib/features/chat/speech/speech.dart @@ -0,0 +1,4 @@ +export 'speech_controller.dart'; +export 'speech_document.dart'; +export 'speech_engine.dart'; +export 'speech_preferences.dart'; diff --git a/app/lib/features/chat/speech/speech_controller.dart b/app/lib/features/chat/speech/speech_controller.dart new file mode 100644 index 00000000..a60d3d19 --- /dev/null +++ b/app/lib/features/chat/speech/speech_controller.dart @@ -0,0 +1,265 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +import 'flutter_tts_speech_engine.dart'; +import 'speech_document.dart'; +import 'speech_engine.dart'; +import 'speech_preferences.dart'; + +enum SpeechState { idle, speaking, paused } + +/// The app-wide reader: one reply speaks at a time. It notifies on state +/// changes; the spoken word moves on the separate, frequent [word]. +class SpeechController extends ChangeNotifier with WidgetsBindingObserver { + SpeechController({SpeechEngine? engine, SpeechPreferencesStore? store}) + : engine = engine ?? FlutterTtsSpeechEngine(), + _store = store ?? const SharedSpeechPreferencesStore() { + WidgetsBinding.instance.addObserver(this); + this.engine + ..onStart = _started + ..onPause = (() => _set(SpeechState.paused)) + ..onResume = (() => _set(SpeechState.speaking)) + ..onCompletion = _ended + ..onCancel = _ended + ..onError = _failed + ..onProgress = _progress; + } + + final SpeechEngine engine; + final SpeechPreferencesStore _store; + + SpeechState _state = SpeechState.idle; + String? _messageId; + SpeechDocument? _document; + Completer? _stopping; + Future? _restoring; + var _awaitingStart = false; + + double _rate = defaultSpeechRate; + SpeechVoice? _voice; + bool saveFailed = false; + + /// The range of the word being spoken in [document]'s text, or null. + final word = ValueNotifier(null); + + bool get supported => engine.supported; + SpeechState get state => _state; + String? get messageId => _messageId; + SpeechDocument? get document => _document; + double get rate => _rate; + SpeechVoice? get voice => _voice; + bool get isDefault => _rate == defaultSpeechRate && _voice == null; + + /// Loads the saved voice and rate once. + Future restore() => _restoring ??= _load(); + + Future _load() async { + final saved = await _store.load(); + if (saved == null) return; + _rate = saved.rate.clamp(minSpeechRate, maxSpeechRate); + _voice = saved.voice; + notifyListeners(); + } + + /// Reads reply [id] made of markdown [parts], stopping any other reading. + Future read(String id, List parts) async { + final document = speechDocumentFor(parts); + await stop(); + await restore(); + if (document.text.isEmpty) return; + await _speak(id, document, _voice); + } + + /// Speaks a short sample in [voice] (or the chosen one) at the chosen rate. + Future sample([SpeechVoice? voice]) async { + await stop(); + await restore(); + await _speak( + speechSampleId, + speechDocumentFor(['This is how replies will sound when read aloud.']), + voice ?? _voice, + ); + } + + Future _speak(String id, SpeechDocument document, SpeechVoice? voice) { + _messageId = id; + _document = document; + _awaitingStart = true; + word.value = null; + _set(SpeechState.speaking); + return _guard(() async { + await engine.setRate(_rate); + await engine.setVoice(voice); + await engine.speak(document.text); + }); + } + + Future pause() async { + if (_state != SpeechState.speaking) return; + _set(SpeechState.paused); + await _guard(engine.pause); + } + + /// Continues a paused reply where it stopped; every platform resumes when + /// the paused utterance's text is spoken again. + Future resume() async { + final document = _document; + if (_state != SpeechState.paused || document == null) return; + _set(SpeechState.speaking); + await _guard(() => engine.speak(document.text)); + } + + Future stop() async { + if (_state == SpeechState.idle) return; + final stopping = _stopping = Completer(); + await _guard(engine.stop); + // Wait briefly for the platform's cancel so it cannot end the next reply + // and so web speech is idle before the next utterance is queued. + await stopping.future.timeout( + const Duration(milliseconds: 400), + onTimeout: () {}, + ); + if (identical(_stopping, stopping)) _stopping = null; + _clear(); + } + + /// Stops reading when the reply being read has an id under [scope]. + Future stopScope(String scope) async { + if (_messageId?.startsWith(speechScopePrefix(scope)) ?? false) { + await stop(); + } + } + + /// Sets the rate for the next reading; [save] false previews a slider drag. + Future setRate(double value, {bool save = true}) async { + _rate = value.clamp(minSpeechRate, maxSpeechRate); + notifyListeners(); + if (save) await _save(); + } + + Future setVoice(SpeechVoice? value) async { + _voice = value; + notifyListeners(); + await _save(); + } + + Future reset() async { + _rate = defaultSpeechRate; + _voice = null; + notifyListeners(); + await _save(); + } + + Future _save() async { + saveFailed = !await _store.save((rate: _rate, voice: _voice)); + notifyListeners(); + } + + Future> voices() => engine.voices(); + + void _started() { + _awaitingStart = false; + if (_messageId != null) _set(SpeechState.speaking); + } + + void _ended() { + final stopping = _stopping; + if (stopping != null) { + if (!stopping.isCompleted) stopping.complete(); + return; + } + // A late cancel from the previous utterance must not end the new one. + if (_awaitingStart) return; + _clear(); + } + + void _failed(String message) { + if (message == 'interrupted' || message == 'canceled') return _ended(); + final stopping = _stopping; + if (stopping != null && !stopping.isCompleted) stopping.complete(); + _clear(); + } + + void _progress(String utterance, int start, int end) { + final document = _document; + if (document == null || _state == SpeechState.idle) return; + // After a resume a platform may speak only the remaining suffix. + final base = document.text.endsWith(utterance) + ? document.text.length - utterance.length + : 0; + _awaitingStart = false; + word.value = (start: start + base, end: end + base); + } + + Future _guard(Future Function() action) async { + try { + await action(); + } catch (error) { + debugPrint('Read aloud failed: $error'); + _clear(); + } + } + + void _set(SpeechState value) { + if (_state == value) return; + _state = value; + notifyListeners(); + } + + void _clear() { + _awaitingStart = false; + _messageId = null; + _document = null; + word.value = null; + _state = SpeechState.idle; + notifyListeners(); + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + // Inactive is only a lost focus (a desktop window behind another, the + // Android shade); hidden and paused mean the app is in the background. + if (state == AppLifecycleState.hidden || + state == AppLifecycleState.paused || + state == AppLifecycleState.detached) { + unawaited(stop()); + } + } + + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + engine + ..onStart = null + ..onPause = null + ..onResume = null + ..onCompletion = null + ..onCancel = null + ..onError = null + ..onProgress = null; + if (_state != SpeechState.idle) unawaited(engine.stop().catchError((_) {})); + word.dispose(); + super.dispose(); + } +} + +/// The reader's id for the Settings voice sample. +const speechSampleId = 'settings-sample'; + +/// The id prefix for replies of one conversation (see [speechMessageId]). +String speechScopePrefix(String scope) => '$scope/'; + +/// A reply's id for the reader, unique across conversations. +String speechMessageId(String scope, String rowId) => '$scope/$rowId'; + +SpeechController? _speechController; + +/// The app's single reader. +SpeechController get speechController => + _speechController ??= SpeechController(); + +/// Replaces the app's reader (a fake engine in tests and capture fixtures); +/// null restores the platform one. +set debugSpeechController(SpeechController? controller) => + _speechController = controller; diff --git a/app/lib/features/chat/speech/speech_document.dart b/app/lib/features/chat/speech/speech_document.dart new file mode 100644 index 00000000..8623367e --- /dev/null +++ b/app/lib/features/chat/speech/speech_document.dart @@ -0,0 +1,171 @@ +import '../ui/transcript/chat_markdown.dart'; +import '../ui/transcript/chat_markdown_cache.dart'; + +typedef SpeechRange = ({int start, int end}); + +/// Where one spoken character came from: an offset into one markdown part. +typedef SpeechSource = ({int part, int offset}); + +/// A reply as spoken text, with the markdown source of every spoken character. +/// A reply can have several text parts; each is rendered by its own view, so +/// sources name the part as well as the offset within its trimmed text. +class SpeechDocument { + const SpeechDocument({ + required this.text, + required this.parts, + required this.sources, + }); + + final String text; + + /// The trimmed markdown of each part, in reading order. + final List parts; + final List sources; + + /// The index of the part whose markdown is [markdown], or -1. + int partOf(String markdown) => parts.indexOf(markdown.trim()); + + /// The source range in [part] that the spoken [range] covers, or null when + /// the spoken text there came from another part. + SpeechRange? sourceRange(int part, SpeechRange range) { + int? start; + int? end; + final last = range.end.clamp(0, sources.length); + for (var index = range.start.clamp(0, last); index < last; index += 1) { + final source = sources[index]; + if (source.part != part) continue; + if (start == null || source.offset < start) start = source.offset; + if (end == null || source.offset + 1 > end) end = source.offset + 1; + } + return start == null ? null : (start: start, end: end!); + } +} + +SpeechDocument markdownToSpeech(String markdown) => + speechDocumentFor([markdown]); + +/// Converts reply parts to natural speech: markup is dropped, blocks become +/// sentences, and a fenced code block is announced as "Code block." rather +/// than read out. Visible characters are located in order in the source, so +/// repeated words map to their own occurrence. +SpeechDocument speechDocumentFor(List markdownParts) { + final parts = [ + for (final part in markdownParts) + if (part.trim().isNotEmpty) part.trim(), + ]; + final builder = _SpeechBuilder(); + for (var index = 0; index < parts.length; index += 1) { + builder.addPart(index, parts[index]); + } + return SpeechDocument( + text: builder.spoken.toString(), + parts: parts, + sources: builder.sources, + ); +} + +class _SpeechBuilder { + final spoken = StringBuffer(); + final sources = []; + String _markdown = ''; + var _part = 0; + var _cursor = 0; + + void addPart(int part, String markdown) { + _part = part; + _markdown = markdown; + _cursor = 0; + for (final block in chatMarkdownCache.blocksFor(markdown)) { + switch (block) { + case MarkdownHeading(text: final text): + _visible(text); + case MarkdownParagraph(text: final text): + _visible(text); + case MarkdownQuote(lines: final lines): + lines.where((line) => line.trim().isNotEmpty).forEach(_visible); + case MarkdownList(items: final items): + items.forEach(_visible); + case MarkdownTable(header: final header, rows: final rows): + _phrase('Table.', _cursor); + [...header, ...rows.expand((row) => row)].forEach(_visible); + case MarkdownCode(code: final code): + _skipCode(code); + case MarkdownRule(): + break; + } + } + } + + /// Ends the previous sentence so blocks are not read as one run-on line. + void _sentenceBreak(int at) { + if (spoken.isEmpty) return; + if (!RegExp(r'[.!?:;]$').hasMatch(spoken.toString())) _write('.', at); + _write(' ', at); + } + + void _write(String value, int at) { + spoken.write(value); + for (var index = 0; index < value.length; index += 1) { + sources.add((part: _part, offset: at)); + } + } + + void _phrase(String value, int at) { + _sentenceBreak(at); + _write(value, at); + } + + void _visible(String blockText) { + final value = _plain(blockText).trim(); + if (value.isEmpty) return; + final start = _markdown.indexOf(blockText, _cursor); + final blockStart = start < 0 ? _cursor : start; + final blockEnd = start < 0 ? _markdown.length : start + blockText.length; + _sentenceBreak(blockStart); + var search = blockStart; + for (var index = 0; index < value.length; index += 1) { + final character = value[index]; + final found = search >= blockEnd + ? -1 + : _markdown.indexOf(character, search); + final inBlock = found >= 0 && found < blockEnd; + spoken.write(character); + sources.add((part: _part, offset: inBlock ? found : search)); + if (inBlock) search = found + 1; + } + _cursor = start < 0 ? search : blockEnd; + } + + void _skipCode(String code) { + final span = fencedCodeSpan(_markdown, _cursor, code); + _phrase('Code block.', span.start); + _cursor = span.end; + } +} + +/// The source range of the fenced code block holding [code] that starts at or +/// after [cursor] (``` or ~~~ fences); an unclosed fence runs to its code's end. +SpeechRange fencedCodeSpan(String source, int cursor, String code) { + int find(String marker) => source.indexOf(marker, cursor); + final fences = [find('```'), find('~~~')].where((index) => index >= 0); + if (fences.isEmpty) return (start: cursor, end: cursor); + final opening = fences.reduce((a, b) => a < b ? a : b); + final marker = source.substring(opening, opening + 3); + final closing = source.indexOf(marker, opening + 3); + if (closing >= 0) return (start: opening, end: closing + 3); + final codeStart = source.indexOf(code, opening + 3); + return ( + start: opening, + end: codeStart < 0 ? opening + 3 : codeStart + code.length, + ); +} + +String _plain(String value) => parseInline(value).map(_inlinePlain).join(); + +String _inlinePlain(MarkdownInline span) => switch (span) { + MarkdownPlain(text: final text) => text, + MarkdownInlineCode(text: final text) => text, + MarkdownStrong(spans: final spans) => spans.map(_inlinePlain).join(), + MarkdownEmphasis(spans: final spans) => spans.map(_inlinePlain).join(), + MarkdownLink(spans: final spans) => spans.map(_inlinePlain).join(), +}; diff --git a/app/lib/features/chat/speech/speech_engine.dart b/app/lib/features/chat/speech/speech_engine.dart new file mode 100644 index 00000000..fddba634 --- /dev/null +++ b/app/lib/features/chat/speech/speech_engine.dart @@ -0,0 +1,50 @@ +import 'package:flutter/foundation.dart'; + +/// One platform voice. [locale] is a BCP 47 tag such as `en-GB`. +@immutable +class SpeechVoice { + const SpeechVoice({required this.name, required this.locale}); + + final String name; + final String locale; + + /// The primary language subtag (`en` for `en-GB`), lower case. + String get language => locale.split(RegExp('[-_]')).first.toLowerCase(); + + @override + bool operator ==(Object other) => + other is SpeechVoice && other.name == name && other.locale == locale; + + @override + int get hashCode => Object.hash(name, locale); +} + +/// The platform text-to-speech service the [SpeechController] drives. +/// +/// [setRate] takes a multiplier where 1.0 is the platform's normal speed; an +/// implementation maps it onto its own scale. [onProgress] reports the word +/// being spoken as offsets into [utterance], the text the platform is actually +/// speaking (after a resume that can be a suffix of the text passed to +/// [speak]). Platforms without word boundaries never call it. +abstract class SpeechEngine { + /// Whether this platform can speak at all; the UI hides read aloud if not. + bool get supported; + + VoidCallback? onStart; + VoidCallback? onPause; + VoidCallback? onResume; + VoidCallback? onCompletion; + VoidCallback? onCancel; + ValueChanged? onError; + void Function(String utterance, int start, int end)? onProgress; + + /// Speaks [text]. Called with the same text while paused, it resumes. + Future speak(String text); + Future pause(); + Future stop(); + + /// Selects [voice], or the platform default when null. + Future setVoice(SpeechVoice? voice); + Future setRate(double rate); + Future> voices(); +} diff --git a/app/lib/features/chat/speech/speech_preferences.dart b/app/lib/features/chat/speech/speech_preferences.dart new file mode 100644 index 00000000..366c54dd --- /dev/null +++ b/app/lib/features/chat/speech/speech_preferences.dart @@ -0,0 +1,62 @@ +import 'package:shared_preferences/shared_preferences.dart'; + +import 'speech_engine.dart'; + +const speechRateKey = 'pipelines.readAloud.rate.v1'; +const speechVoiceNameKey = 'pipelines.readAloud.voiceName.v1'; +const speechVoiceLocaleKey = 'pipelines.readAloud.voiceLocale.v1'; + +/// The normal speaking rate and the slider's bounds (a multiplier). +const defaultSpeechRate = 1.0; +const minSpeechRate = 0.5; +const maxSpeechRate = 2.0; + +typedef SpeechPreferences = ({double rate, SpeechVoice? voice}); + +/// Device-local read-aloud settings. +abstract interface class SpeechPreferencesStore { + Future load(); + Future save(SpeechPreferences preferences); +} + +class SharedSpeechPreferencesStore implements SpeechPreferencesStore { + const SharedSpeechPreferencesStore(); + + @override + Future load() async { + try { + final prefs = await SharedPreferences.getInstance(); + final name = prefs.getString(speechVoiceNameKey); + final locale = prefs.getString(speechVoiceLocaleKey); + return ( + rate: prefs.getDouble(speechRateKey) ?? defaultSpeechRate, + voice: name == null || locale == null + ? null + : SpeechVoice(name: name, locale: locale), + ); + } catch (_) { + return null; + } + } + + @override + Future save(SpeechPreferences preferences) async { + try { + final prefs = await SharedPreferences.getInstance(); + final voice = preferences.voice; + final saved = [ + await prefs.setDouble(speechRateKey, preferences.rate), + if (voice == null) ...[ + await prefs.remove(speechVoiceNameKey), + await prefs.remove(speechVoiceLocaleKey), + ] else ...[ + await prefs.setString(speechVoiceNameKey, voice.name), + await prefs.setString(speechVoiceLocaleKey, voice.locale), + ], + ]; + return !saved.contains(false); + } catch (_) { + return false; + } + } +} diff --git a/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart b/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart index b4d84d31..27e68aa8 100644 --- a/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart +++ b/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart @@ -51,6 +51,7 @@ class ChatCopyGutterRow extends StatefulWidget { required this.width, required this.copy, required this.child, + this.pinned = false, }); final double width; @@ -59,6 +60,9 @@ class ChatCopyGutterRow extends StatefulWidget { final Widget? copy; final Widget child; + /// Keeps the buttons shown without hover (while the reply reads aloud). + final bool pinned; + @override State createState() => _ChatCopyGutterRowState(); } @@ -89,7 +93,9 @@ class _ChatCopyGutterRowState extends State { setState(() => _focused = focused), child: AnimatedOpacity( key: const Key('chat-copy-gutter-visibility'), - opacity: !hover || _hovered || _focused ? 1 : 0, + opacity: !hover || widget.pinned || _hovered || _focused + ? 1 + : 0, duration: const Duration(milliseconds: 120), alwaysIncludeSemantics: true, child: copy, diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart new file mode 100644 index 00000000..72c1dbe7 --- /dev/null +++ b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart @@ -0,0 +1,181 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; + +import '../../speech/speech.dart'; + +/// Space kept between a revealed word and the transcript's edge. +const double _revealMargin = 24; + +/// The range of [text] (one part of reply [messageId]) that [speech] is +/// speaking now, or null. +SpeechRange? spokenSourceIn( + SpeechController speech, + String? messageId, + String text, +) { + final document = speech.document; + final word = speech.word.value; + if (messageId == null || speech.messageId != messageId) return null; + if (document == null || word == null) return null; + final part = document.partOf(text); + if (part < 0) return null; + final range = document.sourceRange(part, word); + if (range == null) return null; + // Document parts are trimmed; the view's offsets include the lead. + final lead = text.length - text.trimLeft().length; + return (start: range.start + lead, end: range.end + lead); +} + +/// [text] as spans whose characters with source [offsets] inside [range] are +/// filled with [fill]; plain text when nothing in it is spoken. +TextSpan spokenTextSpan( + String text, + List offsets, + SpeechRange? range, + Color fill, +) { + if (range == null || offsets.isEmpty) return TextSpan(text: text); + bool spoken(int index) => + offsets[index] >= range.start && offsets[index] < range.end; + final children = []; + var runStart = 0; + for (var index = 1; index <= text.length; index += 1) { + if (index < text.length && spoken(index) == spoken(runStart)) continue; + children.add( + TextSpan( + text: text.substring(runStart, index), + style: spoken(runStart) ? TextStyle(backgroundColor: fill) : null, + ), + ); + runStart = index; + } + return TextSpan(children: children); +} + +/// Scrolls [scroll] just enough to show the spoken [range] when it has left +/// the viewport; a word already in view never moves the transcript. +void revealSpokenWord( + ScrollPosition scroll, + List segments, + SpeechRange range, { + required bool reduceMotion, +}) { + if (!scroll.hasContentDimensions) return; + final segment = segments + .where( + (item) => item.sourceStart < range.end && item.sourceEnd > range.start, + ) + .firstOrNull; + final paragraph = _paragraphIn(segment?.key.currentContext); + if (segment == null || paragraph == null) return; + final boxes = paragraph.getBoxesForSelection( + TextSelection( + baseOffset: segment.renderedStart, + extentOffset: segment.renderedEnd, + ), + ); + final viewport = RenderAbstractViewport.maybeOf(paragraph); + if (boxes.isEmpty || viewport == null) return; + final rect = boxes.first.toRect().inflate(_revealMargin); + final leading = viewport.getOffsetToReveal(paragraph, 0, rect: rect).offset; + final trailing = viewport.getOffsetToReveal(paragraph, 1, rect: rect).offset; + final low = leading < trailing ? leading : trailing; + final high = leading < trailing ? trailing : leading; + final target = scroll.pixels + .clamp(low, high) + .clamp(scroll.minScrollExtent, scroll.maxScrollExtent); + if ((target - scroll.pixels).abs() < 1) return; + if (reduceMotion) { + scroll.jumpTo(target); + return; + } + unawaited( + scroll.animateTo( + target, + duration: const Duration(milliseconds: 240), + curve: Curves.easeOut, + ), + ); +} + +/// The paragraph a keyed [Text] draws; selectable text wraps it in a mouse +/// region, so the keyed element's own render object is not it. +RenderParagraph? _paragraphIn(BuildContext? context) { + RenderParagraph? found; + void visit(RenderObject object) { + if (found != null) return; + if (object is RenderParagraph) { + found = object; + } else { + object.visitChildren(visit); + } + } + + final root = context?.findRenderObject(); + if (root != null) visit(root); + return found; +} + +/// Resolves visible inline leaves in order against the original block source. +/// Searching from the previous match skips emphasis/link/code delimiters and +/// keeps repeated words tied to their own occurrence. +class MarkdownSourceCursor { + MarkdownSourceCursor(this.source, int textStart, this.key) + : _cursor = textStart; + + final String source; + + /// The paragraph these leaves render into. + final GlobalKey key; + var _cursor = 0; + var visibleOffset = 0; + + List consume(String text) { + final offsets = []; + for (var index = 0; index < text.length; index += 1) { + _skipLinkDestination(); + final character = text.substring(index, index + 1); + final searchStart = _cursor.clamp(0, source.length); + final found = searchStart >= source.length + ? -1 + : source.indexOf(character, searchStart); + final offset = found < 0 ? searchStart : found; + offsets.add(offset); + _cursor = found < 0 ? searchStart + 1 : found + 1; + } + return offsets; + } + + void _skipLinkDestination() { + if (_cursor >= source.length || source[_cursor] != ']') return; + if (_cursor + 1 >= source.length || source[_cursor + 1] != '(') return; + var depth = 0; + for (var index = _cursor + 1; index < source.length; index += 1) { + if (source[index] == '(') depth += 1; + if (source[index] == ')' && --depth == 0) { + _cursor = index + 1; + return; + } + } + } +} + +/// One inline leaf: its source range and its character range in [key]'s +/// rendered paragraph. +class MarkdownSpokenSegment { + const MarkdownSpokenSegment( + this.key, + this.sourceStart, + this.sourceEnd, + this.renderedStart, + this.renderedEnd, + ); + + final GlobalKey key; + final int sourceStart; + final int sourceEnd; + final int renderedStart; + final int renderedEnd; +} diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart index af2862d6..065f8393 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart @@ -1,7 +1,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; -import 'package:flutter/semantics.dart'; +import 'package:flutter/rendering.dart'; import 'chat_transcript_selection.dart'; import 'chat_code_hover.dart'; @@ -10,7 +10,9 @@ import 'chat_link_launcher.dart'; import 'chat_link_spans.dart'; import 'chat_markdown.dart'; import 'chat_markdown_cache.dart'; +import 'chat_markdown_speech.dart'; import 'chat_message_bubble.dart'; +import '../../speech/speech.dart'; /// Renders one assistant text part with native text widgets and the host's own /// theme roles. Links (`http`, `https`, `mailto` and `www.` hosts only) open @@ -19,9 +21,15 @@ import 'chat_message_bubble.dart'; /// dimension is overridden; code keeps its exact whitespace and scrolls /// sideways instead of widening the transcript. class ChatMarkdownView extends StatefulWidget { - const ChatMarkdownView({super.key, required this.text, this.onOpenLink}); + const ChatMarkdownView({ + super.key, + required this.text, + this.messageId, + this.onOpenLink, + }); final String text; + final String? messageId; final ValueChanged? onOpenLink; @override @@ -44,6 +52,14 @@ class _ChatMarkdownViewState extends State { final _codeHover = ChatCodeHover(); ScrollPosition? _scroll; ModalRoute? _route; + var _sourceCursor = 0; + + /// The source range of [ChatMarkdownView.text] being spoken, while building. + SpeechRange? _highlight; + SpeechRange? _revealed; + final _textKeys = []; + var _textCount = 0; + final _segments = []; @override void didUpdateWidget(ChatMarkdownView oldWidget) { @@ -77,12 +93,35 @@ class _ChatMarkdownViewState extends State { @override Widget build(BuildContext context) { - // A new token or usage update elsewhere in the turn must not reparse and - // rebuild every earlier answer part, or disturb its text selection. - if (_rendered != null) return _rendered!; + // Keep the state subscribed to inherited styling even when the cached + // subtree is returned without rebuilding its Markdown spans. + Theme.of(context); + final speech = speechController; + return ListenableBuilder( + listenable: Listenable.merge([speech, speech.word]), + builder: (context, _) { + final range = spokenSourceIn(speech, widget.messageId, widget.text); + if (range == null) { + _revealed = null; + return _rendered ?? _buildRendered(context, null); + } + if (range != _revealed) { + _revealed = range; + WidgetsBinding.instance.addPostFrameCallback((_) => _reveal(range)); + } + return _buildRendered(context, range); + }, + ); + } + + Widget _buildRendered(BuildContext context, SpeechRange? range) { + _sourceCursor = 0; + _textCount = 0; + _highlight = range; + _segments.clear(); final gap = chatParagraphGap(context); final blocks = chatMarkdownCache.blocksFor(widget.text); - return _rendered = Listener( + final rendered = Listener( onPointerDown: (_) => _codeHover.hide(), onPointerSignal: (_) => _codeHover.hide(), child: NotificationListener( @@ -108,6 +147,8 @@ class _ChatMarkdownViewState extends State { ), ), ); + if (range == null) _rendered = rendered; + return rendered; } /// Blocks are separated by an empty line of body text, like paragraphs; @@ -196,6 +237,7 @@ class _ChatMarkdownViewState extends State { ), ); case MarkdownCode(language: final language, code: final code): + _skipCodeBlock(code); final labelStyle = _roleStyle( context, Theme.of(context).textTheme.labelSmall, @@ -220,6 +262,14 @@ class _ChatMarkdownViewState extends State { } } + void _skipCodeBlock(String code) { + _sourceCursor = fencedCodeSpan( + widget.text, + _sourceCursor, + code, + ).end.clamp(0, widget.text.length); + } + Widget _cell(BuildContext context, String text, {bool header = false}) { final style = header ? _roleStyle(context, Theme.of(context).textTheme.labelLarge) @@ -234,43 +284,77 @@ class _ChatMarkdownViewState extends State { } Widget _text(BuildContext context, String text, {TextStyle? style}) { + if (text.isEmpty) return const SizedBox.shrink(); + final sourceStart = widget.text.indexOf(text, _sourceCursor); + if (sourceStart >= 0) _sourceCursor = sourceStart + text.length; + // Keys follow render order, which is stable for the same text, so the + // revealed word's paragraph can be found after the frame. + if (_textCount == _textKeys.length) _textKeys.add(GlobalKey()); + final key = _textKeys[_textCount++]; + final source = MarkdownSourceCursor( + widget.text, + sourceStart < 0 ? _sourceCursor : sourceStart, + key, + ); return Builder( builder: (textContext) => Text.rich( - TextSpan(children: _spans(textContext, parseInline(text))), + key: key, + TextSpan(children: _spans(textContext, parseInline(text), source)), style: style, ), ); } - List _spans(BuildContext context, List spans) => [ + List _spans( + BuildContext context, + List spans, + MarkdownSourceCursor source, + ) => [ for (final span in spans) switch (span) { - MarkdownPlain(text: final text) => TextSpan(text: text), - MarkdownInlineCode(text: final text) => _inlineCode(context, text), + MarkdownPlain(text: final text) => _leaf(context, text, source), + MarkdownInlineCode(text: final text) => _inlineCode( + context, + text, + source, + ), MarkdownStrong(spans: final inner) => TextSpan( style: const TextStyle(fontWeight: FontWeight.bold), - children: _spans(context, inner), + children: _spans(context, inner, source), ), MarkdownEmphasis(spans: final inner) => TextSpan( style: const TextStyle(fontStyle: FontStyle.italic), - children: _spans(context, inner), + children: _spans(context, inner, source), ), MarkdownLink(url: final url, spans: final inner) => _link( context, url, inner, + source, ), }, ]; - TextSpan _inlineCode(BuildContext context, String text) { + TextSpan _leaf( + BuildContext context, + String text, + MarkdownSourceCursor source, + ) => _highlightedText(context, text, source); + + TextSpan _inlineCode( + BuildContext context, + String text, + MarkdownSourceCursor source, + ) { final token = text; + final highlighted = _highlightedText(context, text, source); return TextSpan( - text: text, style: TextStyle( fontFamily: appMonospaceFamily, backgroundColor: _codeFill(context), ), + text: highlighted.text, + children: highlighted.children, onEnter: (event) { if (_route?.isCurrent != false) { _codeHover.show(context, token, text, event.position); @@ -284,10 +368,11 @@ class _ChatMarkdownViewState extends State { BuildContext context, String url, List spans, + MarkdownSourceCursor source, ) => _links.link( context, url, - _spans(context, spans), + _spans(context, spans, source), onOpen: (uri) { final onOpenLink = widget.onOpenLink; if (onOpenLink != null) return onOpenLink(uri.toString()); @@ -295,6 +380,46 @@ class _ChatMarkdownViewState extends State { }, ); + /// A leaf's spans, recording where it renders so a spoken word in it can + /// be revealed, with the spoken characters highlighted. + TextSpan _highlightedText( + BuildContext context, + String text, + MarkdownSourceCursor source, + ) { + final offsets = source.consume(text); + final renderedStart = source.visibleOffset; + source.visibleOffset += text.length; + if (offsets.isNotEmpty) { + _segments.add( + MarkdownSpokenSegment( + source.key, + offsets.first, + offsets.last + 1, + renderedStart, + source.visibleOffset, + ), + ); + } + return spokenTextSpan( + text, + offsets, + _highlight, + Theme.of(context).colorScheme.primaryContainer, + ); + } + + void _reveal(SpeechRange range) { + final scroll = _scroll; + if (!mounted || range != _revealed || scroll == null) return; + revealSpokenWord( + scroll, + _segments, + range, + reduceMotion: MediaQuery.maybeDisableAnimationsOf(context) ?? false, + ); + } + /// Heading roles are a non-increasing sequence from the host's own text /// theme. H3–H6 share one title role, matching the web renderer's flat /// lower-heading treatment, and no level falls back to a body role, so a diff --git a/app/lib/features/chat/ui/transcript/chat_message_view.dart b/app/lib/features/chat/ui/transcript/chat_message_view.dart index 1fee3081..813663e5 100644 --- a/app/lib/features/chat/ui/transcript/chat_message_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_message_view.dart @@ -14,6 +14,8 @@ import 'chat_message_bubble.dart'; import 'chat_turn_activity_view.dart'; import 'native_disclosure.dart'; import 'chat_message_timestamp.dart'; +import 'chat_read_aloud_button.dart'; +import '../../speech/speech_controller.dart'; /// Builds an assistant text part; the default keeps the server's text exactly. typedef ChatMessageTextBuilder = Widget Function(BuildContext, String); @@ -27,9 +29,14 @@ class ChatMessageView extends StatelessWidget { this.textBuilder, this.footer = const [], this.images, + this.speechScope = 'chat', }); final JsonMap row; + + /// Names the conversation in the reader's message ids, so leaving it can + /// stop only its own reply (see [speechMessageId]). + final String speechScope; final ChatMessageTextBuilder? textBuilder; /// Loads attached images for the operator's turn; without one, only images @@ -179,13 +186,41 @@ class ChatMessageView extends StatelessWidget { Widget answer, double gutter, ) { - final copyText = row['streaming'] == true ? '' : _answerText(_partsOf(row)); - final copy = copyText.isEmpty ? null : ChatCopyButton(text: copyText); - if (gutter <= 0) return _agentTurn(context, content, answer, copy); - return ChatCopyGutterRow( - width: gutter, - copy: copy, - child: _agentTurn(context, content, answer, null), + final answerParts = row['streaming'] == true + ? const [] + : _answerParts(_partsOf(row)); + final copyText = answerParts.join('\n\n'); + final id = speechMessageId(speechScope, '${row['key'] ?? row['id'] ?? ''}'); + Widget? actions(Axis direction) => copyText.isEmpty + ? null + : Flex( + direction: direction, + mainAxisSize: MainAxisSize.min, + verticalDirection: VerticalDirection.up, + children: [ + ChatCopyButton(text: copyText), + ChatReadAloudButton( + messageId: id, + parts: answerParts, + direction: direction, + ), + ], + ); + if (gutter <= 0) { + return _agentTurn(context, content, answer, actions(Axis.horizontal)); + } + // The gutter is one button wide: Read aloud stacks above Copy, and stays + // shown on a desktop while this reply reads. + final turn = _agentTurn(context, content, answer, null); + return ListenableBuilder( + listenable: speechController, + child: turn, + builder: (context, child) => ChatCopyGutterRow( + width: gutter, + copy: actions(Axis.vertical), + pinned: speechController.messageId == id, + child: child!, + ), ); } @@ -267,13 +302,13 @@ class ChatMessageView extends StatelessWidget { /// The answer as the operator reads it: the turn's text parts in order, /// without reasoning, tools or attachments. - String _answerText(List parts) => [ + List _answerParts(List parts) => [ for (final part in parts) if ((part['type'] == null || part['type'] == 'text') && part['text'] is String && (part['text'] as String).trim().isNotEmpty) (part['text'] as String).trim(), - ].join('\n\n'); + ]; List _agentParts(BuildContext context) { final widgets = []; diff --git a/app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart b/app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart new file mode 100644 index 00000000..543b7dca --- /dev/null +++ b/app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart @@ -0,0 +1,69 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../../speech/speech.dart'; + +const chatReadAloudLabel = 'Read aloud'; +const chatPauseReadingLabel = 'Pause reading'; +const chatResumeReadingLabel = 'Resume reading'; +const chatStopReadingLabel = 'Stop reading'; + +/// Read aloud for one reply, styled like its Copy button. While this reply +/// reads it becomes Pause (or Resume) and Stop, laid out along [direction] +/// (a column in the narrow copy gutter). Absent where speech is unsupported. +class ChatReadAloudButton extends StatelessWidget { + const ChatReadAloudButton({ + super.key, + required this.messageId, + required this.parts, + this.direction = Axis.horizontal, + }); + + final String messageId; + + /// The reply's markdown text parts, in order. + final List parts; + final Axis direction; + + @override + Widget build(BuildContext context) { + final speech = speechController; + if (!speech.supported) return const SizedBox.shrink(); + return ListenableBuilder( + listenable: speech, + builder: (context, _) { + if (speech.messageId != messageId) { + return IconButton( + key: const Key('chat-message-read-aloud'), + tooltip: chatReadAloudLabel, + onPressed: () => unawaited(speech.read(messageId, parts)), + icon: const Icon(Icons.volume_up_outlined), + ); + } + final paused = speech.state == SpeechState.paused; + return Flex( + direction: direction, + mainAxisSize: MainAxisSize.min, + children: [ + IconButton( + key: const Key('chat-message-read-aloud-pause'), + tooltip: paused ? chatResumeReadingLabel : chatPauseReadingLabel, + onPressed: () => + unawaited(paused ? speech.resume() : speech.pause()), + icon: Icon( + paused ? Icons.play_arrow_outlined : Icons.pause_outlined, + ), + ), + IconButton( + key: const Key('chat-message-read-aloud-stop'), + tooltip: chatStopReadingLabel, + onPressed: () => unawaited(speech.stop()), + icon: const Icon(Icons.stop_outlined), + ), + ], + ); + }, + ); + } +} diff --git a/app/lib/features/settings/settings_page_sections.dart b/app/lib/features/settings/settings_page_sections.dart index ce285e21..1551b1cf 100644 --- a/app/lib/features/settings/settings_page_sections.dart +++ b/app/lib/features/settings/settings_page_sections.dart @@ -12,6 +12,13 @@ extension _SettingsPageSections on _SettingsScreenState { keywords: 'text size scale zoom large small font', builder: (_) => const SettingsScaleSection(), ), + if (speechController.supported) + const SettingsSection( + id: 'read-aloud', + title: 'Read aloud', + keywords: 'speech voice audio speaking rate read aloud', + builder: _readAloudBuilder, + ), SettingsSection( id: 'animations', title: 'Animations', @@ -216,6 +223,9 @@ extension _SettingsPageSections on _SettingsScreenState { ]; } + static Widget _readAloudBuilder(BuildContext _) => + const SettingsReadAloudSection(); + /// A group holding one free-form block (a form, a status with buttons). Widget _single(String title, Widget child) => SettingsGroup( title: title, diff --git a/app/lib/features/settings/settings_read_aloud_section.dart b/app/lib/features/settings/settings_read_aloud_section.dart new file mode 100644 index 00000000..826ba555 --- /dev/null +++ b/app/lib/features/settings/settings_read_aloud_section.dart @@ -0,0 +1,134 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../chat/speech/speech.dart'; +import 'settings_group.dart'; +import 'settings_row.dart'; +import 'settings_theme.dart'; +import 'settings_voice_picker.dart'; + +/// Read aloud: the voice and speaking rate replies are read with, saved on +/// this device, with a sample and a reset to the system voice at normal speed. +class SettingsReadAloudSection extends StatefulWidget { + const SettingsReadAloudSection({super.key, this.controller}); + + /// The reader; the app's own by default. + final SpeechController? controller; + + @override + State createState() => + _SettingsReadAloudSectionState(); +} + +class _SettingsReadAloudSectionState extends State { + SpeechController get _speech => widget.controller ?? speechController; + + @override + void initState() { + super.initState(); + unawaited(_speech.restore()); + } + + Future _pickVoice() async { + final choice = await showSpeechVoicePicker(context, _speech); + if (choice != null) await _speech.setVoice(choice.voice); + } + + @override + Widget build(BuildContext context) => ListenableBuilder( + listenable: _speech, + builder: (context, _) { + final rate = _speech.rate; + final label = '${rate.toStringAsFixed(1)}×'; + final voice = _speech.voice; + final sampling = _speech.messageId == speechSampleId; + return SettingsGroup( + title: 'Read aloud', + footer: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + const Text( + 'Saved on this device. Replies are read with this voice and rate.', + ), + if (_speech.saveFailed) + Text( + 'Could not save; applies until reload.', + style: TextStyle(color: settingsErrorColor(context)), + ), + ], + ), + children: [ + SettingsRow( + key: const Key('settings-read-aloud-voice'), + leading: const Icon(Icons.record_voice_over_outlined), + title: const Text('Voice'), + subtitle: Text( + voice == null + ? 'System default' + : '${voice.name} · ${voice.locale}', + ), + chevron: true, + onTap: () => unawaited(_pickVoice()), + ), + SettingsGroupItem( + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Text( + 'Speaking rate $label', + style: settingsTextStyle(context, title: true), + ), + Semantics( + label: 'Speaking rate', + child: Slider( + key: const Key('settings-read-aloud-rate'), + value: rate, + min: minSpeechRate, + max: maxSpeechRate, + divisions: 15, + label: label, + semanticFormatterCallback: (value) => + '${value.toStringAsFixed(1)} times', + onChanged: (value) => + unawaited(_speech.setRate(value, save: false)), + onChangeEnd: (value) => unawaited(_speech.setRate(value)), + ), + ), + const Wrap( + alignment: WrapAlignment.spaceBetween, + children: [Text('0.5× · Slower'), Text('2.0× · Faster')], + ), + ], + ), + ), + SettingsGroupItem( + child: Wrap( + spacing: 8, + runSpacing: 8, + crossAxisAlignment: WrapCrossAlignment.center, + children: [ + FilledButton.tonalIcon( + key: const Key('settings-read-aloud-sample'), + onPressed: () => + unawaited(sampling ? _speech.stop() : _speech.sample()), + icon: Icon( + sampling ? Icons.stop_outlined : Icons.play_arrow_outlined, + ), + label: Text(sampling ? 'Stop sample' : 'Play sample'), + ), + TextButton( + key: const Key('settings-read-aloud-reset'), + onPressed: _speech.isDefault + ? null + : () => unawaited(_speech.reset()), + child: const Text('Reset to defaults'), + ), + ], + ), + ), + ], + ); + }, + ); +} diff --git a/app/lib/features/settings/settings_screen.dart b/app/lib/features/settings/settings_screen.dart index 6c8ac6f2..0a126c6f 100644 --- a/app/lib/features/settings/settings_screen.dart +++ b/app/lib/features/settings/settings_screen.dart @@ -40,6 +40,8 @@ import 'managed_connection_section.dart'; import 'settings_keyboard_shortcuts_section.dart'; import 'settings_keys.dart'; import 'settings_row.dart'; +import 'settings_read_aloud_section.dart'; +import '../chat/speech/speech_controller.dart'; part 'settings_page_sections.dart'; diff --git a/app/lib/features/settings/settings_voice_picker.dart b/app/lib/features/settings/settings_voice_picker.dart new file mode 100644 index 00000000..c714474d --- /dev/null +++ b/app/lib/features/settings/settings_voice_picker.dart @@ -0,0 +1,203 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../chat/speech/speech.dart'; + +/// The picker's answer: a voice, or null for the system default. +typedef SpeechVoiceChoice = ({SpeechVoice? voice}); + +/// Lists the platform's voices grouped by language, the device's language +/// first, each with a sample. Resolves to null when dismissed. +Future showSpeechVoicePicker( + BuildContext context, + SpeechController speech, +) async { + final choice = await showDialog( + context: context, + builder: (_) => _VoicePickerDialog(speech: speech), + ); + if (speech.messageId == speechSampleId) unawaited(speech.stop()); + return choice; +} + +/// Voices grouped by language subtag, the [device] language first and the +/// rest by name; voices within a group by name. +List<({String language, List voices})> groupSpeechVoices( + List voices, + String device, +) { + final groups = >{}; + for (final voice in voices) { + (groups[voice.language] ??= []).add(voice); + } + final languages = groups.keys.toList() + ..sort((a, b) { + if (a == device.toLowerCase()) return -1; + if (b == device.toLowerCase()) return 1; + return speechLanguageName(a).compareTo(speechLanguageName(b)); + }); + return [ + for (final language in languages) + ( + language: language, + voices: groups[language]! + ..sort((a, b) { + final name = a.name.compareTo(b.name); + return name != 0 ? name : a.locale.compareTo(b.locale); + }), + ), + ]; +} + +/// An English name for common language subtags; others show their code. +String speechLanguageName(String language) => + _languageNames[language] ?? language.toUpperCase(); + +const _languageNames = { + 'ar': 'Arabic', + 'bg': 'Bulgarian', + 'ca': 'Catalan', + 'cs': 'Czech', + 'da': 'Danish', + 'de': 'German', + 'el': 'Greek', + 'en': 'English', + 'es': 'Spanish', + 'fi': 'Finnish', + 'fr': 'French', + 'he': 'Hebrew', + 'hi': 'Hindi', + 'hr': 'Croatian', + 'hu': 'Hungarian', + 'id': 'Indonesian', + 'it': 'Italian', + 'ja': 'Japanese', + 'ko': 'Korean', + 'ms': 'Malay', + 'nb': 'Norwegian', + 'nl': 'Dutch', + 'pl': 'Polish', + 'pt': 'Portuguese', + 'ro': 'Romanian', + 'ru': 'Russian', + 'sk': 'Slovak', + 'sv': 'Swedish', + 'th': 'Thai', + 'tr': 'Turkish', + 'uk': 'Ukrainian', + 'vi': 'Vietnamese', + 'zh': 'Chinese', +}; + +class _VoicePickerDialog extends StatefulWidget { + const _VoicePickerDialog({required this.speech}); + + final SpeechController speech; + + @override + State<_VoicePickerDialog> createState() => _VoicePickerDialogState(); +} + +class _VoicePickerDialogState extends State<_VoicePickerDialog> { + late final Future> _voices = widget.speech.voices(); + + String _keyOf(SpeechVoice? voice) => + voice == null ? '' : '${voice.name}\n${voice.locale}'; + + @override + Widget build(BuildContext context) { + final device = View.of(context).platformDispatcher.locale.languageCode; + return AlertDialog( + title: const Text('Voice'), + contentPadding: const EdgeInsets.symmetric(vertical: 16), + // A list needs a definite width; keep it to a readable dialog measure. + content: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 560), + child: SizedBox( + width: double.maxFinite, + child: FutureBuilder>( + future: _voices, + builder: (context, snapshot) { + if (snapshot.connectionState != ConnectionState.done) { + return const Center(child: CircularProgressIndicator()); + } + final voices = snapshot.data ?? const []; + return _list(context, groupSpeechVoices(voices, device)); + }, + ), + ), + ), + actions: [ + TextButton( + onPressed: () => Navigator.of(context).pop(), + child: const Text('Cancel'), + ), + ], + ); + } + + Widget _list( + BuildContext context, + List<({String language, List voices})> groups, + ) { + final theme = Theme.of(context); + final header = theme.textTheme.titleSmall?.copyWith( + color: theme.colorScheme.primary, + ); + return RadioGroup( + groupValue: _keyOf(widget.speech.voice), + onChanged: (key) { + final voice = [ + for (final group in groups) ...group.voices, + ].where((voice) => _keyOf(voice) == key).firstOrNull; + Navigator.of(context).pop((voice: voice)); + }, + child: ListView( + shrinkWrap: true, + children: [ + const RadioListTile(value: '', title: Text('System default')), + if (groups.isEmpty) + Padding( + padding: const EdgeInsets.symmetric(horizontal: 24, vertical: 8), + child: Text( + 'This device offers no other voices.', + style: TextStyle(color: theme.colorScheme.onSurfaceVariant), + ), + ), + for (final group in groups) ...[ + Padding( + padding: const EdgeInsetsDirectional.fromSTEB(24, 16, 24, 4), + child: Semantics( + header: true, + child: Text(speechLanguageName(group.language), style: header), + ), + ), + for (final voice in group.voices) + // The sample is its own control beside the choice, not merged + // into the radio tile's semantics. + Row( + children: [ + Expanded( + child: RadioListTile( + value: _keyOf(voice), + title: Text(voice.name), + subtitle: Text(voice.locale), + ), + ), + Padding( + padding: const EdgeInsetsDirectional.only(end: 12), + child: IconButton( + tooltip: 'Play sample of ${voice.name}', + onPressed: () => unawaited(widget.speech.sample(voice)), + icon: const Icon(Icons.play_arrow_outlined), + ), + ), + ], + ), + ], + ], + ), + ); + } +} diff --git a/app/macos/Flutter/Flutter-Debug.xcconfig b/app/macos/Flutter/Flutter-Debug.xcconfig index c2efd0b6..4b81f9b2 100644 --- a/app/macos/Flutter/Flutter-Debug.xcconfig +++ b/app/macos/Flutter/Flutter-Debug.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" #include "ephemeral/Flutter-Generated.xcconfig" diff --git a/app/macos/Flutter/Flutter-Release.xcconfig b/app/macos/Flutter/Flutter-Release.xcconfig index c2efd0b6..5caa9d15 100644 --- a/app/macos/Flutter/Flutter-Release.xcconfig +++ b/app/macos/Flutter/Flutter-Release.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" #include "ephemeral/Flutter-Generated.xcconfig" diff --git a/app/macos/Flutter/GeneratedPluginRegistrant.swift b/app/macos/Flutter/GeneratedPluginRegistrant.swift index 3da2e7b4..8e871d2b 100644 --- a/app/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/app/macos/Flutter/GeneratedPluginRegistrant.swift @@ -8,6 +8,7 @@ import Foundation import bonsoir_darwin import file_picker_darwin import flutter_local_notifications +import flutter_tts import package_info_plus import share_plus import shared_preferences_foundation @@ -19,6 +20,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { SwiftBonsoirPlugin.register(with: registry.registrar(forPlugin: "SwiftBonsoirPlugin")) FilePickerPlugin.register(with: registry.registrar(forPlugin: "FilePickerPlugin")) FlutterLocalNotificationsPlugin.register(with: registry.registrar(forPlugin: "FlutterLocalNotificationsPlugin")) + FlutterTtsPlugin.register(with: registry.registrar(forPlugin: "FlutterTtsPlugin")) FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) SharePlusMacosPlugin.register(with: registry.registrar(forPlugin: "SharePlusMacosPlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) diff --git a/app/macos/Podfile b/app/macos/Podfile new file mode 100644 index 00000000..88ac4486 --- /dev/null +++ b/app/macos/Podfile @@ -0,0 +1,47 @@ +# Match the Runner's MACOSX_DEPLOYMENT_TARGET; current Xcode rejects 10.15. +platform :osx, '12.0' + +# CocoaPods analytics sends network stats synchronously affecting flutter build latency. +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'ephemeral', 'Flutter-Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure \"flutter pub get\" is executed first" + end + + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Flutter-Generated.xcconfig, then run \"flutter pub get\"" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_macos_podfile_setup + +target 'Runner' do + use_frameworks! + + flutter_install_all_macos_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_macos_build_settings(target) + # Plugin podspecs (flutter_tts) still declare 10.15. + target.build_configurations.each do |config| + config.build_settings['MACOSX_DEPLOYMENT_TARGET'] = '12.0' + end + end +end diff --git a/app/macos/Podfile.lock b/app/macos/Podfile.lock new file mode 100644 index 00000000..f6c9bbcf --- /dev/null +++ b/app/macos/Podfile.lock @@ -0,0 +1,22 @@ +PODS: + - flutter_tts (0.0.1): + - FlutterMacOS + - FlutterMacOS (1.0.0) + +DEPENDENCIES: + - flutter_tts (from `Flutter/ephemeral/.symlinks/plugins/flutter_tts/macos`) + - FlutterMacOS (from `Flutter/ephemeral`) + +EXTERNAL SOURCES: + flutter_tts: + :path: Flutter/ephemeral/.symlinks/plugins/flutter_tts/macos + FlutterMacOS: + :path: Flutter/ephemeral + +SPEC CHECKSUMS: + flutter_tts: ae915565cc6948444b513acc8ee021993281e027 + FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1 + +PODFILE CHECKSUM: 65be95680b03b164066c9a7e265a6f0a7b094da4 + +COCOAPODS: 1.17.0 diff --git a/app/macos/Runner.xcodeproj/project.pbxproj b/app/macos/Runner.xcodeproj/project.pbxproj index f2a52d37..be55c5d8 100644 --- a/app/macos/Runner.xcodeproj/project.pbxproj +++ b/app/macos/Runner.xcodeproj/project.pbxproj @@ -28,6 +28,8 @@ 33CC10F62044A3C60003C045 /* MainMenu.xib in Resources */ = {isa = PBXBuildFile; fileRef = 33CC10F42044A3C60003C045 /* MainMenu.xib */; }; 33CC11132044BFA00003C045 /* MainFlutterWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33CC11122044BFA00003C045 /* MainFlutterWindow.swift */; }; 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */ = {isa = PBXBuildFile; productRef = 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */; }; + C7BD897E1E312E5E095D3400 /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = BA982DFB3805F270CAE1EC57 /* Pods_Runner.framework */; }; + EA524FBC4D8125AD71D1AB9E /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 52DA5BA5A5AAB1A3F8E8CACC /* Pods_RunnerTests.framework */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -61,11 +63,13 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 12FE65799543E39BACA69E1B /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.profile.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig"; sourceTree = ""; }; + 30F06383B0F1A8CC493DE4C3 /* Pods-RunnerTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.debug.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.debug.xcconfig"; sourceTree = ""; }; 331C80D5294CF71000263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 331C80D7294CF71000263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; 333000ED22D3DE5D00554162 /* Warnings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Warnings.xcconfig; sourceTree = ""; }; 335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GeneratedPluginRegistrant.swift; sourceTree = ""; }; - 33CC10ED2044A3C60003C045 /* Pipelines.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Pipelines.app"; sourceTree = BUILT_PRODUCTS_DIR; }; + 33CC10ED2044A3C60003C045 /* Pipelines.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Pipelines.app; sourceTree = BUILT_PRODUCTS_DIR; }; 33CC10F02044A3C60003C045 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 33CC10F22044A3C60003C045 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Assets.xcassets; path = Runner/Assets.xcassets; sourceTree = ""; }; 33CC10F52044A3C60003C045 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = ""; }; @@ -77,9 +81,15 @@ 33E51913231747F40026EE4D /* DebugProfile.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = DebugProfile.entitlements; sourceTree = ""; }; 33E51914231749380026EE4D /* Release.entitlements */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.entitlements; path = Release.entitlements; sourceTree = ""; }; 33E5194F232828860026EE4D /* AppInfo.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = AppInfo.xcconfig; sourceTree = ""; }; + 52DA5BA5A5AAB1A3F8E8CACC /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 769023E35970ADB12A4AC2D7 /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = ""; }; 78E0A7A72DC9AD7400C4905E /* FlutterGeneratedPluginSwiftPackage */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = FlutterGeneratedPluginSwiftPackage; path = ephemeral/Packages/FlutterGeneratedPluginSwiftPackage; sourceTree = ""; }; 7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Release.xcconfig; sourceTree = ""; }; + 8C9F29817EA0A7110E900E6F /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = ""; }; 9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = Debug.xcconfig; sourceTree = ""; }; + B396C4E4E3EE1B4BB1E67366 /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = ""; }; + BA982DFB3805F270CAE1EC57 /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D9747DDA8126822426B339BB /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -87,6 +97,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + EA524FBC4D8125AD71D1AB9E /* Pods_RunnerTests.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -95,6 +106,7 @@ buildActionMask = 2147483647; files = ( 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */, + C7BD897E1E312E5E095D3400 /* Pods_Runner.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -128,6 +140,7 @@ 331C80D6294CF71000263BE5 /* RunnerTests */, 33CC10EE2044A3C60003C045 /* Products */, D73912EC22F37F3D000D13A0 /* Frameworks */, + 46B086E826D3D29B49316EF3 /* Pods */, ); sourceTree = ""; }; @@ -176,9 +189,25 @@ path = Runner; sourceTree = ""; }; + 46B086E826D3D29B49316EF3 /* Pods */ = { + isa = PBXGroup; + children = ( + B396C4E4E3EE1B4BB1E67366 /* Pods-Runner.debug.xcconfig */, + 8C9F29817EA0A7110E900E6F /* Pods-Runner.release.xcconfig */, + D9747DDA8126822426B339BB /* Pods-Runner.profile.xcconfig */, + 30F06383B0F1A8CC493DE4C3 /* Pods-RunnerTests.debug.xcconfig */, + 769023E35970ADB12A4AC2D7 /* Pods-RunnerTests.release.xcconfig */, + 12FE65799543E39BACA69E1B /* Pods-RunnerTests.profile.xcconfig */, + ); + name = Pods; + path = Pods; + sourceTree = ""; + }; D73912EC22F37F3D000D13A0 /* Frameworks */ = { isa = PBXGroup; children = ( + BA982DFB3805F270CAE1EC57 /* Pods_Runner.framework */, + 52DA5BA5A5AAB1A3F8E8CACC /* Pods_RunnerTests.framework */, ); name = Frameworks; sourceTree = ""; @@ -190,6 +219,7 @@ isa = PBXNativeTarget; buildConfigurationList = 331C80DE294CF71000263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */; buildPhases = ( + 955AC6544A6B3BB98A250AB4 /* [CP] Check Pods Manifest.lock */, 331C80D1294CF70F00263BE5 /* Sources */, 331C80D2294CF70F00263BE5 /* Frameworks */, 331C80D3294CF70F00263BE5 /* Resources */, @@ -208,11 +238,13 @@ isa = PBXNativeTarget; buildConfigurationList = 33CC10FB2044A3C60003C045 /* Build configuration list for PBXNativeTarget "Runner" */; buildPhases = ( + F7F400CE023C878F98CF65C2 /* [CP] Check Pods Manifest.lock */, 33CC10E92044A3C60003C045 /* Sources */, 33CC10EA2044A3C60003C045 /* Frameworks */, 33CC10EB2044A3C60003C045 /* Resources */, 33CC110E2044A8840003C045 /* Bundle Framework */, 3399D490228B24CF009A79C7 /* ShellScript */, + 86DA14025ABF7D83888D3660 /* [CP] Embed Pods Frameworks */, ); buildRules = ( ); @@ -268,7 +300,7 @@ ); mainGroup = 33CC10E42044A3C60003C045; packageReferences = ( - 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage" */, + 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "FlutterGeneratedPluginSwiftPackage" */, ); productRefGroup = 33CC10EE2044A3C60003C045 /* Products */; projectDirPath = ""; @@ -339,6 +371,67 @@ shellPath = /bin/sh; shellScript = "\"$FLUTTER_ROOT\"/packages/flutter_tools/bin/macos_assemble.sh && touch Flutter/ephemeral/tripwire"; }; + 86DA14025ABF7D83888D3660 /* [CP] Embed Pods Frameworks */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Embed Pods Frameworks"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\"\n"; + showEnvVarsInLog = 0; + }; + 955AC6544A6B3BB98A250AB4 /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-RunnerTests-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + F7F400CE023C878F98CF65C2 /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-Runner-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -390,6 +483,7 @@ /* Begin XCBuildConfiguration section */ 331C80DB294CF71000263BE5 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 30F06383B0F1A8CC493DE4C3 /* Pods-RunnerTests.debug.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CURRENT_PROJECT_VERSION = 1; @@ -404,6 +498,7 @@ }; 331C80DC294CF71000263BE5 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 769023E35970ADB12A4AC2D7 /* Pods-RunnerTests.release.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CURRENT_PROJECT_VERSION = 1; @@ -418,6 +513,7 @@ }; 331C80DD294CF71000263BE5 /* Profile */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 12FE65799543E39BACA69E1B /* Pods-RunnerTests.profile.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CURRENT_PROJECT_VERSION = 1; @@ -712,7 +808,7 @@ /* End XCConfigurationList section */ /* Begin XCLocalSwiftPackageReference section */ - 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage" */ = { + 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "FlutterGeneratedPluginSwiftPackage" */ = { isa = XCLocalSwiftPackageReference; relativePath = Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage; }; diff --git a/app/macos/Runner.xcworkspace/contents.xcworkspacedata b/app/macos/Runner.xcworkspace/contents.xcworkspacedata index 1d526a16..21a3cc14 100644 --- a/app/macos/Runner.xcworkspace/contents.xcworkspacedata +++ b/app/macos/Runner.xcworkspace/contents.xcworkspacedata @@ -4,4 +4,7 @@ + + diff --git a/app/notes/read-aloud.md b/app/notes/read-aloud.md new file mode 100644 index 00000000..96ccb190 --- /dev/null +++ b/app/notes/read-aloud.md @@ -0,0 +1,55 @@ +# Read aloud + +An assistant reply has Read aloud beside Copy (same `IconButton` style). While +that reply reads it becomes Pause/Resume and Stop; one reply reads at a time +app-wide. The spoken word is highlighted (theme `primaryContainer`) in the +rendered reply and the transcript scrolls only when that word leaves the +viewport. Settings > Read aloud picks the voice (grouped by language, device +language first, a sample per voice), the rate (0.5x to 2.0x) and resets both; +they are saved on the device. + +## Structure + +- `lib/features/chat/speech/speech_engine.dart`: the `SpeechEngine` seam + (speak, pause, stop, setVoice, setRate, voices, start/pause/resume/ + progress/completion/cancel/error callbacks). +- `flutter_tts_speech_engine.dart`: the `flutter_tts` implementation. The rate + is a multiplier (1.0 = normal); Apple platforms get half of it on their 0..1 + scale. Linux reports unsupported, so the button and section are hidden. +- `speech_controller.dart`: the app's single reader (`speechController`, a fake + in tests via `debugSpeechController`). State changes notify the controller; + the spoken word is the separate `word` notifier. Stops on `hidden`, `paused` + or `detached` (not `inactive`, which is only a lost focus). +- `speech_document.dart`: Markdown to sentences, with each spoken character's + part and source offset. Code blocks (``` and ~~~) are "Code block.", tables + "Table." then cells; rules are skipped. +- `chat_markdown_speech.dart`: highlight spans and the reveal scroll for + `ChatMarkdownView`. +- Reader ids are `/` so a chat that closes or is swapped + for another stops only its own reply (`stopScope`). + +## Platform notes + +- Resume speaks the paused text again; Android, Apple and web plugins all + treat that as a resume. Android then reports progress against the remaining + suffix, which the controller maps back to the whole text. +- After a stop the controller waits (at most 400 ms) for the platform cancel, + so a late cancel cannot end the next reply and web speech is idle before + the next utterance. +- Apple and web `flutter_tts` cannot clear a chosen voice. After a voice was + set in a run, "System default" uses the device language's first voice. +- Platforms without word boundaries read without a highlight. +- `flutter_tts` has no Swift package, so macOS (and iOS) builds use CocoaPods + for it: `Podfile` and the `Pods-Runner` xcconfig includes are the generated + plugin integration. + +## Verification + +Tests (fake engine): `test/features/chat/speech/`, +`test/features/chat/ui/transcript/chat_read_aloud_button_test.dart`, +`chat_markdown_read_aloud_test.dart`, +`test/features/chat/conversation/chat_thread_read_aloud_test.dart`, +`test/features/settings/settings_read_aloud_section_test.dart`. + +Not verified on a device: real speech output, word boundaries and voices on +Android, macOS and browsers. diff --git a/app/pubspec.lock b/app/pubspec.lock index 6d19a1ef..2778d579 100644 --- a/app/pubspec.lock +++ b/app/pubspec.lock @@ -304,6 +304,14 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_tts: + dependency: "direct main" + description: + name: flutter_tts + sha256: ce5eb209b40e95f2f4a1397116c87ab2fcdff32257d04ed7a764e75894c03775 + url: "https://pub.dev" + source: hosted + version: "4.2.5" flutter_web_plugins: dependency: "direct main" description: flutter diff --git a/app/pubspec.yaml b/app/pubspec.yaml index c3d39b1a..66c0294a 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -48,6 +48,7 @@ dependencies: url_launcher: ^6.3.3 bonsoir: ^7.1.5 xterm: ^4.0.0 + flutter_tts: ^4.2.3 dev_dependencies: integration_test: diff --git a/app/test/features/chat/conversation/chat_thread_read_aloud_test.dart b/app/test/features/chat/conversation/chat_thread_read_aloud_test.dart new file mode 100644 index 00000000..487e1a10 --- /dev/null +++ b/app/test/features/chat/conversation/chat_thread_read_aloud_test.dart @@ -0,0 +1,74 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/conversation/chat_thread_view.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_read_aloud_button.dart'; + +import '../speech/fake_speech_engine.dart'; +import 'chat_thread_fixtures.dart'; + +Widget _thread(FakeChatGateway gateway, String sessionId) => MaterialApp( + theme: ThemeData(platform: TargetPlatform.android), + home: Scaffold( + body: ChatThreadView(gateway: gateway, sessionId: sessionId, onBack: () {}), + ), +); + +Future<(FakeSpeechEngine, SpeechController, FakeChatGateway)> _reading( + WidgetTester tester, +) async { + tester.view.physicalSize = const Size(800, 1600); + tester.view.devicePixelRatio = 1; + addTearDown(tester.view.reset); + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + final gateway = FakeChatGateway(); + addTearDown(gateway.close); + gateway.history = (_) async => chatPage([ + chatMessage( + 'a1', + 'assistant', + parts: [ + {'type': 'text', 'text': 'Hello from the agent.'}, + ], + ), + ]); + await tester.pumpWidget(_thread(gateway, 's1')); + await tester.pumpAndSettle(); + await tester.tap(find.byTooltip(chatReadAloudLabel)); + await tester.pump(); + engine.onStart!(); + expect(engine.spoken, ['Hello from the agent.']); + expect(speech.messageId, startsWith(speechScopePrefix('s1'))); + return (engine, speech, gateway); +} + +void main() { + testWidgets('switching the pane to another chat stops reading', ( + tester, + ) async { + final (engine, speech, gateway) = await _reading(tester); + await tester.pumpWidget(_thread(gateway, 's2')); + await tester.pumpAndSettle(); + expect(speech.state, SpeechState.idle); + expect(engine.calls, contains('stop')); + }); + + testWidgets('closing the chat stops reading', (tester) async { + final (engine, speech, _) = await _reading(tester); + await tester.pumpWidget(const MaterialApp(home: SizedBox())); + await tester.pumpAndSettle(); + expect(speech.state, SpeechState.idle); + expect(engine.calls.last, 'stop'); + }); + + testWidgets('closing a chat leaves another chat reading', (tester) async { + final (engine, speech, _) = await _reading(tester); + await speech.read(speechMessageId('elsewhere', 'b1'), ['Other reply']); + engine.onStart!(); + await tester.pumpWidget(const MaterialApp(home: SizedBox())); + await tester.pumpAndSettle(); + expect(speech.state, SpeechState.speaking); + expect(speech.messageId, speechMessageId('elsewhere', 'b1')); + }); +} diff --git a/app/test/features/chat/speech/fake_speech_engine.dart b/app/test/features/chat/speech/fake_speech_engine.dart new file mode 100644 index 00000000..060a4789 --- /dev/null +++ b/app/test/features/chat/speech/fake_speech_engine.dart @@ -0,0 +1,79 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; + +/// A scripted [SpeechEngine]: it records calls and a test fires the platform's +/// events. [stop] reports a cancel like every real platform does. +class FakeSpeechEngine extends SpeechEngine { + FakeSpeechEngine({this.supported = true, this.available = const []}); + + @override + final bool supported; + final List available; + final calls = []; + final spoken = []; + double? rate; + SpeechVoice? voice; + + @override + Future speak(String text) async { + calls.add('speak'); + spoken.add(text); + } + + @override + Future pause() async => calls.add('pause'); + + @override + Future stop() async { + calls.add('stop'); + onCancel?.call(); + } + + @override + Future setRate(double rate) async => this.rate = rate; + + @override + Future setVoice(SpeechVoice? voice) async => this.voice = voice; + + @override + Future> voices() async => available; + + /// Reports [word] (its first occurrence at or after [from]) being spoken. + void speakWord(String word, {int from = 0}) { + final text = spoken.last; + final start = text.indexOf(word, from); + expect(start, greaterThanOrEqualTo(0), reason: '"$word" is not spoken'); + onProgress?.call(text, start, start + word.length); + } +} + +/// Saved read-aloud settings held in memory, surviving a "reload". +class MemorySpeechPreferences implements SpeechPreferencesStore { + SpeechPreferences? saved; + + @override + Future load() async => saved; + + @override + Future save(SpeechPreferences preferences) async { + saved = preferences; + return true; + } +} + +/// Installs a reader over [engine] as the app's own for one test. +SpeechController installFakeSpeech( + FakeSpeechEngine engine, { + SpeechPreferencesStore? store, +}) { + final controller = SpeechController( + engine: engine, + store: store ?? MemorySpeechPreferences(), + ); + debugSpeechController = controller; + addTearDown(() { + debugSpeechController = null; + controller.dispose(); + }); + return controller; +} diff --git a/app/test/features/chat/speech/speech_controller_test.dart b/app/test/features/chat/speech/speech_controller_test.dart new file mode 100644 index 00000000..164b885a --- /dev/null +++ b/app/test/features/chat/speech/speech_controller_test.dart @@ -0,0 +1,168 @@ +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; + +import 'fake_speech_engine.dart'; + +void main() { + late FakeSpeechEngine engine; + late SpeechController speech; + + setUp(() { + engine = FakeSpeechEngine(); + }); + + SpeechController reader([SpeechPreferencesStore? store]) => + speech = SpeechController( + engine: engine, + store: store ?? MemorySpeechPreferences(), + ); + + tearDown(() => speech.dispose()); + + testWidgets('read, pause, resume and stop move through the states', ( + tester, + ) async { + reader(); + await speech.read('a', ['Hello **there**.']); + expect(speech.state, SpeechState.speaking); + expect(speech.messageId, 'a'); + expect(engine.spoken.last, 'Hello there.'); + engine.onStart!(); + + await speech.pause(); + expect(speech.state, SpeechState.paused); + expect(engine.calls.last, 'pause'); + engine.onPause!(); + expect(speech.state, SpeechState.paused); + + await speech.resume(); + expect(speech.state, SpeechState.speaking); + // Every platform resumes when the paused text is spoken again. + expect(engine.spoken.last, 'Hello there.'); + engine.onResume!(); + + await speech.stop(); + expect(speech.state, SpeechState.idle); + expect(speech.messageId, isNull); + expect(speech.document, isNull); + }); + + testWidgets('starting another reply stops the first', (tester) async { + reader(); + await speech.read('a', ['First reply']); + engine.onStart!(); + engine.speakWord('reply'); + await speech.read('b', ['Second reply']); + expect(engine.calls, ['speak', 'stop', 'speak']); + expect(speech.messageId, 'b'); + expect(speech.word.value, isNull); + // A late cancel of the first reply does not end the second. + engine.onCancel!(); + expect(speech.messageId, 'b'); + expect(speech.state, SpeechState.speaking); + }); + + testWidgets('progress sets the word and maps back to the source', ( + tester, + ) async { + reader(); + const first = 'Intro **bold** word'; + const second = '- item word'; + await speech.read('a', [first, second]); + engine.onStart!(); + engine.speakWord('bold'); + final document = speech.document!; + expect(speech.word.value, isNotNull); + expect(document.sourceRange(0, speech.word.value!), ( + start: first.indexOf('bold'), + end: first.indexOf('bold') + 4, + )); + expect(document.sourceRange(1, speech.word.value!), isNull); + + final spoken = engine.spoken.last; + engine.speakWord('word', from: spoken.lastIndexOf('item')); + expect(document.sourceRange(0, speech.word.value!), isNull); + expect(document.sourceRange(1, speech.word.value!), ( + start: second.indexOf('word'), + end: second.length, + )); + }); + + testWidgets('progress after a resumed suffix keeps document offsets', ( + tester, + ) async { + reader(); + await speech.read('a', ['one two three']); + engine.onStart!(); + // Android speaks the remaining text after a resume. + engine.onProgress!('three', 0, 5); + expect(speech.word.value, (start: 8, end: 13)); + }); + + testWidgets('completion and cancel clear the reply', (tester) async { + reader(); + await speech.read('a', ['Done soon']); + engine.onStart!(); + engine.speakWord('soon'); + engine.onCompletion!(); + expect(speech.state, SpeechState.idle); + expect(speech.messageId, isNull); + expect(speech.word.value, isNull); + + await speech.read('b', ['Cancelled']); + engine.onStart!(); + engine.onCancel!(); + expect(speech.state, SpeechState.idle); + expect(speech.messageId, isNull); + }); + + testWidgets('going to the background stops reading', (tester) async { + reader(); + await speech.read('a', ['Background']); + engine.onStart!(); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.inactive); + await tester.pump(); + expect(speech.state, SpeechState.speaking, reason: 'focus loss only'); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.hidden); + await tester.pump(); + expect(speech.state, SpeechState.idle); + expect(engine.calls.last, 'stop'); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.resumed); + }); + + testWidgets('stopScope stops only replies of that conversation', ( + tester, + ) async { + reader(); + await speech.read(speechMessageId('chat-1', 'm1'), ['Hello']); + engine.onStart!(); + await speech.stopScope('chat-2'); + expect(speech.state, SpeechState.speaking); + await speech.stopScope('chat-1'); + expect(speech.state, SpeechState.idle); + }); + + testWidgets('voice and rate are applied and persist across a reload', ( + tester, + ) async { + final store = MemorySpeechPreferences(); + reader(store); + const voice = SpeechVoice(name: 'Daniel', locale: 'en-GB'); + await speech.setVoice(voice); + await speech.setRate(1.5); + await speech.read('a', ['Hello']); + expect(engine.voice, voice); + expect(engine.rate, 1.5); + speech.dispose(); + + reader(store); + await speech.restore(); + expect(speech.voice, voice); + expect(speech.rate, 1.5); + await speech.reset(); + expect(speech.voice, isNull); + expect(speech.rate, defaultSpeechRate); + expect(store.saved, (rate: defaultSpeechRate, voice: null)); + }); +} diff --git a/app/test/features/chat/speech/speech_document_test.dart b/app/test/features/chat/speech/speech_document_test.dart new file mode 100644 index 00000000..0bf900de --- /dev/null +++ b/app/test/features/chat/speech/speech_document_test.dart @@ -0,0 +1,56 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech_document.dart'; + +void main() { + test('markdown is spoken as sentences and code is announced', () { + final result = markdownToSpeech( + '# Welcome\n\n- **First**\n- [Docs](https://example.com)\n\n```dart\nprint(1);\n```\n\nThat is all.', + ); + expect(result.text, 'Welcome. First. Docs. Code block. That is all.'); + expect(result.sources.length, result.text.length); + }); + + test('tilde fences and tables are summarised', () { + final result = markdownToSpeech( + '~~~\nsecret();\n~~~\n\n| A | B |\n|---|---|\n| 1 | 2 |', + ); + expect(result.text, 'Code block. Table. A. B. 1. 2'); + expect(result.text, isNot(contains('secret'))); + }); + + test('repeated words map to their own occurrence', () { + const markdown = 'First repeat\n\nSecond repeat'; + final result = markdownToSpeech(markdown); + final first = result.text.indexOf('repeat'); + final second = result.text.lastIndexOf('repeat'); + expect(result.sourceRange(0, (start: first, end: first + 6)), ( + start: 6, + end: 12, + )); + expect(result.sourceRange(0, (start: second, end: second + 6)), ( + start: markdown.lastIndexOf('repeat'), + end: markdown.length, + )); + }); + + test('text after a code block maps after the skipped code', () { + const markdown = 'Before repeat\n\n```dart\nrepeat();\n```\n\nAfter repeat'; + final result = markdownToSpeech(markdown); + final spoken = result.text.lastIndexOf('repeat'); + final range = result.sourceRange(0, (start: spoken, end: spoken + 6))!; + expect(range.start, markdown.lastIndexOf('repeat')); + }); + + test('parts are trimmed and keep their own sources', () { + final result = speechDocumentFor([' One part\n', '', 'Two part']); + expect(result.parts, ['One part', 'Two part']); + expect(result.text, 'One part. Two part'); + expect(result.partOf('\nTwo part '), 1); + final two = result.text.indexOf('Two'); + expect(result.sourceRange(1, (start: two, end: two + 3)), ( + start: 0, + end: 3, + )); + expect(result.sourceRange(0, (start: two, end: two + 3)), isNull); + }); +} diff --git a/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart new file mode 100644 index 00000000..29f8205d --- /dev/null +++ b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart @@ -0,0 +1,120 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart'; + +import '../../speech/fake_speech_engine.dart'; + +/// The text of every span drawn with the spoken-word highlight. +List _highlighted(WidgetTester tester, [Finder? within]) { + final fill = Theme.of( + tester.element(find.byType(ChatMarkdownView).first), + ).colorScheme.primaryContainer; + final spans = []; + final texts = within == null + ? find.byType(RichText) + : find.descendant(of: within, matching: find.byType(RichText)); + for (final rich in tester.widgetList(texts)) { + rich.text.visitChildren((span) { + if (span is TextSpan && + span.text != null && + span.style?.backgroundColor == fill) { + spans.add(span.text!); + } + return true; + }); + } + return spans; +} + +void main() { + testWidgets('the highlight follows progress and clears on stop', ( + tester, + ) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + const first = 'Alpha **beta** gamma'; + const second = '- delta beta'; + await tester.pumpWidget( + const MaterialApp( + home: Scaffold( + body: Column( + children: [ + ChatMarkdownView(text: first, messageId: 'm'), + ChatMarkdownView(text: second, messageId: 'm'), + ChatMarkdownView(text: first, messageId: 'other'), + ], + ), + ), + ), + ); + await speech.read('m', [first, second]); + engine.onStart!(); + await tester.pump(); + expect(_highlighted(tester), isEmpty, reason: 'no word reported yet'); + + engine.speakWord('beta'); + await tester.pump(); + expect(_highlighted(tester), ['beta']); + + engine.speakWord('gamma'); + await tester.pump(); + expect(_highlighted(tester), ['gamma']); + + // The second part's "beta" lights in the second view only. + engine.speakWord('beta', from: engine.spoken.last.indexOf('delta')); + await tester.pump(); + expect(_highlighted(tester), ['beta']); + final views = find.byType(ChatMarkdownView); + expect(_highlighted(tester, views.at(1)), ['beta']); + expect(_highlighted(tester, views.at(0)), isEmpty); + + await speech.stop(); + await tester.pump(); + expect(_highlighted(tester), isEmpty); + }); + + testWidgets('the transcript scrolls only when the word leaves the view', ( + tester, + ) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + final text = [ + for (var index = 0; index < 40; index += 1) 'Paragraph $index words.', + ].join('\n\n'); + final scroll = ScrollController(); + addTearDown(scroll.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SizedBox( + height: 300, + child: ListView( + controller: scroll, + children: [ChatMarkdownView(text: text, messageId: 'm')], + ), + ), + ), + ), + ); + await speech.read('m', [text]); + engine.onStart!(); + + engine.speakWord('Paragraph 2 '); + await tester.pumpAndSettle(); + expect(scroll.offset, 0, reason: 'the word is already in view'); + + engine.speakWord('Paragraph 30'); + await tester.pumpAndSettle(); + expect(scroll.offset, greaterThan(0)); + final viewport = tester.getRect(find.byType(ListView)); + final word = find.textContaining('Paragraph 30', findRichText: true); + expect(viewport.contains(tester.getRect(word).bottomLeft), isTrue); + final followed = scroll.offset; + + // A word still inside the view does not move the transcript. + final spoken = engine.spoken.last; + engine.speakWord('words', from: spoken.indexOf('Paragraph 30')); + await tester.pumpAndSettle(); + expect(scroll.offset, followed); + }); +} diff --git a/app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart b/app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart new file mode 100644 index 00000000..5ed3ea45 --- /dev/null +++ b/app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart @@ -0,0 +1,136 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/core/api_gateway.dart'; +import 'package:pipelines_app/features/chat/domain/chat_transcript.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_copy_gutter.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_read_aloud_button.dart'; + +import '../../speech/fake_speech_engine.dart'; + +Widget _host(Widget child) => MaterialApp( + home: Scaffold(body: Center(child: child)), +); + +JsonMap _answer(String id, String text) => transcriptRows([ + { + 'id': id, + 'role': 'assistant', + 'created_at': 1000, + 'parts': [ + {'type': 'text', 'text': text}, + ], + }, +]).single; + +final _read = find.byTooltip(chatReadAloudLabel); +final _pause = find.byTooltip(chatPauseReadingLabel); +final _resume = find.byTooltip(chatResumeReadingLabel); +final _stop = find.byTooltip(chatStopReadingLabel); + +void main() { + testWidgets('Read aloud becomes Pause and Stop, then Resume', (tester) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + await tester.pumpWidget( + _host(const ChatReadAloudButton(messageId: 'a', parts: ['Hello'])), + ); + expect(_read, findsOneWidget); + expect(_pause, findsNothing); + + await tester.tap(_read); + await tester.pump(); + expect(engine.spoken, ['Hello']); + expect(_read, findsNothing); + expect(_pause, findsOneWidget); + expect(_stop, findsOneWidget); + + await tester.tap(_pause); + await tester.pump(); + expect(speech.state, SpeechState.paused); + expect(_resume, findsOneWidget); + expect(_pause, findsNothing); + + await tester.tap(_resume); + await tester.pump(); + expect(speech.state, SpeechState.speaking); + expect(_pause, findsOneWidget); + + await tester.tap(_stop); + await tester.pump(); + expect(speech.state, SpeechState.idle); + expect(_read, findsOneWidget); + expect(_stop, findsNothing); + }); + + testWidgets('only one reply reads at a time', (tester) async { + final engine = FakeSpeechEngine(); + installFakeSpeech(engine); + await tester.pumpWidget( + _host( + const Column( + mainAxisSize: MainAxisSize.min, + children: [ + ChatReadAloudButton(messageId: 'a', parts: ['First']), + ChatReadAloudButton(messageId: 'b', parts: ['Second']), + ], + ), + ), + ); + await tester.tap(_read.first); + await tester.pump(); + expect(_stop, findsOneWidget); + await tester.tap(_read); + await tester.pump(); + expect(engine.spoken, ['First', 'Second']); + expect(engine.calls, ['speak', 'stop', 'speak']); + expect(_stop, findsOneWidget); + expect(_read, findsOneWidget); + }); + + testWidgets('hidden where the platform cannot speak', (tester) async { + installFakeSpeech(FakeSpeechEngine(supported: false)); + await tester.pumpWidget( + _host(const ChatReadAloudButton(messageId: 'a', parts: ['Hello'])), + ); + expect(_read, findsNothing); + }); + + testWidgets('an answer offers Read aloud beside Copy, footer and gutter', ( + tester, + ) async { + installFakeSpeech(FakeSpeechEngine()); + Future pump(double gutter) => tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ListView( + children: [ + ChatCopyGutter( + width: gutter, + child: ChatMessageView(row: _answer('a1', 'The answer')), + ), + ], + ), + ), + ), + ); + final copy = find.byKey(const Key('chat-message-copy')); + + await pump(0); + expect(_read, findsOneWidget); + expect(tester.getCenter(_read).dy, tester.getCenter(copy).dy); + expect(tester.getCenter(_read).dx, greaterThan(tester.getCenter(copy).dx)); + + // The 48 px gutter stacks the buttons; while reading, Pause and Stop sit + // above Copy and nothing overflows. + await pump(48); + expect(tester.getCenter(_read).dx, tester.getCenter(copy).dx); + expect(tester.getCenter(_read).dy, lessThan(tester.getCenter(copy).dy)); + await tester.tap(_read); + await tester.pump(); + expect(tester.takeException(), isNull); + expect(tester.getCenter(_stop).dy, lessThan(tester.getCenter(copy).dy)); + expect(tester.getCenter(_pause).dy, lessThan(tester.getCenter(_stop).dy)); + }); +} diff --git a/app/test/features/settings/settings_read_aloud_section_test.dart b/app/test/features/settings/settings_read_aloud_section_test.dart new file mode 100644 index 00000000..7537bd67 --- /dev/null +++ b/app/test/features/settings/settings_read_aloud_section_test.dart @@ -0,0 +1,148 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; +import 'package:pipelines_app/features/settings/settings_read_aloud_section.dart'; +import 'package:pipelines_app/features/settings/settings_voice_picker.dart'; + +import '../chat/speech/fake_speech_engine.dart'; + +const _daniel = SpeechVoice(name: 'Daniel', locale: 'en-GB'); +const _samantha = SpeechVoice(name: 'Samantha', locale: 'en-US'); +const _thomas = SpeechVoice(name: 'Thomas', locale: 'fr-FR'); +const _anna = SpeechVoice(name: 'Anna', locale: 'de-DE'); + +final _voiceRow = find.byKey(const Key('settings-read-aloud-voice')); +final _slider = find.byKey(const Key('settings-read-aloud-rate')); +final _reset = find.byKey(const Key('settings-read-aloud-reset')); +final _sample = find.byKey(const Key('settings-read-aloud-sample')); + +late FakeSpeechEngine _engine; + +/// A fresh reader over [store], as after an app reload, and its section. +Future _open( + WidgetTester tester, + MemorySpeechPreferences store, +) async { + _engine = FakeSpeechEngine(available: [_samantha, _thomas, _anna, _daniel]); + final speech = SpeechController(engine: _engine, store: store); + addTearDown(speech.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ListView( + children: [ + SettingsReadAloudSection(key: UniqueKey(), controller: speech), + ], + ), + ), + ), + ); + await tester.pumpAndSettle(); + return speech; +} + +void main() { + testWidgets('a picked voice persists across a reload', (tester) async { + tester.platformDispatcher.localeTestValue = const Locale('fr', 'FR'); + addTearDown(tester.platformDispatcher.clearLocaleTestValue); + final store = MemorySpeechPreferences(); + await _open(tester, store); + expect(find.text('System default'), findsOneWidget); + + await tester.tap(_voiceRow); + await tester.pumpAndSettle(); + // Grouped by language, the device's language first. + final french = tester.getTopLeft(find.text('French')).dy; + final english = tester.getTopLeft(find.text('English')).dy; + final german = tester.getTopLeft(find.text('German')).dy; + expect(french, lessThan(english)); + expect(english, lessThan(german)); + expect( + tester.getTopLeft(find.text('Daniel')).dy, + lessThan(tester.getTopLeft(find.text('Samantha')).dy), + ); + + await tester.tap(find.text('Daniel')); + await tester.pumpAndSettle(); + expect(find.text('Daniel · en-GB'), findsOneWidget); + expect(store.saved?.voice, _daniel); + + final reloaded = await _open(tester, store); + expect(reloaded.voice, _daniel); + expect(find.text('Daniel · en-GB'), findsOneWidget); + await reloaded.read('m', ['Hello']); + expect(_engine.voice, _daniel); + }); + + testWidgets('the rate persists across a reload', (tester) async { + final store = MemorySpeechPreferences(); + final speech = await _open(tester, store); + expect(find.text('Speaking rate 1.0×'), findsOneWidget); + await tester.drag(_slider, const Offset(400, 0)); + await tester.pumpAndSettle(); + expect(speech.rate, greaterThan(defaultSpeechRate)); + expect(store.saved?.rate, speech.rate); + + final reloaded = await _open(tester, store); + expect(reloaded.rate, store.saved!.rate); + expect( + find.text('Speaking rate ${reloaded.rate.toStringAsFixed(1)}×'), + findsOneWidget, + ); + await reloaded.read('m', ['Hello']); + expect(_engine.rate, reloaded.rate); + }); + + testWidgets('reset restores the system voice and normal rate', ( + tester, + ) async { + final store = MemorySpeechPreferences() + ..saved = (rate: 1.5, voice: _thomas); + final speech = await _open(tester, store); + expect(find.text('Thomas · fr-FR'), findsOneWidget); + expect(find.text('Speaking rate 1.5×'), findsOneWidget); + + await tester.ensureVisible(_reset); + await tester.tap(_reset); + await tester.pumpAndSettle(); + expect(speech.voice, isNull); + expect(speech.rate, defaultSpeechRate); + expect(store.saved, (rate: defaultSpeechRate, voice: null)); + expect(find.text('System default'), findsOneWidget); + expect(find.text('Speaking rate 1.0×'), findsOneWidget); + expect(tester.widget(_reset).onPressed, isNull); + }); + + testWidgets('samples play through the engine', (tester) async { + final store = MemorySpeechPreferences()..saved = (rate: 1, voice: _anna); + final speech = await _open(tester, store); + await tester.ensureVisible(_sample); + await tester.tap(_sample); + await tester.pump(); + expect(_engine.spoken.single, contains('read aloud')); + expect(_engine.voice, _anna); + expect(find.text('Stop sample'), findsOneWidget); + await tester.tap(_sample); + await tester.pump(); + expect(speech.state, SpeechState.idle); + expect(find.text('Play sample'), findsOneWidget); + + // Each voice in the picker has its own sample; it does not choose it. + await tester.tap(_voiceRow); + await tester.pumpAndSettle(); + await tester.tap(find.byTooltip('Play sample of Samantha')); + await tester.pump(); + expect(_engine.voice, _samantha); + expect(_engine.spoken, hasLength(2)); + await tester.tap(find.text('Cancel')); + await tester.pumpAndSettle(); + expect(speech.voice, _anna); + expect(speech.state, SpeechState.idle, reason: 'closing stops the sample'); + }); + + test('voices group by language, device language first', () { + final groups = groupSpeechVoices([_samantha, _anna, _daniel], 'de'); + expect(groups.map((group) => group.language), ['de', 'en']); + expect(groups.last.voices, [_daniel, _samantha]); + }); +} diff --git a/app/tool/capture/README.md b/app/tool/capture/README.md index a701b2e8..7a9609de 100644 --- a/app/tool/capture/README.md +++ b/app/tool/capture/README.md @@ -33,6 +33,10 @@ URL parameters (the kit sets them from `open=` and the options): or script's `params`) adds fixture options such as `chats=2&rich=1`. Use `route=/settings` to open Settings and then click `Server health`; the fixture channel returns `server_stats` automatically. +`speech=fake` (with `speechStep=`, default 600) reads replies aloud with a +scripted engine that reports one word per step, since headless Chrome has no +voices: Read aloud, Pause/Resume/Stop, the word highlight and the Settings +voice list can then be captured. Without `theme`, the app follows the browser's colour scheme. `--theme` also sets that colour scheme through Chrome emulation. diff --git a/app/tool/probe_speech_fixture.dart b/app/tool/probe_speech_fixture.dart new file mode 100644 index 00000000..3e1283a0 --- /dev/null +++ b/app/tool/probe_speech_fixture.dart @@ -0,0 +1,92 @@ +import 'dart:async'; + +import 'package:pipelines_app/features/chat/speech/speech.dart'; + +/// Opt-in (`?speech=fake`) scripted reader for captures: headless browsers +/// have no speech voices, so this engine "speaks" one word every +/// `?speechStep=` ms (default 600) and reports each as progress, which drives +/// the real controller, buttons, highlight and reveal scrolling. +void installSpeechFixture(Map params) { + if (params['speech'] != 'fake') return; + final step = int.tryParse(params['speechStep'] ?? '') ?? 600; + debugSpeechController = SpeechController( + engine: _ScriptedSpeechEngine(Duration(milliseconds: step)), + ); +} + +class _ScriptedSpeechEngine extends SpeechEngine { + _ScriptedSpeechEngine(this.step); + + final Duration step; + Timer? _timer; + String _text = ''; + List _words = const []; + var _next = 0; + var _paused = false; + + @override + bool get supported => true; + + @override + Future speak(String text) async { + if (_paused && text == _text) { + _paused = false; + onResume?.call(); + _run(); + return; + } + _text = text; + _words = RegExp(r'\S+').allMatches(text).toList(); + _next = 0; + _paused = false; + Timer.run(() => onStart?.call()); + _run(); + } + + void _run() { + _timer?.cancel(); + _emit(); + _timer = Timer.periodic(step, (_) => _emit()); + } + + void _emit() { + if (_next >= _words.length) { + _timer?.cancel(); + onCompletion?.call(); + return; + } + final word = _words[_next++]; + onProgress?.call(_text, word.start, word.end); + } + + @override + Future pause() async { + _timer?.cancel(); + _paused = true; + if (_next > 0) _next -= 1; + onPause?.call(); + } + + @override + Future stop() async { + _timer?.cancel(); + _paused = false; + onCancel?.call(); + } + + @override + Future setRate(double rate) async {} + + @override + Future setVoice(SpeechVoice? voice) async {} + + @override + Future> voices() async => const [ + SpeechVoice(name: 'Samantha', locale: 'en-US'), + SpeechVoice(name: 'Daniel', locale: 'en-GB'), + SpeechVoice(name: 'Karen', locale: 'en-AU'), + SpeechVoice(name: 'Thomas', locale: 'fr-FR'), + SpeechVoice(name: 'Anna', locale: 'de-DE'), + SpeechVoice(name: 'Kyoko', locale: 'ja-JP'), + ]; +} diff --git a/app/tool/search_selection_probe_entry.dart b/app/tool/search_selection_probe_entry.dart index 47a8bdc1..b25a7e7a 100644 --- a/app/tool/search_selection_probe_entry.dart +++ b/app/tool/search_selection_probe_entry.dart @@ -47,6 +47,7 @@ import 'perf/perf_fixture.dart'; import 'probe_folder_fixture.dart'; import 'probe_files_fixture.dart'; import 'probe_usage_fixture.dart'; +import 'probe_speech_fixture.dart'; /// The run every run-scoped hit names. Its recorded sequences are deliberately /// not its row indexes (plan 5, build 2, plan 1), exactly like the unit @@ -786,6 +787,8 @@ class _FixtureSink implements WebSocketSink { /// `?terminal=float` opens them at launch (tool/perf/terminal_fixture.dart); /// `?windows=N` opens N floating probe windows whose content opens every /// kind of popup and a pushed page (tool/capture/window_probe.dart); +/// `?speech=fake` reads replies aloud with a scripted engine, one word per +/// `?speechStep=` ms (tool/probe_speech_fixture.dart); /// `?theme=dark` / `?theme=light` pins the platform brightness the app's /// `ThemeMode.system` follows (omitted: the browser's color scheme). /// @@ -806,6 +809,7 @@ Future main() async { return; } configurePerfFixture(params); + installSpeechFixture(params); if (params['files'] == '1') { probeMissingRememberedFolder = params['filesMissing'] == '1'; installFilesFixture();