diff --git a/app/android/app/src/main/AndroidManifest.xml b/app/android/app/src/main/AndroidManifest.xml index 2e2bdf08..844805a2 100644 --- a/app/android/app/src/main/AndroidManifest.xml +++ b/app/android/app/src/main/AndroidManifest.xml @@ -115,5 +115,9 @@ + + + + diff --git a/app/ios/Flutter/Debug.xcconfig b/app/ios/Flutter/Debug.xcconfig index 592ceee8..ec97fc6f 100644 --- a/app/ios/Flutter/Debug.xcconfig +++ b/app/ios/Flutter/Debug.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" #include "Generated.xcconfig" diff --git a/app/ios/Flutter/Release.xcconfig b/app/ios/Flutter/Release.xcconfig index 592ceee8..c4855bfe 100644 --- a/app/ios/Flutter/Release.xcconfig +++ b/app/ios/Flutter/Release.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" #include "Generated.xcconfig" diff --git a/app/ios/Podfile b/app/ios/Podfile new file mode 100644 index 00000000..3f878a59 --- /dev/null +++ b/app/ios/Podfile @@ -0,0 +1,43 @@ +# Match the Runner's IPHONEOS_DEPLOYMENT_TARGET. +platform :ios, '15.0' + +# CocoaPods analytics sends network stats synchronously affecting flutter build latency. +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure flutter pub get is executed first" + end + + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Generated.xcconfig, then run flutter pub get" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_ios_podfile_setup + +target 'Runner' do + use_frameworks! + + flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_ios_build_settings(target) + end +end diff --git a/app/lib/features/chat/conversation/chat_thread_view.dart b/app/lib/features/chat/conversation/chat_thread_view.dart index f1c248e8..63cbe05e 100644 --- a/app/lib/features/chat/conversation/chat_thread_view.dart +++ b/app/lib/features/chat/conversation/chat_thread_view.dart @@ -63,6 +63,7 @@ import '../../../ui/layout/app_sheet_header.dart'; import '../../../ui/layout/wheel_scroll_forwarding.dart'; import 'chat_reading_width.dart'; import '../ui/transcript/chat_copy_gutter.dart'; +import '../speech/speech.dart'; import 'chat_pane_presentation.dart'; import 'chat_thread_header.dart'; import 'chat_transcript_anchor.dart'; @@ -655,6 +656,12 @@ class _ChatThreadViewState extends State { @override void didUpdateWidget(covariant ChatThreadView oldWidget) { super.didUpdateWidget(oldWidget); + if (oldWidget.sessionId != widget.sessionId || + oldWidget.gateway != widget.gateway) { + // A pane reused for another conversation stops reading the previous + // one's reply, so neither its audio nor its highlight carries over. + unawaited(speechController.stopScope(oldWidget.sessionId)); + } if (!identical(oldWidget.paneCommands, widget.paneCommands)) { oldWidget.paneCommands?.removeListener(_takeCommands); oldWidget.paneCommands?.setFinished(oldWidget.sessionId, false); @@ -751,6 +758,9 @@ class _ChatThreadViewState extends State { @override void dispose() { + // Leaving or closing the conversation stops its reply reading aloud; a + // reply another pane is reading carries on. + unawaited(speechController.stopScope(widget.sessionId)); _persistPosition(); _saveTranscript(widget.transcripts, widget.sessionId); widget.paneCommands?.removeListener(_takeCommands); @@ -1715,7 +1725,16 @@ class _ChatThreadViewState extends State { [delivery, onRemove != null, _images, _controller, _composer, _shellRuns], () => ChatMessageView( row: row, - textBuilder: chatMarkdownText, + speechScope: widget.sessionId, + // The same reader id as the reply's Read aloud button, so the spoken + // word is highlighted in this reply's text. + textBuilder: (context, text) => ChatMarkdownView( + text: text, + messageId: speechMessageId( + widget.sessionId, + '${row['key'] ?? row['id'] ?? ''}', + ), + ), footer: [ ...chips, if (delivery != null) diff --git a/app/lib/features/chat/speech/flutter_tts_speech_engine.dart b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart new file mode 100644 index 00000000..2ab2700e --- /dev/null +++ b/app/lib/features/chat/speech/flutter_tts_speech_engine.dart @@ -0,0 +1,112 @@ +import 'package:flutter/foundation.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_tts/flutter_tts.dart'; + +import 'speech_engine.dart'; + +/// [SpeechEngine] over the `flutter_tts` plugin (Android, Apple, web). +/// The plugin is created on first use so merely building the UI opens no +/// platform channel. +class FlutterTtsSpeechEngine extends SpeechEngine { + FlutterTts? _plugin; + SpeechVoice? _applied; + + @override + bool get supported => + kIsWeb || + switch (defaultTargetPlatform) { + TargetPlatform.android || + TargetPlatform.iOS || + TargetPlatform.macOS || + TargetPlatform.windows => true, + TargetPlatform.linux || TargetPlatform.fuchsia => false, + }; + + /// AVSpeechSynthesizer's normal rate is 0.5 on a 0..1 scale; Android's + /// TextToSpeech and the Web Speech API use 1.0 as normal. + bool get _appleScale => + !kIsWeb && + (defaultTargetPlatform == TargetPlatform.iOS || + defaultTargetPlatform == TargetPlatform.macOS); + + FlutterTts get _tts => _plugin ??= _create(); + + FlutterTts _create() { + final tts = FlutterTts(); + tts.setStartHandler(() => onStart?.call()); + tts.setPauseHandler(() => onPause?.call()); + tts.setContinueHandler(() => onResume?.call()); + tts.setCompletionHandler(() => onCompletion?.call()); + tts.setCancelHandler(() => onCancel?.call()); + tts.setErrorHandler((message) => onError?.call('$message')); + tts.setProgressHandler( + (utterance, start, end, _) => onProgress?.call(utterance, start, end), + ); + return tts; + } + + @override + Future speak(String text) async => _tts.speak(text); + + @override + Future pause() async => _tts.pause(); + + @override + Future stop() async => _tts.stop(); + + @override + Future setRate(double rate) async => + _tts.setSpeechRate(_appleScale ? (rate * 0.5).clamp(0, 1) : rate); + + @override + Future setVoice(SpeechVoice? voice) async { + if (voice == _applied) return; + if (voice != null) { + await _tts.setVoice({'name': voice.name, 'locale': voice.locale}); + _applied = voice; + return; + } + if (!kIsWeb && defaultTargetPlatform == TargetPlatform.android) { + await _tts.clearVoice(); + _applied = null; + return; + } + // Apple and web plugins cannot clear a chosen voice. After one was chosen + // in this run (a sample, or a setting that was reset) the device + // language's first voice stands in for the system default. + final device = PlatformDispatcher.instance.locale.languageCode; + final fallback = (await voices()) + .where((item) => item.language == device.toLowerCase()) + .firstOrNull; + if (fallback == null) return; + await _tts.setVoice({'name': fallback.name, 'locale': fallback.locale}); + _applied = null; + } + + @override + Future> voices() async { + var voices = await _readVoices(); + // Browsers load their voice list asynchronously after the first request. + for (var attempt = 0; kIsWeb && voices.isEmpty && attempt < 4; attempt++) { + await Future.delayed(const Duration(milliseconds: 250)); + voices = await _readVoices(); + } + return voices; + } + + Future> _readVoices() async { + Object? raw; + try { + raw = await _tts.getVoices; + } on PlatformException { + return const []; + } on MissingPluginException { + return const []; + } + return [ + for (final item in raw is List ? raw : const []) + if (item is Map && item['name'] != null && item['locale'] != null) + SpeechVoice(name: '${item['name']}', locale: '${item['locale']}'), + ]; + } +} diff --git a/app/lib/features/chat/speech/speech.dart b/app/lib/features/chat/speech/speech.dart new file mode 100644 index 00000000..7b73f4c7 --- /dev/null +++ b/app/lib/features/chat/speech/speech.dart @@ -0,0 +1,4 @@ +export 'speech_controller.dart'; +export 'speech_document.dart'; +export 'speech_engine.dart'; +export 'speech_preferences.dart'; diff --git a/app/lib/features/chat/speech/speech_controller.dart b/app/lib/features/chat/speech/speech_controller.dart new file mode 100644 index 00000000..a60d3d19 --- /dev/null +++ b/app/lib/features/chat/speech/speech_controller.dart @@ -0,0 +1,265 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +import 'flutter_tts_speech_engine.dart'; +import 'speech_document.dart'; +import 'speech_engine.dart'; +import 'speech_preferences.dart'; + +enum SpeechState { idle, speaking, paused } + +/// The app-wide reader: one reply speaks at a time. It notifies on state +/// changes; the spoken word moves on the separate, frequent [word]. +class SpeechController extends ChangeNotifier with WidgetsBindingObserver { + SpeechController({SpeechEngine? engine, SpeechPreferencesStore? store}) + : engine = engine ?? FlutterTtsSpeechEngine(), + _store = store ?? const SharedSpeechPreferencesStore() { + WidgetsBinding.instance.addObserver(this); + this.engine + ..onStart = _started + ..onPause = (() => _set(SpeechState.paused)) + ..onResume = (() => _set(SpeechState.speaking)) + ..onCompletion = _ended + ..onCancel = _ended + ..onError = _failed + ..onProgress = _progress; + } + + final SpeechEngine engine; + final SpeechPreferencesStore _store; + + SpeechState _state = SpeechState.idle; + String? _messageId; + SpeechDocument? _document; + Completer? _stopping; + Future? _restoring; + var _awaitingStart = false; + + double _rate = defaultSpeechRate; + SpeechVoice? _voice; + bool saveFailed = false; + + /// The range of the word being spoken in [document]'s text, or null. + final word = ValueNotifier(null); + + bool get supported => engine.supported; + SpeechState get state => _state; + String? get messageId => _messageId; + SpeechDocument? get document => _document; + double get rate => _rate; + SpeechVoice? get voice => _voice; + bool get isDefault => _rate == defaultSpeechRate && _voice == null; + + /// Loads the saved voice and rate once. + Future restore() => _restoring ??= _load(); + + Future _load() async { + final saved = await _store.load(); + if (saved == null) return; + _rate = saved.rate.clamp(minSpeechRate, maxSpeechRate); + _voice = saved.voice; + notifyListeners(); + } + + /// Reads reply [id] made of markdown [parts], stopping any other reading. + Future read(String id, List parts) async { + final document = speechDocumentFor(parts); + await stop(); + await restore(); + if (document.text.isEmpty) return; + await _speak(id, document, _voice); + } + + /// Speaks a short sample in [voice] (or the chosen one) at the chosen rate. + Future sample([SpeechVoice? voice]) async { + await stop(); + await restore(); + await _speak( + speechSampleId, + speechDocumentFor(['This is how replies will sound when read aloud.']), + voice ?? _voice, + ); + } + + Future _speak(String id, SpeechDocument document, SpeechVoice? voice) { + _messageId = id; + _document = document; + _awaitingStart = true; + word.value = null; + _set(SpeechState.speaking); + return _guard(() async { + await engine.setRate(_rate); + await engine.setVoice(voice); + await engine.speak(document.text); + }); + } + + Future pause() async { + if (_state != SpeechState.speaking) return; + _set(SpeechState.paused); + await _guard(engine.pause); + } + + /// Continues a paused reply where it stopped; every platform resumes when + /// the paused utterance's text is spoken again. + Future resume() async { + final document = _document; + if (_state != SpeechState.paused || document == null) return; + _set(SpeechState.speaking); + await _guard(() => engine.speak(document.text)); + } + + Future stop() async { + if (_state == SpeechState.idle) return; + final stopping = _stopping = Completer(); + await _guard(engine.stop); + // Wait briefly for the platform's cancel so it cannot end the next reply + // and so web speech is idle before the next utterance is queued. + await stopping.future.timeout( + const Duration(milliseconds: 400), + onTimeout: () {}, + ); + if (identical(_stopping, stopping)) _stopping = null; + _clear(); + } + + /// Stops reading when the reply being read has an id under [scope]. + Future stopScope(String scope) async { + if (_messageId?.startsWith(speechScopePrefix(scope)) ?? false) { + await stop(); + } + } + + /// Sets the rate for the next reading; [save] false previews a slider drag. + Future setRate(double value, {bool save = true}) async { + _rate = value.clamp(minSpeechRate, maxSpeechRate); + notifyListeners(); + if (save) await _save(); + } + + Future setVoice(SpeechVoice? value) async { + _voice = value; + notifyListeners(); + await _save(); + } + + Future reset() async { + _rate = defaultSpeechRate; + _voice = null; + notifyListeners(); + await _save(); + } + + Future _save() async { + saveFailed = !await _store.save((rate: _rate, voice: _voice)); + notifyListeners(); + } + + Future> voices() => engine.voices(); + + void _started() { + _awaitingStart = false; + if (_messageId != null) _set(SpeechState.speaking); + } + + void _ended() { + final stopping = _stopping; + if (stopping != null) { + if (!stopping.isCompleted) stopping.complete(); + return; + } + // A late cancel from the previous utterance must not end the new one. + if (_awaitingStart) return; + _clear(); + } + + void _failed(String message) { + if (message == 'interrupted' || message == 'canceled') return _ended(); + final stopping = _stopping; + if (stopping != null && !stopping.isCompleted) stopping.complete(); + _clear(); + } + + void _progress(String utterance, int start, int end) { + final document = _document; + if (document == null || _state == SpeechState.idle) return; + // After a resume a platform may speak only the remaining suffix. + final base = document.text.endsWith(utterance) + ? document.text.length - utterance.length + : 0; + _awaitingStart = false; + word.value = (start: start + base, end: end + base); + } + + Future _guard(Future Function() action) async { + try { + await action(); + } catch (error) { + debugPrint('Read aloud failed: $error'); + _clear(); + } + } + + void _set(SpeechState value) { + if (_state == value) return; + _state = value; + notifyListeners(); + } + + void _clear() { + _awaitingStart = false; + _messageId = null; + _document = null; + word.value = null; + _state = SpeechState.idle; + notifyListeners(); + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + // Inactive is only a lost focus (a desktop window behind another, the + // Android shade); hidden and paused mean the app is in the background. + if (state == AppLifecycleState.hidden || + state == AppLifecycleState.paused || + state == AppLifecycleState.detached) { + unawaited(stop()); + } + } + + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + engine + ..onStart = null + ..onPause = null + ..onResume = null + ..onCompletion = null + ..onCancel = null + ..onError = null + ..onProgress = null; + if (_state != SpeechState.idle) unawaited(engine.stop().catchError((_) {})); + word.dispose(); + super.dispose(); + } +} + +/// The reader's id for the Settings voice sample. +const speechSampleId = 'settings-sample'; + +/// The id prefix for replies of one conversation (see [speechMessageId]). +String speechScopePrefix(String scope) => '$scope/'; + +/// A reply's id for the reader, unique across conversations. +String speechMessageId(String scope, String rowId) => '$scope/$rowId'; + +SpeechController? _speechController; + +/// The app's single reader. +SpeechController get speechController => + _speechController ??= SpeechController(); + +/// Replaces the app's reader (a fake engine in tests and capture fixtures); +/// null restores the platform one. +set debugSpeechController(SpeechController? controller) => + _speechController = controller; diff --git a/app/lib/features/chat/speech/speech_document.dart b/app/lib/features/chat/speech/speech_document.dart new file mode 100644 index 00000000..8623367e --- /dev/null +++ b/app/lib/features/chat/speech/speech_document.dart @@ -0,0 +1,171 @@ +import '../ui/transcript/chat_markdown.dart'; +import '../ui/transcript/chat_markdown_cache.dart'; + +typedef SpeechRange = ({int start, int end}); + +/// Where one spoken character came from: an offset into one markdown part. +typedef SpeechSource = ({int part, int offset}); + +/// A reply as spoken text, with the markdown source of every spoken character. +/// A reply can have several text parts; each is rendered by its own view, so +/// sources name the part as well as the offset within its trimmed text. +class SpeechDocument { + const SpeechDocument({ + required this.text, + required this.parts, + required this.sources, + }); + + final String text; + + /// The trimmed markdown of each part, in reading order. + final List parts; + final List sources; + + /// The index of the part whose markdown is [markdown], or -1. + int partOf(String markdown) => parts.indexOf(markdown.trim()); + + /// The source range in [part] that the spoken [range] covers, or null when + /// the spoken text there came from another part. + SpeechRange? sourceRange(int part, SpeechRange range) { + int? start; + int? end; + final last = range.end.clamp(0, sources.length); + for (var index = range.start.clamp(0, last); index < last; index += 1) { + final source = sources[index]; + if (source.part != part) continue; + if (start == null || source.offset < start) start = source.offset; + if (end == null || source.offset + 1 > end) end = source.offset + 1; + } + return start == null ? null : (start: start, end: end!); + } +} + +SpeechDocument markdownToSpeech(String markdown) => + speechDocumentFor([markdown]); + +/// Converts reply parts to natural speech: markup is dropped, blocks become +/// sentences, and a fenced code block is announced as "Code block." rather +/// than read out. Visible characters are located in order in the source, so +/// repeated words map to their own occurrence. +SpeechDocument speechDocumentFor(List markdownParts) { + final parts = [ + for (final part in markdownParts) + if (part.trim().isNotEmpty) part.trim(), + ]; + final builder = _SpeechBuilder(); + for (var index = 0; index < parts.length; index += 1) { + builder.addPart(index, parts[index]); + } + return SpeechDocument( + text: builder.spoken.toString(), + parts: parts, + sources: builder.sources, + ); +} + +class _SpeechBuilder { + final spoken = StringBuffer(); + final sources = []; + String _markdown = ''; + var _part = 0; + var _cursor = 0; + + void addPart(int part, String markdown) { + _part = part; + _markdown = markdown; + _cursor = 0; + for (final block in chatMarkdownCache.blocksFor(markdown)) { + switch (block) { + case MarkdownHeading(text: final text): + _visible(text); + case MarkdownParagraph(text: final text): + _visible(text); + case MarkdownQuote(lines: final lines): + lines.where((line) => line.trim().isNotEmpty).forEach(_visible); + case MarkdownList(items: final items): + items.forEach(_visible); + case MarkdownTable(header: final header, rows: final rows): + _phrase('Table.', _cursor); + [...header, ...rows.expand((row) => row)].forEach(_visible); + case MarkdownCode(code: final code): + _skipCode(code); + case MarkdownRule(): + break; + } + } + } + + /// Ends the previous sentence so blocks are not read as one run-on line. + void _sentenceBreak(int at) { + if (spoken.isEmpty) return; + if (!RegExp(r'[.!?:;]$').hasMatch(spoken.toString())) _write('.', at); + _write(' ', at); + } + + void _write(String value, int at) { + spoken.write(value); + for (var index = 0; index < value.length; index += 1) { + sources.add((part: _part, offset: at)); + } + } + + void _phrase(String value, int at) { + _sentenceBreak(at); + _write(value, at); + } + + void _visible(String blockText) { + final value = _plain(blockText).trim(); + if (value.isEmpty) return; + final start = _markdown.indexOf(blockText, _cursor); + final blockStart = start < 0 ? _cursor : start; + final blockEnd = start < 0 ? _markdown.length : start + blockText.length; + _sentenceBreak(blockStart); + var search = blockStart; + for (var index = 0; index < value.length; index += 1) { + final character = value[index]; + final found = search >= blockEnd + ? -1 + : _markdown.indexOf(character, search); + final inBlock = found >= 0 && found < blockEnd; + spoken.write(character); + sources.add((part: _part, offset: inBlock ? found : search)); + if (inBlock) search = found + 1; + } + _cursor = start < 0 ? search : blockEnd; + } + + void _skipCode(String code) { + final span = fencedCodeSpan(_markdown, _cursor, code); + _phrase('Code block.', span.start); + _cursor = span.end; + } +} + +/// The source range of the fenced code block holding [code] that starts at or +/// after [cursor] (``` or ~~~ fences); an unclosed fence runs to its code's end. +SpeechRange fencedCodeSpan(String source, int cursor, String code) { + int find(String marker) => source.indexOf(marker, cursor); + final fences = [find('```'), find('~~~')].where((index) => index >= 0); + if (fences.isEmpty) return (start: cursor, end: cursor); + final opening = fences.reduce((a, b) => a < b ? a : b); + final marker = source.substring(opening, opening + 3); + final closing = source.indexOf(marker, opening + 3); + if (closing >= 0) return (start: opening, end: closing + 3); + final codeStart = source.indexOf(code, opening + 3); + return ( + start: opening, + end: codeStart < 0 ? opening + 3 : codeStart + code.length, + ); +} + +String _plain(String value) => parseInline(value).map(_inlinePlain).join(); + +String _inlinePlain(MarkdownInline span) => switch (span) { + MarkdownPlain(text: final text) => text, + MarkdownInlineCode(text: final text) => text, + MarkdownStrong(spans: final spans) => spans.map(_inlinePlain).join(), + MarkdownEmphasis(spans: final spans) => spans.map(_inlinePlain).join(), + MarkdownLink(spans: final spans) => spans.map(_inlinePlain).join(), +}; diff --git a/app/lib/features/chat/speech/speech_engine.dart b/app/lib/features/chat/speech/speech_engine.dart new file mode 100644 index 00000000..fddba634 --- /dev/null +++ b/app/lib/features/chat/speech/speech_engine.dart @@ -0,0 +1,50 @@ +import 'package:flutter/foundation.dart'; + +/// One platform voice. [locale] is a BCP 47 tag such as `en-GB`. +@immutable +class SpeechVoice { + const SpeechVoice({required this.name, required this.locale}); + + final String name; + final String locale; + + /// The primary language subtag (`en` for `en-GB`), lower case. + String get language => locale.split(RegExp('[-_]')).first.toLowerCase(); + + @override + bool operator ==(Object other) => + other is SpeechVoice && other.name == name && other.locale == locale; + + @override + int get hashCode => Object.hash(name, locale); +} + +/// The platform text-to-speech service the [SpeechController] drives. +/// +/// [setRate] takes a multiplier where 1.0 is the platform's normal speed; an +/// implementation maps it onto its own scale. [onProgress] reports the word +/// being spoken as offsets into [utterance], the text the platform is actually +/// speaking (after a resume that can be a suffix of the text passed to +/// [speak]). Platforms without word boundaries never call it. +abstract class SpeechEngine { + /// Whether this platform can speak at all; the UI hides read aloud if not. + bool get supported; + + VoidCallback? onStart; + VoidCallback? onPause; + VoidCallback? onResume; + VoidCallback? onCompletion; + VoidCallback? onCancel; + ValueChanged? onError; + void Function(String utterance, int start, int end)? onProgress; + + /// Speaks [text]. Called with the same text while paused, it resumes. + Future speak(String text); + Future pause(); + Future stop(); + + /// Selects [voice], or the platform default when null. + Future setVoice(SpeechVoice? voice); + Future setRate(double rate); + Future> voices(); +} diff --git a/app/lib/features/chat/speech/speech_preferences.dart b/app/lib/features/chat/speech/speech_preferences.dart new file mode 100644 index 00000000..366c54dd --- /dev/null +++ b/app/lib/features/chat/speech/speech_preferences.dart @@ -0,0 +1,62 @@ +import 'package:shared_preferences/shared_preferences.dart'; + +import 'speech_engine.dart'; + +const speechRateKey = 'pipelines.readAloud.rate.v1'; +const speechVoiceNameKey = 'pipelines.readAloud.voiceName.v1'; +const speechVoiceLocaleKey = 'pipelines.readAloud.voiceLocale.v1'; + +/// The normal speaking rate and the slider's bounds (a multiplier). +const defaultSpeechRate = 1.0; +const minSpeechRate = 0.5; +const maxSpeechRate = 2.0; + +typedef SpeechPreferences = ({double rate, SpeechVoice? voice}); + +/// Device-local read-aloud settings. +abstract interface class SpeechPreferencesStore { + Future load(); + Future save(SpeechPreferences preferences); +} + +class SharedSpeechPreferencesStore implements SpeechPreferencesStore { + const SharedSpeechPreferencesStore(); + + @override + Future load() async { + try { + final prefs = await SharedPreferences.getInstance(); + final name = prefs.getString(speechVoiceNameKey); + final locale = prefs.getString(speechVoiceLocaleKey); + return ( + rate: prefs.getDouble(speechRateKey) ?? defaultSpeechRate, + voice: name == null || locale == null + ? null + : SpeechVoice(name: name, locale: locale), + ); + } catch (_) { + return null; + } + } + + @override + Future save(SpeechPreferences preferences) async { + try { + final prefs = await SharedPreferences.getInstance(); + final voice = preferences.voice; + final saved = [ + await prefs.setDouble(speechRateKey, preferences.rate), + if (voice == null) ...[ + await prefs.remove(speechVoiceNameKey), + await prefs.remove(speechVoiceLocaleKey), + ] else ...[ + await prefs.setString(speechVoiceNameKey, voice.name), + await prefs.setString(speechVoiceLocaleKey, voice.locale), + ], + ]; + return !saved.contains(false); + } catch (_) { + return false; + } + } +} diff --git a/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart b/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart index b4d84d31..27e68aa8 100644 --- a/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart +++ b/app/lib/features/chat/ui/transcript/chat_copy_gutter.dart @@ -51,6 +51,7 @@ class ChatCopyGutterRow extends StatefulWidget { required this.width, required this.copy, required this.child, + this.pinned = false, }); final double width; @@ -59,6 +60,9 @@ class ChatCopyGutterRow extends StatefulWidget { final Widget? copy; final Widget child; + /// Keeps the buttons shown without hover (while the reply reads aloud). + final bool pinned; + @override State createState() => _ChatCopyGutterRowState(); } @@ -89,7 +93,9 @@ class _ChatCopyGutterRowState extends State { setState(() => _focused = focused), child: AnimatedOpacity( key: const Key('chat-copy-gutter-visibility'), - opacity: !hover || _hovered || _focused ? 1 : 0, + opacity: !hover || widget.pinned || _hovered || _focused + ? 1 + : 0, duration: const Duration(milliseconds: 120), alwaysIncludeSemantics: true, child: copy, diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart new file mode 100644 index 00000000..72c1dbe7 --- /dev/null +++ b/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart @@ -0,0 +1,181 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; + +import '../../speech/speech.dart'; + +/// Space kept between a revealed word and the transcript's edge. +const double _revealMargin = 24; + +/// The range of [text] (one part of reply [messageId]) that [speech] is +/// speaking now, or null. +SpeechRange? spokenSourceIn( + SpeechController speech, + String? messageId, + String text, +) { + final document = speech.document; + final word = speech.word.value; + if (messageId == null || speech.messageId != messageId) return null; + if (document == null || word == null) return null; + final part = document.partOf(text); + if (part < 0) return null; + final range = document.sourceRange(part, word); + if (range == null) return null; + // Document parts are trimmed; the view's offsets include the lead. + final lead = text.length - text.trimLeft().length; + return (start: range.start + lead, end: range.end + lead); +} + +/// [text] as spans whose characters with source [offsets] inside [range] are +/// filled with [fill]; plain text when nothing in it is spoken. +TextSpan spokenTextSpan( + String text, + List offsets, + SpeechRange? range, + Color fill, +) { + if (range == null || offsets.isEmpty) return TextSpan(text: text); + bool spoken(int index) => + offsets[index] >= range.start && offsets[index] < range.end; + final children = []; + var runStart = 0; + for (var index = 1; index <= text.length; index += 1) { + if (index < text.length && spoken(index) == spoken(runStart)) continue; + children.add( + TextSpan( + text: text.substring(runStart, index), + style: spoken(runStart) ? TextStyle(backgroundColor: fill) : null, + ), + ); + runStart = index; + } + return TextSpan(children: children); +} + +/// Scrolls [scroll] just enough to show the spoken [range] when it has left +/// the viewport; a word already in view never moves the transcript. +void revealSpokenWord( + ScrollPosition scroll, + List segments, + SpeechRange range, { + required bool reduceMotion, +}) { + if (!scroll.hasContentDimensions) return; + final segment = segments + .where( + (item) => item.sourceStart < range.end && item.sourceEnd > range.start, + ) + .firstOrNull; + final paragraph = _paragraphIn(segment?.key.currentContext); + if (segment == null || paragraph == null) return; + final boxes = paragraph.getBoxesForSelection( + TextSelection( + baseOffset: segment.renderedStart, + extentOffset: segment.renderedEnd, + ), + ); + final viewport = RenderAbstractViewport.maybeOf(paragraph); + if (boxes.isEmpty || viewport == null) return; + final rect = boxes.first.toRect().inflate(_revealMargin); + final leading = viewport.getOffsetToReveal(paragraph, 0, rect: rect).offset; + final trailing = viewport.getOffsetToReveal(paragraph, 1, rect: rect).offset; + final low = leading < trailing ? leading : trailing; + final high = leading < trailing ? trailing : leading; + final target = scroll.pixels + .clamp(low, high) + .clamp(scroll.minScrollExtent, scroll.maxScrollExtent); + if ((target - scroll.pixels).abs() < 1) return; + if (reduceMotion) { + scroll.jumpTo(target); + return; + } + unawaited( + scroll.animateTo( + target, + duration: const Duration(milliseconds: 240), + curve: Curves.easeOut, + ), + ); +} + +/// The paragraph a keyed [Text] draws; selectable text wraps it in a mouse +/// region, so the keyed element's own render object is not it. +RenderParagraph? _paragraphIn(BuildContext? context) { + RenderParagraph? found; + void visit(RenderObject object) { + if (found != null) return; + if (object is RenderParagraph) { + found = object; + } else { + object.visitChildren(visit); + } + } + + final root = context?.findRenderObject(); + if (root != null) visit(root); + return found; +} + +/// Resolves visible inline leaves in order against the original block source. +/// Searching from the previous match skips emphasis/link/code delimiters and +/// keeps repeated words tied to their own occurrence. +class MarkdownSourceCursor { + MarkdownSourceCursor(this.source, int textStart, this.key) + : _cursor = textStart; + + final String source; + + /// The paragraph these leaves render into. + final GlobalKey key; + var _cursor = 0; + var visibleOffset = 0; + + List consume(String text) { + final offsets = []; + for (var index = 0; index < text.length; index += 1) { + _skipLinkDestination(); + final character = text.substring(index, index + 1); + final searchStart = _cursor.clamp(0, source.length); + final found = searchStart >= source.length + ? -1 + : source.indexOf(character, searchStart); + final offset = found < 0 ? searchStart : found; + offsets.add(offset); + _cursor = found < 0 ? searchStart + 1 : found + 1; + } + return offsets; + } + + void _skipLinkDestination() { + if (_cursor >= source.length || source[_cursor] != ']') return; + if (_cursor + 1 >= source.length || source[_cursor + 1] != '(') return; + var depth = 0; + for (var index = _cursor + 1; index < source.length; index += 1) { + if (source[index] == '(') depth += 1; + if (source[index] == ')' && --depth == 0) { + _cursor = index + 1; + return; + } + } + } +} + +/// One inline leaf: its source range and its character range in [key]'s +/// rendered paragraph. +class MarkdownSpokenSegment { + const MarkdownSpokenSegment( + this.key, + this.sourceStart, + this.sourceEnd, + this.renderedStart, + this.renderedEnd, + ); + + final GlobalKey key; + final int sourceStart; + final int sourceEnd; + final int renderedStart; + final int renderedEnd; +} diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart index af2862d6..065f8393 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart @@ -1,7 +1,7 @@ import 'dart:async'; import 'package:flutter/material.dart'; -import 'package:flutter/semantics.dart'; +import 'package:flutter/rendering.dart'; import 'chat_transcript_selection.dart'; import 'chat_code_hover.dart'; @@ -10,7 +10,9 @@ import 'chat_link_launcher.dart'; import 'chat_link_spans.dart'; import 'chat_markdown.dart'; import 'chat_markdown_cache.dart'; +import 'chat_markdown_speech.dart'; import 'chat_message_bubble.dart'; +import '../../speech/speech.dart'; /// Renders one assistant text part with native text widgets and the host's own /// theme roles. Links (`http`, `https`, `mailto` and `www.` hosts only) open @@ -19,9 +21,15 @@ import 'chat_message_bubble.dart'; /// dimension is overridden; code keeps its exact whitespace and scrolls /// sideways instead of widening the transcript. class ChatMarkdownView extends StatefulWidget { - const ChatMarkdownView({super.key, required this.text, this.onOpenLink}); + const ChatMarkdownView({ + super.key, + required this.text, + this.messageId, + this.onOpenLink, + }); final String text; + final String? messageId; final ValueChanged? onOpenLink; @override @@ -44,6 +52,14 @@ class _ChatMarkdownViewState extends State { final _codeHover = ChatCodeHover(); ScrollPosition? _scroll; ModalRoute? _route; + var _sourceCursor = 0; + + /// The source range of [ChatMarkdownView.text] being spoken, while building. + SpeechRange? _highlight; + SpeechRange? _revealed; + final _textKeys = []; + var _textCount = 0; + final _segments = []; @override void didUpdateWidget(ChatMarkdownView oldWidget) { @@ -77,12 +93,35 @@ class _ChatMarkdownViewState extends State { @override Widget build(BuildContext context) { - // A new token or usage update elsewhere in the turn must not reparse and - // rebuild every earlier answer part, or disturb its text selection. - if (_rendered != null) return _rendered!; + // Keep the state subscribed to inherited styling even when the cached + // subtree is returned without rebuilding its Markdown spans. + Theme.of(context); + final speech = speechController; + return ListenableBuilder( + listenable: Listenable.merge([speech, speech.word]), + builder: (context, _) { + final range = spokenSourceIn(speech, widget.messageId, widget.text); + if (range == null) { + _revealed = null; + return _rendered ?? _buildRendered(context, null); + } + if (range != _revealed) { + _revealed = range; + WidgetsBinding.instance.addPostFrameCallback((_) => _reveal(range)); + } + return _buildRendered(context, range); + }, + ); + } + + Widget _buildRendered(BuildContext context, SpeechRange? range) { + _sourceCursor = 0; + _textCount = 0; + _highlight = range; + _segments.clear(); final gap = chatParagraphGap(context); final blocks = chatMarkdownCache.blocksFor(widget.text); - return _rendered = Listener( + final rendered = Listener( onPointerDown: (_) => _codeHover.hide(), onPointerSignal: (_) => _codeHover.hide(), child: NotificationListener( @@ -108,6 +147,8 @@ class _ChatMarkdownViewState extends State { ), ), ); + if (range == null) _rendered = rendered; + return rendered; } /// Blocks are separated by an empty line of body text, like paragraphs; @@ -196,6 +237,7 @@ class _ChatMarkdownViewState extends State { ), ); case MarkdownCode(language: final language, code: final code): + _skipCodeBlock(code); final labelStyle = _roleStyle( context, Theme.of(context).textTheme.labelSmall, @@ -220,6 +262,14 @@ class _ChatMarkdownViewState extends State { } } + void _skipCodeBlock(String code) { + _sourceCursor = fencedCodeSpan( + widget.text, + _sourceCursor, + code, + ).end.clamp(0, widget.text.length); + } + Widget _cell(BuildContext context, String text, {bool header = false}) { final style = header ? _roleStyle(context, Theme.of(context).textTheme.labelLarge) @@ -234,43 +284,77 @@ class _ChatMarkdownViewState extends State { } Widget _text(BuildContext context, String text, {TextStyle? style}) { + if (text.isEmpty) return const SizedBox.shrink(); + final sourceStart = widget.text.indexOf(text, _sourceCursor); + if (sourceStart >= 0) _sourceCursor = sourceStart + text.length; + // Keys follow render order, which is stable for the same text, so the + // revealed word's paragraph can be found after the frame. + if (_textCount == _textKeys.length) _textKeys.add(GlobalKey()); + final key = _textKeys[_textCount++]; + final source = MarkdownSourceCursor( + widget.text, + sourceStart < 0 ? _sourceCursor : sourceStart, + key, + ); return Builder( builder: (textContext) => Text.rich( - TextSpan(children: _spans(textContext, parseInline(text))), + key: key, + TextSpan(children: _spans(textContext, parseInline(text), source)), style: style, ), ); } - List _spans(BuildContext context, List spans) => [ + List _spans( + BuildContext context, + List spans, + MarkdownSourceCursor source, + ) => [ for (final span in spans) switch (span) { - MarkdownPlain(text: final text) => TextSpan(text: text), - MarkdownInlineCode(text: final text) => _inlineCode(context, text), + MarkdownPlain(text: final text) => _leaf(context, text, source), + MarkdownInlineCode(text: final text) => _inlineCode( + context, + text, + source, + ), MarkdownStrong(spans: final inner) => TextSpan( style: const TextStyle(fontWeight: FontWeight.bold), - children: _spans(context, inner), + children: _spans(context, inner, source), ), MarkdownEmphasis(spans: final inner) => TextSpan( style: const TextStyle(fontStyle: FontStyle.italic), - children: _spans(context, inner), + children: _spans(context, inner, source), ), MarkdownLink(url: final url, spans: final inner) => _link( context, url, inner, + source, ), }, ]; - TextSpan _inlineCode(BuildContext context, String text) { + TextSpan _leaf( + BuildContext context, + String text, + MarkdownSourceCursor source, + ) => _highlightedText(context, text, source); + + TextSpan _inlineCode( + BuildContext context, + String text, + MarkdownSourceCursor source, + ) { final token = text; + final highlighted = _highlightedText(context, text, source); return TextSpan( - text: text, style: TextStyle( fontFamily: appMonospaceFamily, backgroundColor: _codeFill(context), ), + text: highlighted.text, + children: highlighted.children, onEnter: (event) { if (_route?.isCurrent != false) { _codeHover.show(context, token, text, event.position); @@ -284,10 +368,11 @@ class _ChatMarkdownViewState extends State { BuildContext context, String url, List spans, + MarkdownSourceCursor source, ) => _links.link( context, url, - _spans(context, spans), + _spans(context, spans, source), onOpen: (uri) { final onOpenLink = widget.onOpenLink; if (onOpenLink != null) return onOpenLink(uri.toString()); @@ -295,6 +380,46 @@ class _ChatMarkdownViewState extends State { }, ); + /// A leaf's spans, recording where it renders so a spoken word in it can + /// be revealed, with the spoken characters highlighted. + TextSpan _highlightedText( + BuildContext context, + String text, + MarkdownSourceCursor source, + ) { + final offsets = source.consume(text); + final renderedStart = source.visibleOffset; + source.visibleOffset += text.length; + if (offsets.isNotEmpty) { + _segments.add( + MarkdownSpokenSegment( + source.key, + offsets.first, + offsets.last + 1, + renderedStart, + source.visibleOffset, + ), + ); + } + return spokenTextSpan( + text, + offsets, + _highlight, + Theme.of(context).colorScheme.primaryContainer, + ); + } + + void _reveal(SpeechRange range) { + final scroll = _scroll; + if (!mounted || range != _revealed || scroll == null) return; + revealSpokenWord( + scroll, + _segments, + range, + reduceMotion: MediaQuery.maybeDisableAnimationsOf(context) ?? false, + ); + } + /// Heading roles are a non-increasing sequence from the host's own text /// theme. H3–H6 share one title role, matching the web renderer's flat /// lower-heading treatment, and no level falls back to a body role, so a diff --git a/app/lib/features/chat/ui/transcript/chat_message_view.dart b/app/lib/features/chat/ui/transcript/chat_message_view.dart index 1fee3081..813663e5 100644 --- a/app/lib/features/chat/ui/transcript/chat_message_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_message_view.dart @@ -14,6 +14,8 @@ import 'chat_message_bubble.dart'; import 'chat_turn_activity_view.dart'; import 'native_disclosure.dart'; import 'chat_message_timestamp.dart'; +import 'chat_read_aloud_button.dart'; +import '../../speech/speech_controller.dart'; /// Builds an assistant text part; the default keeps the server's text exactly. typedef ChatMessageTextBuilder = Widget Function(BuildContext, String); @@ -27,9 +29,14 @@ class ChatMessageView extends StatelessWidget { this.textBuilder, this.footer = const [], this.images, + this.speechScope = 'chat', }); final JsonMap row; + + /// Names the conversation in the reader's message ids, so leaving it can + /// stop only its own reply (see [speechMessageId]). + final String speechScope; final ChatMessageTextBuilder? textBuilder; /// Loads attached images for the operator's turn; without one, only images @@ -179,13 +186,41 @@ class ChatMessageView extends StatelessWidget { Widget answer, double gutter, ) { - final copyText = row['streaming'] == true ? '' : _answerText(_partsOf(row)); - final copy = copyText.isEmpty ? null : ChatCopyButton(text: copyText); - if (gutter <= 0) return _agentTurn(context, content, answer, copy); - return ChatCopyGutterRow( - width: gutter, - copy: copy, - child: _agentTurn(context, content, answer, null), + final answerParts = row['streaming'] == true + ? const [] + : _answerParts(_partsOf(row)); + final copyText = answerParts.join('\n\n'); + final id = speechMessageId(speechScope, '${row['key'] ?? row['id'] ?? ''}'); + Widget? actions(Axis direction) => copyText.isEmpty + ? null + : Flex( + direction: direction, + mainAxisSize: MainAxisSize.min, + verticalDirection: VerticalDirection.up, + children: [ + ChatCopyButton(text: copyText), + ChatReadAloudButton( + messageId: id, + parts: answerParts, + direction: direction, + ), + ], + ); + if (gutter <= 0) { + return _agentTurn(context, content, answer, actions(Axis.horizontal)); + } + // The gutter is one button wide: Read aloud stacks above Copy, and stays + // shown on a desktop while this reply reads. + final turn = _agentTurn(context, content, answer, null); + return ListenableBuilder( + listenable: speechController, + child: turn, + builder: (context, child) => ChatCopyGutterRow( + width: gutter, + copy: actions(Axis.vertical), + pinned: speechController.messageId == id, + child: child!, + ), ); } @@ -267,13 +302,13 @@ class ChatMessageView extends StatelessWidget { /// The answer as the operator reads it: the turn's text parts in order, /// without reasoning, tools or attachments. - String _answerText(List parts) => [ + List _answerParts(List parts) => [ for (final part in parts) if ((part['type'] == null || part['type'] == 'text') && part['text'] is String && (part['text'] as String).trim().isNotEmpty) (part['text'] as String).trim(), - ].join('\n\n'); + ]; List _agentParts(BuildContext context) { final widgets = []; diff --git a/app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart b/app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart new file mode 100644 index 00000000..543b7dca --- /dev/null +++ b/app/lib/features/chat/ui/transcript/chat_read_aloud_button.dart @@ -0,0 +1,69 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../../speech/speech.dart'; + +const chatReadAloudLabel = 'Read aloud'; +const chatPauseReadingLabel = 'Pause reading'; +const chatResumeReadingLabel = 'Resume reading'; +const chatStopReadingLabel = 'Stop reading'; + +/// Read aloud for one reply, styled like its Copy button. While this reply +/// reads it becomes Pause (or Resume) and Stop, laid out along [direction] +/// (a column in the narrow copy gutter). Absent where speech is unsupported. +class ChatReadAloudButton extends StatelessWidget { + const ChatReadAloudButton({ + super.key, + required this.messageId, + required this.parts, + this.direction = Axis.horizontal, + }); + + final String messageId; + + /// The reply's markdown text parts, in order. + final List parts; + final Axis direction; + + @override + Widget build(BuildContext context) { + final speech = speechController; + if (!speech.supported) return const SizedBox.shrink(); + return ListenableBuilder( + listenable: speech, + builder: (context, _) { + if (speech.messageId != messageId) { + return IconButton( + key: const Key('chat-message-read-aloud'), + tooltip: chatReadAloudLabel, + onPressed: () => unawaited(speech.read(messageId, parts)), + icon: const Icon(Icons.volume_up_outlined), + ); + } + final paused = speech.state == SpeechState.paused; + return Flex( + direction: direction, + mainAxisSize: MainAxisSize.min, + children: [ + IconButton( + key: const Key('chat-message-read-aloud-pause'), + tooltip: paused ? chatResumeReadingLabel : chatPauseReadingLabel, + onPressed: () => + unawaited(paused ? speech.resume() : speech.pause()), + icon: Icon( + paused ? Icons.play_arrow_outlined : Icons.pause_outlined, + ), + ), + IconButton( + key: const Key('chat-message-read-aloud-stop'), + tooltip: chatStopReadingLabel, + onPressed: () => unawaited(speech.stop()), + icon: const Icon(Icons.stop_outlined), + ), + ], + ); + }, + ); + } +} diff --git a/app/lib/features/settings/settings_page_sections.dart b/app/lib/features/settings/settings_page_sections.dart index ce285e21..1551b1cf 100644 --- a/app/lib/features/settings/settings_page_sections.dart +++ b/app/lib/features/settings/settings_page_sections.dart @@ -12,6 +12,13 @@ extension _SettingsPageSections on _SettingsScreenState { keywords: 'text size scale zoom large small font', builder: (_) => const SettingsScaleSection(), ), + if (speechController.supported) + const SettingsSection( + id: 'read-aloud', + title: 'Read aloud', + keywords: 'speech voice audio speaking rate read aloud', + builder: _readAloudBuilder, + ), SettingsSection( id: 'animations', title: 'Animations', @@ -216,6 +223,9 @@ extension _SettingsPageSections on _SettingsScreenState { ]; } + static Widget _readAloudBuilder(BuildContext _) => + const SettingsReadAloudSection(); + /// A group holding one free-form block (a form, a status with buttons). Widget _single(String title, Widget child) => SettingsGroup( title: title, diff --git a/app/lib/features/settings/settings_read_aloud_section.dart b/app/lib/features/settings/settings_read_aloud_section.dart new file mode 100644 index 00000000..826ba555 --- /dev/null +++ b/app/lib/features/settings/settings_read_aloud_section.dart @@ -0,0 +1,134 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../chat/speech/speech.dart'; +import 'settings_group.dart'; +import 'settings_row.dart'; +import 'settings_theme.dart'; +import 'settings_voice_picker.dart'; + +/// Read aloud: the voice and speaking rate replies are read with, saved on +/// this device, with a sample and a reset to the system voice at normal speed. +class SettingsReadAloudSection extends StatefulWidget { + const SettingsReadAloudSection({super.key, this.controller}); + + /// The reader; the app's own by default. + final SpeechController? controller; + + @override + State createState() => + _SettingsReadAloudSectionState(); +} + +class _SettingsReadAloudSectionState extends State { + SpeechController get _speech => widget.controller ?? speechController; + + @override + void initState() { + super.initState(); + unawaited(_speech.restore()); + } + + Future _pickVoice() async { + final choice = await showSpeechVoicePicker(context, _speech); + if (choice != null) await _speech.setVoice(choice.voice); + } + + @override + Widget build(BuildContext context) => ListenableBuilder( + listenable: _speech, + builder: (context, _) { + final rate = _speech.rate; + final label = '${rate.toStringAsFixed(1)}×'; + final voice = _speech.voice; + final sampling = _speech.messageId == speechSampleId; + return SettingsGroup( + title: 'Read aloud', + footer: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + const Text( + 'Saved on this device. Replies are read with this voice and rate.', + ), + if (_speech.saveFailed) + Text( + 'Could not save; applies until reload.', + style: TextStyle(color: settingsErrorColor(context)), + ), + ], + ), + children: [ + SettingsRow( + key: const Key('settings-read-aloud-voice'), + leading: const Icon(Icons.record_voice_over_outlined), + title: const Text('Voice'), + subtitle: Text( + voice == null + ? 'System default' + : '${voice.name} · ${voice.locale}', + ), + chevron: true, + onTap: () => unawaited(_pickVoice()), + ), + SettingsGroupItem( + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Text( + 'Speaking rate $label', + style: settingsTextStyle(context, title: true), + ), + Semantics( + label: 'Speaking rate', + child: Slider( + key: const Key('settings-read-aloud-rate'), + value: rate, + min: minSpeechRate, + max: maxSpeechRate, + divisions: 15, + label: label, + semanticFormatterCallback: (value) => + '${value.toStringAsFixed(1)} times', + onChanged: (value) => + unawaited(_speech.setRate(value, save: false)), + onChangeEnd: (value) => unawaited(_speech.setRate(value)), + ), + ), + const Wrap( + alignment: WrapAlignment.spaceBetween, + children: [Text('0.5× · Slower'), Text('2.0× · Faster')], + ), + ], + ), + ), + SettingsGroupItem( + child: Wrap( + spacing: 8, + runSpacing: 8, + crossAxisAlignment: WrapCrossAlignment.center, + children: [ + FilledButton.tonalIcon( + key: const Key('settings-read-aloud-sample'), + onPressed: () => + unawaited(sampling ? _speech.stop() : _speech.sample()), + icon: Icon( + sampling ? Icons.stop_outlined : Icons.play_arrow_outlined, + ), + label: Text(sampling ? 'Stop sample' : 'Play sample'), + ), + TextButton( + key: const Key('settings-read-aloud-reset'), + onPressed: _speech.isDefault + ? null + : () => unawaited(_speech.reset()), + child: const Text('Reset to defaults'), + ), + ], + ), + ), + ], + ); + }, + ); +} diff --git a/app/lib/features/settings/settings_screen.dart b/app/lib/features/settings/settings_screen.dart index 6c8ac6f2..0a126c6f 100644 --- a/app/lib/features/settings/settings_screen.dart +++ b/app/lib/features/settings/settings_screen.dart @@ -40,6 +40,8 @@ import 'managed_connection_section.dart'; import 'settings_keyboard_shortcuts_section.dart'; import 'settings_keys.dart'; import 'settings_row.dart'; +import 'settings_read_aloud_section.dart'; +import '../chat/speech/speech_controller.dart'; part 'settings_page_sections.dart'; diff --git a/app/lib/features/settings/settings_voice_picker.dart b/app/lib/features/settings/settings_voice_picker.dart new file mode 100644 index 00000000..c714474d --- /dev/null +++ b/app/lib/features/settings/settings_voice_picker.dart @@ -0,0 +1,203 @@ +import 'dart:async'; + +import 'package:flutter/material.dart'; + +import '../chat/speech/speech.dart'; + +/// The picker's answer: a voice, or null for the system default. +typedef SpeechVoiceChoice = ({SpeechVoice? voice}); + +/// Lists the platform's voices grouped by language, the device's language +/// first, each with a sample. Resolves to null when dismissed. +Future showSpeechVoicePicker( + BuildContext context, + SpeechController speech, +) async { + final choice = await showDialog( + context: context, + builder: (_) => _VoicePickerDialog(speech: speech), + ); + if (speech.messageId == speechSampleId) unawaited(speech.stop()); + return choice; +} + +/// Voices grouped by language subtag, the [device] language first and the +/// rest by name; voices within a group by name. +List<({String language, List voices})> groupSpeechVoices( + List voices, + String device, +) { + final groups = >{}; + for (final voice in voices) { + (groups[voice.language] ??= []).add(voice); + } + final languages = groups.keys.toList() + ..sort((a, b) { + if (a == device.toLowerCase()) return -1; + if (b == device.toLowerCase()) return 1; + return speechLanguageName(a).compareTo(speechLanguageName(b)); + }); + return [ + for (final language in languages) + ( + language: language, + voices: groups[language]! + ..sort((a, b) { + final name = a.name.compareTo(b.name); + return name != 0 ? name : a.locale.compareTo(b.locale); + }), + ), + ]; +} + +/// An English name for common language subtags; others show their code. +String speechLanguageName(String language) => + _languageNames[language] ?? language.toUpperCase(); + +const _languageNames = { + 'ar': 'Arabic', + 'bg': 'Bulgarian', + 'ca': 'Catalan', + 'cs': 'Czech', + 'da': 'Danish', + 'de': 'German', + 'el': 'Greek', + 'en': 'English', + 'es': 'Spanish', + 'fi': 'Finnish', + 'fr': 'French', + 'he': 'Hebrew', + 'hi': 'Hindi', + 'hr': 'Croatian', + 'hu': 'Hungarian', + 'id': 'Indonesian', + 'it': 'Italian', + 'ja': 'Japanese', + 'ko': 'Korean', + 'ms': 'Malay', + 'nb': 'Norwegian', + 'nl': 'Dutch', + 'pl': 'Polish', + 'pt': 'Portuguese', + 'ro': 'Romanian', + 'ru': 'Russian', + 'sk': 'Slovak', + 'sv': 'Swedish', + 'th': 'Thai', + 'tr': 'Turkish', + 'uk': 'Ukrainian', + 'vi': 'Vietnamese', + 'zh': 'Chinese', +}; + +class _VoicePickerDialog extends StatefulWidget { + const _VoicePickerDialog({required this.speech}); + + final SpeechController speech; + + @override + State<_VoicePickerDialog> createState() => _VoicePickerDialogState(); +} + +class _VoicePickerDialogState extends State<_VoicePickerDialog> { + late final Future> _voices = widget.speech.voices(); + + String _keyOf(SpeechVoice? voice) => + voice == null ? '' : '${voice.name}\n${voice.locale}'; + + @override + Widget build(BuildContext context) { + final device = View.of(context).platformDispatcher.locale.languageCode; + return AlertDialog( + title: const Text('Voice'), + contentPadding: const EdgeInsets.symmetric(vertical: 16), + // A list needs a definite width; keep it to a readable dialog measure. + content: ConstrainedBox( + constraints: const BoxConstraints(maxWidth: 560), + child: SizedBox( + width: double.maxFinite, + child: FutureBuilder>( + future: _voices, + builder: (context, snapshot) { + if (snapshot.connectionState != ConnectionState.done) { + return const Center(child: CircularProgressIndicator()); + } + final voices = snapshot.data ?? const []; + return _list(context, groupSpeechVoices(voices, device)); + }, + ), + ), + ), + actions: [ + TextButton( + onPressed: () => Navigator.of(context).pop(), + child: const Text('Cancel'), + ), + ], + ); + } + + Widget _list( + BuildContext context, + List<({String language, List voices})> groups, + ) { + final theme = Theme.of(context); + final header = theme.textTheme.titleSmall?.copyWith( + color: theme.colorScheme.primary, + ); + return RadioGroup( + groupValue: _keyOf(widget.speech.voice), + onChanged: (key) { + final voice = [ + for (final group in groups) ...group.voices, + ].where((voice) => _keyOf(voice) == key).firstOrNull; + Navigator.of(context).pop((voice: voice)); + }, + child: ListView( + shrinkWrap: true, + children: [ + const RadioListTile(value: '', title: Text('System default')), + if (groups.isEmpty) + Padding( + padding: const EdgeInsets.symmetric(horizontal: 24, vertical: 8), + child: Text( + 'This device offers no other voices.', + style: TextStyle(color: theme.colorScheme.onSurfaceVariant), + ), + ), + for (final group in groups) ...[ + Padding( + padding: const EdgeInsetsDirectional.fromSTEB(24, 16, 24, 4), + child: Semantics( + header: true, + child: Text(speechLanguageName(group.language), style: header), + ), + ), + for (final voice in group.voices) + // The sample is its own control beside the choice, not merged + // into the radio tile's semantics. + Row( + children: [ + Expanded( + child: RadioListTile( + value: _keyOf(voice), + title: Text(voice.name), + subtitle: Text(voice.locale), + ), + ), + Padding( + padding: const EdgeInsetsDirectional.only(end: 12), + child: IconButton( + tooltip: 'Play sample of ${voice.name}', + onPressed: () => unawaited(widget.speech.sample(voice)), + icon: const Icon(Icons.play_arrow_outlined), + ), + ), + ], + ), + ], + ], + ), + ); + } +} diff --git a/app/macos/Flutter/Flutter-Debug.xcconfig b/app/macos/Flutter/Flutter-Debug.xcconfig index c2efd0b6..4b81f9b2 100644 --- a/app/macos/Flutter/Flutter-Debug.xcconfig +++ b/app/macos/Flutter/Flutter-Debug.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" #include "ephemeral/Flutter-Generated.xcconfig" diff --git a/app/macos/Flutter/Flutter-Release.xcconfig b/app/macos/Flutter/Flutter-Release.xcconfig index c2efd0b6..5caa9d15 100644 --- a/app/macos/Flutter/Flutter-Release.xcconfig +++ b/app/macos/Flutter/Flutter-Release.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" #include "ephemeral/Flutter-Generated.xcconfig" diff --git a/app/macos/Flutter/GeneratedPluginRegistrant.swift b/app/macos/Flutter/GeneratedPluginRegistrant.swift index 3da2e7b4..8e871d2b 100644 --- a/app/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/app/macos/Flutter/GeneratedPluginRegistrant.swift @@ -8,6 +8,7 @@ import Foundation import bonsoir_darwin import file_picker_darwin import flutter_local_notifications +import flutter_tts import package_info_plus import share_plus import shared_preferences_foundation @@ -19,6 +20,7 @@ func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { SwiftBonsoirPlugin.register(with: registry.registrar(forPlugin: "SwiftBonsoirPlugin")) FilePickerPlugin.register(with: registry.registrar(forPlugin: "FilePickerPlugin")) FlutterLocalNotificationsPlugin.register(with: registry.registrar(forPlugin: "FlutterLocalNotificationsPlugin")) + FlutterTtsPlugin.register(with: registry.registrar(forPlugin: "FlutterTtsPlugin")) FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) SharePlusMacosPlugin.register(with: registry.registrar(forPlugin: "SharePlusMacosPlugin")) SharedPreferencesPlugin.register(with: registry.registrar(forPlugin: "SharedPreferencesPlugin")) diff --git a/app/macos/Podfile b/app/macos/Podfile new file mode 100644 index 00000000..88ac4486 --- /dev/null +++ b/app/macos/Podfile @@ -0,0 +1,47 @@ +# Match the Runner's MACOSX_DEPLOYMENT_TARGET; current Xcode rejects 10.15. +platform :osx, '12.0' + +# CocoaPods analytics sends network stats synchronously affecting flutter build latency. +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'ephemeral', 'Flutter-Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure \"flutter pub get\" is executed first" + end + + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Flutter-Generated.xcconfig, then run \"flutter pub get\"" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_macos_podfile_setup + +target 'Runner' do + use_frameworks! + + flutter_install_all_macos_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_macos_build_settings(target) + # Plugin podspecs (flutter_tts) still declare 10.15. + target.build_configurations.each do |config| + config.build_settings['MACOSX_DEPLOYMENT_TARGET'] = '12.0' + end + end +end diff --git a/app/macos/Podfile.lock b/app/macos/Podfile.lock new file mode 100644 index 00000000..f6c9bbcf --- /dev/null +++ b/app/macos/Podfile.lock @@ -0,0 +1,22 @@ +PODS: + - flutter_tts (0.0.1): + - FlutterMacOS + - FlutterMacOS (1.0.0) + +DEPENDENCIES: + - flutter_tts (from `Flutter/ephemeral/.symlinks/plugins/flutter_tts/macos`) + - FlutterMacOS (from `Flutter/ephemeral`) + +EXTERNAL SOURCES: + flutter_tts: + :path: Flutter/ephemeral/.symlinks/plugins/flutter_tts/macos + FlutterMacOS: + :path: Flutter/ephemeral + +SPEC CHECKSUMS: + flutter_tts: ae915565cc6948444b513acc8ee021993281e027 + FlutterMacOS: d0db08ddef1a9af05a5ec4b724367152bb0500b1 + +PODFILE CHECKSUM: 65be95680b03b164066c9a7e265a6f0a7b094da4 + +COCOAPODS: 1.17.0 diff --git a/app/macos/Runner.xcodeproj/project.pbxproj b/app/macos/Runner.xcodeproj/project.pbxproj index f2a52d37..be55c5d8 100644 --- a/app/macos/Runner.xcodeproj/project.pbxproj +++ b/app/macos/Runner.xcodeproj/project.pbxproj @@ -28,6 +28,8 @@ 33CC10F62044A3C60003C045 /* MainMenu.xib in Resources */ = {isa = PBXBuildFile; fileRef = 33CC10F42044A3C60003C045 /* MainMenu.xib */; }; 33CC11132044BFA00003C045 /* MainFlutterWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 33CC11122044BFA00003C045 /* MainFlutterWindow.swift */; }; 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */ = {isa = PBXBuildFile; productRef = 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */; }; + C7BD897E1E312E5E095D3400 /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = BA982DFB3805F270CAE1EC57 /* Pods_Runner.framework */; }; + EA524FBC4D8125AD71D1AB9E /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 52DA5BA5A5AAB1A3F8E8CACC /* Pods_RunnerTests.framework */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -61,11 +63,13 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 12FE65799543E39BACA69E1B /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.profile.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig"; sourceTree = ""; }; + 30F06383B0F1A8CC493DE4C3 /* Pods-RunnerTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.debug.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.debug.xcconfig"; sourceTree = ""; }; 331C80D5294CF71000263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 331C80D7294CF71000263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; 333000ED22D3DE5D00554162 /* Warnings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Warnings.xcconfig; sourceTree = ""; }; 335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GeneratedPluginRegistrant.swift; sourceTree = ""; }; - 33CC10ED2044A3C60003C045 /* Pipelines.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Pipelines.app"; sourceTree = BUILT_PRODUCTS_DIR; }; + 33CC10ED2044A3C60003C045 /* Pipelines.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Pipelines.app; sourceTree = BUILT_PRODUCTS_DIR; }; 33CC10F02044A3C60003C045 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 33CC10F22044A3C60003C045 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Assets.xcassets; path = Runner/Assets.xcassets; sourceTree = ""; }; 33CC10F52044A3C60003C045 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = ""; }; @@ -77,9 +81,15 @@ 33E51913231747F40026EE4D /* DebugProfile.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = DebugProfile.entitlements; sourceTree = ""; }; 33E51914231749380026EE4D /* Release.entitlements */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.entitlements; path = Release.entitlements; sourceTree = ""; }; 33E5194F232828860026EE4D /* AppInfo.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = AppInfo.xcconfig; sourceTree = ""; }; + 52DA5BA5A5AAB1A3F8E8CACC /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 769023E35970ADB12A4AC2D7 /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = ""; }; 78E0A7A72DC9AD7400C4905E /* FlutterGeneratedPluginSwiftPackage */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = FlutterGeneratedPluginSwiftPackage; path = ephemeral/Packages/FlutterGeneratedPluginSwiftPackage; sourceTree = ""; }; 7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Release.xcconfig; sourceTree = ""; }; + 8C9F29817EA0A7110E900E6F /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = ""; }; 9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = Debug.xcconfig; sourceTree = ""; }; + B396C4E4E3EE1B4BB1E67366 /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = ""; }; + BA982DFB3805F270CAE1EC57 /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D9747DDA8126822426B339BB /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -87,6 +97,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + EA524FBC4D8125AD71D1AB9E /* Pods_RunnerTests.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -95,6 +106,7 @@ buildActionMask = 2147483647; files = ( 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */, + C7BD897E1E312E5E095D3400 /* Pods_Runner.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -128,6 +140,7 @@ 331C80D6294CF71000263BE5 /* RunnerTests */, 33CC10EE2044A3C60003C045 /* Products */, D73912EC22F37F3D000D13A0 /* Frameworks */, + 46B086E826D3D29B49316EF3 /* Pods */, ); sourceTree = ""; }; @@ -176,9 +189,25 @@ path = Runner; sourceTree = ""; }; + 46B086E826D3D29B49316EF3 /* Pods */ = { + isa = PBXGroup; + children = ( + B396C4E4E3EE1B4BB1E67366 /* Pods-Runner.debug.xcconfig */, + 8C9F29817EA0A7110E900E6F /* Pods-Runner.release.xcconfig */, + D9747DDA8126822426B339BB /* Pods-Runner.profile.xcconfig */, + 30F06383B0F1A8CC493DE4C3 /* Pods-RunnerTests.debug.xcconfig */, + 769023E35970ADB12A4AC2D7 /* Pods-RunnerTests.release.xcconfig */, + 12FE65799543E39BACA69E1B /* Pods-RunnerTests.profile.xcconfig */, + ); + name = Pods; + path = Pods; + sourceTree = ""; + }; D73912EC22F37F3D000D13A0 /* Frameworks */ = { isa = PBXGroup; children = ( + BA982DFB3805F270CAE1EC57 /* Pods_Runner.framework */, + 52DA5BA5A5AAB1A3F8E8CACC /* Pods_RunnerTests.framework */, ); name = Frameworks; sourceTree = ""; @@ -190,6 +219,7 @@ isa = PBXNativeTarget; buildConfigurationList = 331C80DE294CF71000263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */; buildPhases = ( + 955AC6544A6B3BB98A250AB4 /* [CP] Check Pods Manifest.lock */, 331C80D1294CF70F00263BE5 /* Sources */, 331C80D2294CF70F00263BE5 /* Frameworks */, 331C80D3294CF70F00263BE5 /* Resources */, @@ -208,11 +238,13 @@ isa = PBXNativeTarget; buildConfigurationList = 33CC10FB2044A3C60003C045 /* Build configuration list for PBXNativeTarget "Runner" */; buildPhases = ( + F7F400CE023C878F98CF65C2 /* [CP] Check Pods Manifest.lock */, 33CC10E92044A3C60003C045 /* Sources */, 33CC10EA2044A3C60003C045 /* Frameworks */, 33CC10EB2044A3C60003C045 /* Resources */, 33CC110E2044A8840003C045 /* Bundle Framework */, 3399D490228B24CF009A79C7 /* ShellScript */, + 86DA14025ABF7D83888D3660 /* [CP] Embed Pods Frameworks */, ); buildRules = ( ); @@ -268,7 +300,7 @@ ); mainGroup = 33CC10E42044A3C60003C045; packageReferences = ( - 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage" */, + 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "FlutterGeneratedPluginSwiftPackage" */, ); productRefGroup = 33CC10EE2044A3C60003C045 /* Products */; projectDirPath = ""; @@ -339,6 +371,67 @@ shellPath = /bin/sh; shellScript = "\"$FLUTTER_ROOT\"/packages/flutter_tools/bin/macos_assemble.sh && touch Flutter/ephemeral/tripwire"; }; + 86DA14025ABF7D83888D3660 /* [CP] Embed Pods Frameworks */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Embed Pods Frameworks"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\"\n"; + showEnvVarsInLog = 0; + }; + 955AC6544A6B3BB98A250AB4 /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-RunnerTests-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + F7F400CE023C878F98CF65C2 /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-Runner-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -390,6 +483,7 @@ /* Begin XCBuildConfiguration section */ 331C80DB294CF71000263BE5 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 30F06383B0F1A8CC493DE4C3 /* Pods-RunnerTests.debug.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CURRENT_PROJECT_VERSION = 1; @@ -404,6 +498,7 @@ }; 331C80DC294CF71000263BE5 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 769023E35970ADB12A4AC2D7 /* Pods-RunnerTests.release.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CURRENT_PROJECT_VERSION = 1; @@ -418,6 +513,7 @@ }; 331C80DD294CF71000263BE5 /* Profile */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 12FE65799543E39BACA69E1B /* Pods-RunnerTests.profile.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CURRENT_PROJECT_VERSION = 1; @@ -712,7 +808,7 @@ /* End XCConfigurationList section */ /* Begin XCLocalSwiftPackageReference section */ - 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage" */ = { + 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "FlutterGeneratedPluginSwiftPackage" */ = { isa = XCLocalSwiftPackageReference; relativePath = Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage; }; diff --git a/app/macos/Runner.xcworkspace/contents.xcworkspacedata b/app/macos/Runner.xcworkspace/contents.xcworkspacedata index 1d526a16..21a3cc14 100644 --- a/app/macos/Runner.xcworkspace/contents.xcworkspacedata +++ b/app/macos/Runner.xcworkspace/contents.xcworkspacedata @@ -4,4 +4,7 @@ + + diff --git a/app/notes/read-aloud.md b/app/notes/read-aloud.md new file mode 100644 index 00000000..96ccb190 --- /dev/null +++ b/app/notes/read-aloud.md @@ -0,0 +1,55 @@ +# Read aloud + +An assistant reply has Read aloud beside Copy (same `IconButton` style). While +that reply reads it becomes Pause/Resume and Stop; one reply reads at a time +app-wide. The spoken word is highlighted (theme `primaryContainer`) in the +rendered reply and the transcript scrolls only when that word leaves the +viewport. Settings > Read aloud picks the voice (grouped by language, device +language first, a sample per voice), the rate (0.5x to 2.0x) and resets both; +they are saved on the device. + +## Structure + +- `lib/features/chat/speech/speech_engine.dart`: the `SpeechEngine` seam + (speak, pause, stop, setVoice, setRate, voices, start/pause/resume/ + progress/completion/cancel/error callbacks). +- `flutter_tts_speech_engine.dart`: the `flutter_tts` implementation. The rate + is a multiplier (1.0 = normal); Apple platforms get half of it on their 0..1 + scale. Linux reports unsupported, so the button and section are hidden. +- `speech_controller.dart`: the app's single reader (`speechController`, a fake + in tests via `debugSpeechController`). State changes notify the controller; + the spoken word is the separate `word` notifier. Stops on `hidden`, `paused` + or `detached` (not `inactive`, which is only a lost focus). +- `speech_document.dart`: Markdown to sentences, with each spoken character's + part and source offset. Code blocks (``` and ~~~) are "Code block.", tables + "Table." then cells; rules are skipped. +- `chat_markdown_speech.dart`: highlight spans and the reveal scroll for + `ChatMarkdownView`. +- Reader ids are `/` so a chat that closes or is swapped + for another stops only its own reply (`stopScope`). + +## Platform notes + +- Resume speaks the paused text again; Android, Apple and web plugins all + treat that as a resume. Android then reports progress against the remaining + suffix, which the controller maps back to the whole text. +- After a stop the controller waits (at most 400 ms) for the platform cancel, + so a late cancel cannot end the next reply and web speech is idle before + the next utterance. +- Apple and web `flutter_tts` cannot clear a chosen voice. After a voice was + set in a run, "System default" uses the device language's first voice. +- Platforms without word boundaries read without a highlight. +- `flutter_tts` has no Swift package, so macOS (and iOS) builds use CocoaPods + for it: `Podfile` and the `Pods-Runner` xcconfig includes are the generated + plugin integration. + +## Verification + +Tests (fake engine): `test/features/chat/speech/`, +`test/features/chat/ui/transcript/chat_read_aloud_button_test.dart`, +`chat_markdown_read_aloud_test.dart`, +`test/features/chat/conversation/chat_thread_read_aloud_test.dart`, +`test/features/settings/settings_read_aloud_section_test.dart`. + +Not verified on a device: real speech output, word boundaries and voices on +Android, macOS and browsers. diff --git a/app/pubspec.lock b/app/pubspec.lock index 6d19a1ef..2778d579 100644 --- a/app/pubspec.lock +++ b/app/pubspec.lock @@ -304,6 +304,14 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_tts: + dependency: "direct main" + description: + name: flutter_tts + sha256: ce5eb209b40e95f2f4a1397116c87ab2fcdff32257d04ed7a764e75894c03775 + url: "https://pub.dev" + source: hosted + version: "4.2.5" flutter_web_plugins: dependency: "direct main" description: flutter diff --git a/app/pubspec.yaml b/app/pubspec.yaml index c3d39b1a..66c0294a 100644 --- a/app/pubspec.yaml +++ b/app/pubspec.yaml @@ -48,6 +48,7 @@ dependencies: url_launcher: ^6.3.3 bonsoir: ^7.1.5 xterm: ^4.0.0 + flutter_tts: ^4.2.3 dev_dependencies: integration_test: diff --git a/app/test/features/chat/conversation/chat_thread_read_aloud_test.dart b/app/test/features/chat/conversation/chat_thread_read_aloud_test.dart new file mode 100644 index 00000000..487e1a10 --- /dev/null +++ b/app/test/features/chat/conversation/chat_thread_read_aloud_test.dart @@ -0,0 +1,74 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/conversation/chat_thread_view.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_read_aloud_button.dart'; + +import '../speech/fake_speech_engine.dart'; +import 'chat_thread_fixtures.dart'; + +Widget _thread(FakeChatGateway gateway, String sessionId) => MaterialApp( + theme: ThemeData(platform: TargetPlatform.android), + home: Scaffold( + body: ChatThreadView(gateway: gateway, sessionId: sessionId, onBack: () {}), + ), +); + +Future<(FakeSpeechEngine, SpeechController, FakeChatGateway)> _reading( + WidgetTester tester, +) async { + tester.view.physicalSize = const Size(800, 1600); + tester.view.devicePixelRatio = 1; + addTearDown(tester.view.reset); + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + final gateway = FakeChatGateway(); + addTearDown(gateway.close); + gateway.history = (_) async => chatPage([ + chatMessage( + 'a1', + 'assistant', + parts: [ + {'type': 'text', 'text': 'Hello from the agent.'}, + ], + ), + ]); + await tester.pumpWidget(_thread(gateway, 's1')); + await tester.pumpAndSettle(); + await tester.tap(find.byTooltip(chatReadAloudLabel)); + await tester.pump(); + engine.onStart!(); + expect(engine.spoken, ['Hello from the agent.']); + expect(speech.messageId, startsWith(speechScopePrefix('s1'))); + return (engine, speech, gateway); +} + +void main() { + testWidgets('switching the pane to another chat stops reading', ( + tester, + ) async { + final (engine, speech, gateway) = await _reading(tester); + await tester.pumpWidget(_thread(gateway, 's2')); + await tester.pumpAndSettle(); + expect(speech.state, SpeechState.idle); + expect(engine.calls, contains('stop')); + }); + + testWidgets('closing the chat stops reading', (tester) async { + final (engine, speech, _) = await _reading(tester); + await tester.pumpWidget(const MaterialApp(home: SizedBox())); + await tester.pumpAndSettle(); + expect(speech.state, SpeechState.idle); + expect(engine.calls.last, 'stop'); + }); + + testWidgets('closing a chat leaves another chat reading', (tester) async { + final (engine, speech, _) = await _reading(tester); + await speech.read(speechMessageId('elsewhere', 'b1'), ['Other reply']); + engine.onStart!(); + await tester.pumpWidget(const MaterialApp(home: SizedBox())); + await tester.pumpAndSettle(); + expect(speech.state, SpeechState.speaking); + expect(speech.messageId, speechMessageId('elsewhere', 'b1')); + }); +} diff --git a/app/test/features/chat/speech/fake_speech_engine.dart b/app/test/features/chat/speech/fake_speech_engine.dart new file mode 100644 index 00000000..060a4789 --- /dev/null +++ b/app/test/features/chat/speech/fake_speech_engine.dart @@ -0,0 +1,79 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; + +/// A scripted [SpeechEngine]: it records calls and a test fires the platform's +/// events. [stop] reports a cancel like every real platform does. +class FakeSpeechEngine extends SpeechEngine { + FakeSpeechEngine({this.supported = true, this.available = const []}); + + @override + final bool supported; + final List available; + final calls = []; + final spoken = []; + double? rate; + SpeechVoice? voice; + + @override + Future speak(String text) async { + calls.add('speak'); + spoken.add(text); + } + + @override + Future pause() async => calls.add('pause'); + + @override + Future stop() async { + calls.add('stop'); + onCancel?.call(); + } + + @override + Future setRate(double rate) async => this.rate = rate; + + @override + Future setVoice(SpeechVoice? voice) async => this.voice = voice; + + @override + Future> voices() async => available; + + /// Reports [word] (its first occurrence at or after [from]) being spoken. + void speakWord(String word, {int from = 0}) { + final text = spoken.last; + final start = text.indexOf(word, from); + expect(start, greaterThanOrEqualTo(0), reason: '"$word" is not spoken'); + onProgress?.call(text, start, start + word.length); + } +} + +/// Saved read-aloud settings held in memory, surviving a "reload". +class MemorySpeechPreferences implements SpeechPreferencesStore { + SpeechPreferences? saved; + + @override + Future load() async => saved; + + @override + Future save(SpeechPreferences preferences) async { + saved = preferences; + return true; + } +} + +/// Installs a reader over [engine] as the app's own for one test. +SpeechController installFakeSpeech( + FakeSpeechEngine engine, { + SpeechPreferencesStore? store, +}) { + final controller = SpeechController( + engine: engine, + store: store ?? MemorySpeechPreferences(), + ); + debugSpeechController = controller; + addTearDown(() { + debugSpeechController = null; + controller.dispose(); + }); + return controller; +} diff --git a/app/test/features/chat/speech/speech_controller_test.dart b/app/test/features/chat/speech/speech_controller_test.dart new file mode 100644 index 00000000..164b885a --- /dev/null +++ b/app/test/features/chat/speech/speech_controller_test.dart @@ -0,0 +1,168 @@ +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; + +import 'fake_speech_engine.dart'; + +void main() { + late FakeSpeechEngine engine; + late SpeechController speech; + + setUp(() { + engine = FakeSpeechEngine(); + }); + + SpeechController reader([SpeechPreferencesStore? store]) => + speech = SpeechController( + engine: engine, + store: store ?? MemorySpeechPreferences(), + ); + + tearDown(() => speech.dispose()); + + testWidgets('read, pause, resume and stop move through the states', ( + tester, + ) async { + reader(); + await speech.read('a', ['Hello **there**.']); + expect(speech.state, SpeechState.speaking); + expect(speech.messageId, 'a'); + expect(engine.spoken.last, 'Hello there.'); + engine.onStart!(); + + await speech.pause(); + expect(speech.state, SpeechState.paused); + expect(engine.calls.last, 'pause'); + engine.onPause!(); + expect(speech.state, SpeechState.paused); + + await speech.resume(); + expect(speech.state, SpeechState.speaking); + // Every platform resumes when the paused text is spoken again. + expect(engine.spoken.last, 'Hello there.'); + engine.onResume!(); + + await speech.stop(); + expect(speech.state, SpeechState.idle); + expect(speech.messageId, isNull); + expect(speech.document, isNull); + }); + + testWidgets('starting another reply stops the first', (tester) async { + reader(); + await speech.read('a', ['First reply']); + engine.onStart!(); + engine.speakWord('reply'); + await speech.read('b', ['Second reply']); + expect(engine.calls, ['speak', 'stop', 'speak']); + expect(speech.messageId, 'b'); + expect(speech.word.value, isNull); + // A late cancel of the first reply does not end the second. + engine.onCancel!(); + expect(speech.messageId, 'b'); + expect(speech.state, SpeechState.speaking); + }); + + testWidgets('progress sets the word and maps back to the source', ( + tester, + ) async { + reader(); + const first = 'Intro **bold** word'; + const second = '- item word'; + await speech.read('a', [first, second]); + engine.onStart!(); + engine.speakWord('bold'); + final document = speech.document!; + expect(speech.word.value, isNotNull); + expect(document.sourceRange(0, speech.word.value!), ( + start: first.indexOf('bold'), + end: first.indexOf('bold') + 4, + )); + expect(document.sourceRange(1, speech.word.value!), isNull); + + final spoken = engine.spoken.last; + engine.speakWord('word', from: spoken.lastIndexOf('item')); + expect(document.sourceRange(0, speech.word.value!), isNull); + expect(document.sourceRange(1, speech.word.value!), ( + start: second.indexOf('word'), + end: second.length, + )); + }); + + testWidgets('progress after a resumed suffix keeps document offsets', ( + tester, + ) async { + reader(); + await speech.read('a', ['one two three']); + engine.onStart!(); + // Android speaks the remaining text after a resume. + engine.onProgress!('three', 0, 5); + expect(speech.word.value, (start: 8, end: 13)); + }); + + testWidgets('completion and cancel clear the reply', (tester) async { + reader(); + await speech.read('a', ['Done soon']); + engine.onStart!(); + engine.speakWord('soon'); + engine.onCompletion!(); + expect(speech.state, SpeechState.idle); + expect(speech.messageId, isNull); + expect(speech.word.value, isNull); + + await speech.read('b', ['Cancelled']); + engine.onStart!(); + engine.onCancel!(); + expect(speech.state, SpeechState.idle); + expect(speech.messageId, isNull); + }); + + testWidgets('going to the background stops reading', (tester) async { + reader(); + await speech.read('a', ['Background']); + engine.onStart!(); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.inactive); + await tester.pump(); + expect(speech.state, SpeechState.speaking, reason: 'focus loss only'); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.hidden); + await tester.pump(); + expect(speech.state, SpeechState.idle); + expect(engine.calls.last, 'stop'); + tester.binding.handleAppLifecycleStateChanged(AppLifecycleState.resumed); + }); + + testWidgets('stopScope stops only replies of that conversation', ( + tester, + ) async { + reader(); + await speech.read(speechMessageId('chat-1', 'm1'), ['Hello']); + engine.onStart!(); + await speech.stopScope('chat-2'); + expect(speech.state, SpeechState.speaking); + await speech.stopScope('chat-1'); + expect(speech.state, SpeechState.idle); + }); + + testWidgets('voice and rate are applied and persist across a reload', ( + tester, + ) async { + final store = MemorySpeechPreferences(); + reader(store); + const voice = SpeechVoice(name: 'Daniel', locale: 'en-GB'); + await speech.setVoice(voice); + await speech.setRate(1.5); + await speech.read('a', ['Hello']); + expect(engine.voice, voice); + expect(engine.rate, 1.5); + speech.dispose(); + + reader(store); + await speech.restore(); + expect(speech.voice, voice); + expect(speech.rate, 1.5); + await speech.reset(); + expect(speech.voice, isNull); + expect(speech.rate, defaultSpeechRate); + expect(store.saved, (rate: defaultSpeechRate, voice: null)); + }); +} diff --git a/app/test/features/chat/speech/speech_document_test.dart b/app/test/features/chat/speech/speech_document_test.dart new file mode 100644 index 00000000..0bf900de --- /dev/null +++ b/app/test/features/chat/speech/speech_document_test.dart @@ -0,0 +1,56 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech_document.dart'; + +void main() { + test('markdown is spoken as sentences and code is announced', () { + final result = markdownToSpeech( + '# Welcome\n\n- **First**\n- [Docs](https://example.com)\n\n```dart\nprint(1);\n```\n\nThat is all.', + ); + expect(result.text, 'Welcome. First. Docs. Code block. That is all.'); + expect(result.sources.length, result.text.length); + }); + + test('tilde fences and tables are summarised', () { + final result = markdownToSpeech( + '~~~\nsecret();\n~~~\n\n| A | B |\n|---|---|\n| 1 | 2 |', + ); + expect(result.text, 'Code block. Table. A. B. 1. 2'); + expect(result.text, isNot(contains('secret'))); + }); + + test('repeated words map to their own occurrence', () { + const markdown = 'First repeat\n\nSecond repeat'; + final result = markdownToSpeech(markdown); + final first = result.text.indexOf('repeat'); + final second = result.text.lastIndexOf('repeat'); + expect(result.sourceRange(0, (start: first, end: first + 6)), ( + start: 6, + end: 12, + )); + expect(result.sourceRange(0, (start: second, end: second + 6)), ( + start: markdown.lastIndexOf('repeat'), + end: markdown.length, + )); + }); + + test('text after a code block maps after the skipped code', () { + const markdown = 'Before repeat\n\n```dart\nrepeat();\n```\n\nAfter repeat'; + final result = markdownToSpeech(markdown); + final spoken = result.text.lastIndexOf('repeat'); + final range = result.sourceRange(0, (start: spoken, end: spoken + 6))!; + expect(range.start, markdown.lastIndexOf('repeat')); + }); + + test('parts are trimmed and keep their own sources', () { + final result = speechDocumentFor([' One part\n', '', 'Two part']); + expect(result.parts, ['One part', 'Two part']); + expect(result.text, 'One part. Two part'); + expect(result.partOf('\nTwo part '), 1); + final two = result.text.indexOf('Two'); + expect(result.sourceRange(1, (start: two, end: two + 3)), ( + start: 0, + end: 3, + )); + expect(result.sourceRange(0, (start: two, end: two + 3)), isNull); + }); +} diff --git a/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart new file mode 100644 index 00000000..29f8205d --- /dev/null +++ b/app/test/features/chat/ui/transcript/chat_markdown_read_aloud_test.dart @@ -0,0 +1,120 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart'; + +import '../../speech/fake_speech_engine.dart'; + +/// The text of every span drawn with the spoken-word highlight. +List _highlighted(WidgetTester tester, [Finder? within]) { + final fill = Theme.of( + tester.element(find.byType(ChatMarkdownView).first), + ).colorScheme.primaryContainer; + final spans = []; + final texts = within == null + ? find.byType(RichText) + : find.descendant(of: within, matching: find.byType(RichText)); + for (final rich in tester.widgetList(texts)) { + rich.text.visitChildren((span) { + if (span is TextSpan && + span.text != null && + span.style?.backgroundColor == fill) { + spans.add(span.text!); + } + return true; + }); + } + return spans; +} + +void main() { + testWidgets('the highlight follows progress and clears on stop', ( + tester, + ) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + const first = 'Alpha **beta** gamma'; + const second = '- delta beta'; + await tester.pumpWidget( + const MaterialApp( + home: Scaffold( + body: Column( + children: [ + ChatMarkdownView(text: first, messageId: 'm'), + ChatMarkdownView(text: second, messageId: 'm'), + ChatMarkdownView(text: first, messageId: 'other'), + ], + ), + ), + ), + ); + await speech.read('m', [first, second]); + engine.onStart!(); + await tester.pump(); + expect(_highlighted(tester), isEmpty, reason: 'no word reported yet'); + + engine.speakWord('beta'); + await tester.pump(); + expect(_highlighted(tester), ['beta']); + + engine.speakWord('gamma'); + await tester.pump(); + expect(_highlighted(tester), ['gamma']); + + // The second part's "beta" lights in the second view only. + engine.speakWord('beta', from: engine.spoken.last.indexOf('delta')); + await tester.pump(); + expect(_highlighted(tester), ['beta']); + final views = find.byType(ChatMarkdownView); + expect(_highlighted(tester, views.at(1)), ['beta']); + expect(_highlighted(tester, views.at(0)), isEmpty); + + await speech.stop(); + await tester.pump(); + expect(_highlighted(tester), isEmpty); + }); + + testWidgets('the transcript scrolls only when the word leaves the view', ( + tester, + ) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + final text = [ + for (var index = 0; index < 40; index += 1) 'Paragraph $index words.', + ].join('\n\n'); + final scroll = ScrollController(); + addTearDown(scroll.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: SizedBox( + height: 300, + child: ListView( + controller: scroll, + children: [ChatMarkdownView(text: text, messageId: 'm')], + ), + ), + ), + ), + ); + await speech.read('m', [text]); + engine.onStart!(); + + engine.speakWord('Paragraph 2 '); + await tester.pumpAndSettle(); + expect(scroll.offset, 0, reason: 'the word is already in view'); + + engine.speakWord('Paragraph 30'); + await tester.pumpAndSettle(); + expect(scroll.offset, greaterThan(0)); + final viewport = tester.getRect(find.byType(ListView)); + final word = find.textContaining('Paragraph 30', findRichText: true); + expect(viewport.contains(tester.getRect(word).bottomLeft), isTrue); + final followed = scroll.offset; + + // A word still inside the view does not move the transcript. + final spoken = engine.spoken.last; + engine.speakWord('words', from: spoken.indexOf('Paragraph 30')); + await tester.pumpAndSettle(); + expect(scroll.offset, followed); + }); +} diff --git a/app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart b/app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart new file mode 100644 index 00000000..5ed3ea45 --- /dev/null +++ b/app/test/features/chat/ui/transcript/chat_read_aloud_button_test.dart @@ -0,0 +1,136 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/core/api_gateway.dart'; +import 'package:pipelines_app/features/chat/domain/chat_transcript.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_copy_gutter.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_read_aloud_button.dart'; + +import '../../speech/fake_speech_engine.dart'; + +Widget _host(Widget child) => MaterialApp( + home: Scaffold(body: Center(child: child)), +); + +JsonMap _answer(String id, String text) => transcriptRows([ + { + 'id': id, + 'role': 'assistant', + 'created_at': 1000, + 'parts': [ + {'type': 'text', 'text': text}, + ], + }, +]).single; + +final _read = find.byTooltip(chatReadAloudLabel); +final _pause = find.byTooltip(chatPauseReadingLabel); +final _resume = find.byTooltip(chatResumeReadingLabel); +final _stop = find.byTooltip(chatStopReadingLabel); + +void main() { + testWidgets('Read aloud becomes Pause and Stop, then Resume', (tester) async { + final engine = FakeSpeechEngine(); + final speech = installFakeSpeech(engine); + await tester.pumpWidget( + _host(const ChatReadAloudButton(messageId: 'a', parts: ['Hello'])), + ); + expect(_read, findsOneWidget); + expect(_pause, findsNothing); + + await tester.tap(_read); + await tester.pump(); + expect(engine.spoken, ['Hello']); + expect(_read, findsNothing); + expect(_pause, findsOneWidget); + expect(_stop, findsOneWidget); + + await tester.tap(_pause); + await tester.pump(); + expect(speech.state, SpeechState.paused); + expect(_resume, findsOneWidget); + expect(_pause, findsNothing); + + await tester.tap(_resume); + await tester.pump(); + expect(speech.state, SpeechState.speaking); + expect(_pause, findsOneWidget); + + await tester.tap(_stop); + await tester.pump(); + expect(speech.state, SpeechState.idle); + expect(_read, findsOneWidget); + expect(_stop, findsNothing); + }); + + testWidgets('only one reply reads at a time', (tester) async { + final engine = FakeSpeechEngine(); + installFakeSpeech(engine); + await tester.pumpWidget( + _host( + const Column( + mainAxisSize: MainAxisSize.min, + children: [ + ChatReadAloudButton(messageId: 'a', parts: ['First']), + ChatReadAloudButton(messageId: 'b', parts: ['Second']), + ], + ), + ), + ); + await tester.tap(_read.first); + await tester.pump(); + expect(_stop, findsOneWidget); + await tester.tap(_read); + await tester.pump(); + expect(engine.spoken, ['First', 'Second']); + expect(engine.calls, ['speak', 'stop', 'speak']); + expect(_stop, findsOneWidget); + expect(_read, findsOneWidget); + }); + + testWidgets('hidden where the platform cannot speak', (tester) async { + installFakeSpeech(FakeSpeechEngine(supported: false)); + await tester.pumpWidget( + _host(const ChatReadAloudButton(messageId: 'a', parts: ['Hello'])), + ); + expect(_read, findsNothing); + }); + + testWidgets('an answer offers Read aloud beside Copy, footer and gutter', ( + tester, + ) async { + installFakeSpeech(FakeSpeechEngine()); + Future pump(double gutter) => tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ListView( + children: [ + ChatCopyGutter( + width: gutter, + child: ChatMessageView(row: _answer('a1', 'The answer')), + ), + ], + ), + ), + ), + ); + final copy = find.byKey(const Key('chat-message-copy')); + + await pump(0); + expect(_read, findsOneWidget); + expect(tester.getCenter(_read).dy, tester.getCenter(copy).dy); + expect(tester.getCenter(_read).dx, greaterThan(tester.getCenter(copy).dx)); + + // The 48 px gutter stacks the buttons; while reading, Pause and Stop sit + // above Copy and nothing overflows. + await pump(48); + expect(tester.getCenter(_read).dx, tester.getCenter(copy).dx); + expect(tester.getCenter(_read).dy, lessThan(tester.getCenter(copy).dy)); + await tester.tap(_read); + await tester.pump(); + expect(tester.takeException(), isNull); + expect(tester.getCenter(_stop).dy, lessThan(tester.getCenter(copy).dy)); + expect(tester.getCenter(_pause).dy, lessThan(tester.getCenter(_stop).dy)); + }); +} diff --git a/app/test/features/settings/settings_read_aloud_section_test.dart b/app/test/features/settings/settings_read_aloud_section_test.dart new file mode 100644 index 00000000..7537bd67 --- /dev/null +++ b/app/test/features/settings/settings_read_aloud_section_test.dart @@ -0,0 +1,148 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/speech/speech.dart'; +import 'package:pipelines_app/features/settings/settings_read_aloud_section.dart'; +import 'package:pipelines_app/features/settings/settings_voice_picker.dart'; + +import '../chat/speech/fake_speech_engine.dart'; + +const _daniel = SpeechVoice(name: 'Daniel', locale: 'en-GB'); +const _samantha = SpeechVoice(name: 'Samantha', locale: 'en-US'); +const _thomas = SpeechVoice(name: 'Thomas', locale: 'fr-FR'); +const _anna = SpeechVoice(name: 'Anna', locale: 'de-DE'); + +final _voiceRow = find.byKey(const Key('settings-read-aloud-voice')); +final _slider = find.byKey(const Key('settings-read-aloud-rate')); +final _reset = find.byKey(const Key('settings-read-aloud-reset')); +final _sample = find.byKey(const Key('settings-read-aloud-sample')); + +late FakeSpeechEngine _engine; + +/// A fresh reader over [store], as after an app reload, and its section. +Future _open( + WidgetTester tester, + MemorySpeechPreferences store, +) async { + _engine = FakeSpeechEngine(available: [_samantha, _thomas, _anna, _daniel]); + final speech = SpeechController(engine: _engine, store: store); + addTearDown(speech.dispose); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ListView( + children: [ + SettingsReadAloudSection(key: UniqueKey(), controller: speech), + ], + ), + ), + ), + ); + await tester.pumpAndSettle(); + return speech; +} + +void main() { + testWidgets('a picked voice persists across a reload', (tester) async { + tester.platformDispatcher.localeTestValue = const Locale('fr', 'FR'); + addTearDown(tester.platformDispatcher.clearLocaleTestValue); + final store = MemorySpeechPreferences(); + await _open(tester, store); + expect(find.text('System default'), findsOneWidget); + + await tester.tap(_voiceRow); + await tester.pumpAndSettle(); + // Grouped by language, the device's language first. + final french = tester.getTopLeft(find.text('French')).dy; + final english = tester.getTopLeft(find.text('English')).dy; + final german = tester.getTopLeft(find.text('German')).dy; + expect(french, lessThan(english)); + expect(english, lessThan(german)); + expect( + tester.getTopLeft(find.text('Daniel')).dy, + lessThan(tester.getTopLeft(find.text('Samantha')).dy), + ); + + await tester.tap(find.text('Daniel')); + await tester.pumpAndSettle(); + expect(find.text('Daniel · en-GB'), findsOneWidget); + expect(store.saved?.voice, _daniel); + + final reloaded = await _open(tester, store); + expect(reloaded.voice, _daniel); + expect(find.text('Daniel · en-GB'), findsOneWidget); + await reloaded.read('m', ['Hello']); + expect(_engine.voice, _daniel); + }); + + testWidgets('the rate persists across a reload', (tester) async { + final store = MemorySpeechPreferences(); + final speech = await _open(tester, store); + expect(find.text('Speaking rate 1.0×'), findsOneWidget); + await tester.drag(_slider, const Offset(400, 0)); + await tester.pumpAndSettle(); + expect(speech.rate, greaterThan(defaultSpeechRate)); + expect(store.saved?.rate, speech.rate); + + final reloaded = await _open(tester, store); + expect(reloaded.rate, store.saved!.rate); + expect( + find.text('Speaking rate ${reloaded.rate.toStringAsFixed(1)}×'), + findsOneWidget, + ); + await reloaded.read('m', ['Hello']); + expect(_engine.rate, reloaded.rate); + }); + + testWidgets('reset restores the system voice and normal rate', ( + tester, + ) async { + final store = MemorySpeechPreferences() + ..saved = (rate: 1.5, voice: _thomas); + final speech = await _open(tester, store); + expect(find.text('Thomas · fr-FR'), findsOneWidget); + expect(find.text('Speaking rate 1.5×'), findsOneWidget); + + await tester.ensureVisible(_reset); + await tester.tap(_reset); + await tester.pumpAndSettle(); + expect(speech.voice, isNull); + expect(speech.rate, defaultSpeechRate); + expect(store.saved, (rate: defaultSpeechRate, voice: null)); + expect(find.text('System default'), findsOneWidget); + expect(find.text('Speaking rate 1.0×'), findsOneWidget); + expect(tester.widget(_reset).onPressed, isNull); + }); + + testWidgets('samples play through the engine', (tester) async { + final store = MemorySpeechPreferences()..saved = (rate: 1, voice: _anna); + final speech = await _open(tester, store); + await tester.ensureVisible(_sample); + await tester.tap(_sample); + await tester.pump(); + expect(_engine.spoken.single, contains('read aloud')); + expect(_engine.voice, _anna); + expect(find.text('Stop sample'), findsOneWidget); + await tester.tap(_sample); + await tester.pump(); + expect(speech.state, SpeechState.idle); + expect(find.text('Play sample'), findsOneWidget); + + // Each voice in the picker has its own sample; it does not choose it. + await tester.tap(_voiceRow); + await tester.pumpAndSettle(); + await tester.tap(find.byTooltip('Play sample of Samantha')); + await tester.pump(); + expect(_engine.voice, _samantha); + expect(_engine.spoken, hasLength(2)); + await tester.tap(find.text('Cancel')); + await tester.pumpAndSettle(); + expect(speech.voice, _anna); + expect(speech.state, SpeechState.idle, reason: 'closing stops the sample'); + }); + + test('voices group by language, device language first', () { + final groups = groupSpeechVoices([_samantha, _anna, _daniel], 'de'); + expect(groups.map((group) => group.language), ['de', 'en']); + expect(groups.last.voices, [_daniel, _samantha]); + }); +} diff --git a/app/tool/capture/README.md b/app/tool/capture/README.md index a701b2e8..7a9609de 100644 --- a/app/tool/capture/README.md +++ b/app/tool/capture/README.md @@ -33,6 +33,10 @@ URL parameters (the kit sets them from `open=` and the options): or script's `params`) adds fixture options such as `chats=2&rich=1`. Use `route=/settings` to open Settings and then click `Server health`; the fixture channel returns `server_stats` automatically. +`speech=fake` (with `speechStep=`, default 600) reads replies aloud with a +scripted engine that reports one word per step, since headless Chrome has no +voices: Read aloud, Pause/Resume/Stop, the word highlight and the Settings +voice list can then be captured. Without `theme`, the app follows the browser's colour scheme. `--theme` also sets that colour scheme through Chrome emulation. diff --git a/app/tool/probe_speech_fixture.dart b/app/tool/probe_speech_fixture.dart new file mode 100644 index 00000000..3e1283a0 --- /dev/null +++ b/app/tool/probe_speech_fixture.dart @@ -0,0 +1,92 @@ +import 'dart:async'; + +import 'package:pipelines_app/features/chat/speech/speech.dart'; + +/// Opt-in (`?speech=fake`) scripted reader for captures: headless browsers +/// have no speech voices, so this engine "speaks" one word every +/// `?speechStep=` ms (default 600) and reports each as progress, which drives +/// the real controller, buttons, highlight and reveal scrolling. +void installSpeechFixture(Map params) { + if (params['speech'] != 'fake') return; + final step = int.tryParse(params['speechStep'] ?? '') ?? 600; + debugSpeechController = SpeechController( + engine: _ScriptedSpeechEngine(Duration(milliseconds: step)), + ); +} + +class _ScriptedSpeechEngine extends SpeechEngine { + _ScriptedSpeechEngine(this.step); + + final Duration step; + Timer? _timer; + String _text = ''; + List _words = const []; + var _next = 0; + var _paused = false; + + @override + bool get supported => true; + + @override + Future speak(String text) async { + if (_paused && text == _text) { + _paused = false; + onResume?.call(); + _run(); + return; + } + _text = text; + _words = RegExp(r'\S+').allMatches(text).toList(); + _next = 0; + _paused = false; + Timer.run(() => onStart?.call()); + _run(); + } + + void _run() { + _timer?.cancel(); + _emit(); + _timer = Timer.periodic(step, (_) => _emit()); + } + + void _emit() { + if (_next >= _words.length) { + _timer?.cancel(); + onCompletion?.call(); + return; + } + final word = _words[_next++]; + onProgress?.call(_text, word.start, word.end); + } + + @override + Future pause() async { + _timer?.cancel(); + _paused = true; + if (_next > 0) _next -= 1; + onPause?.call(); + } + + @override + Future stop() async { + _timer?.cancel(); + _paused = false; + onCancel?.call(); + } + + @override + Future setRate(double rate) async {} + + @override + Future setVoice(SpeechVoice? voice) async {} + + @override + Future> voices() async => const [ + SpeechVoice(name: 'Samantha', locale: 'en-US'), + SpeechVoice(name: 'Daniel', locale: 'en-GB'), + SpeechVoice(name: 'Karen', locale: 'en-AU'), + SpeechVoice(name: 'Thomas', locale: 'fr-FR'), + SpeechVoice(name: 'Anna', locale: 'de-DE'), + SpeechVoice(name: 'Kyoko', locale: 'ja-JP'), + ]; +} diff --git a/app/tool/search_selection_probe_entry.dart b/app/tool/search_selection_probe_entry.dart index 47a8bdc1..b25a7e7a 100644 --- a/app/tool/search_selection_probe_entry.dart +++ b/app/tool/search_selection_probe_entry.dart @@ -47,6 +47,7 @@ import 'perf/perf_fixture.dart'; import 'probe_folder_fixture.dart'; import 'probe_files_fixture.dart'; import 'probe_usage_fixture.dart'; +import 'probe_speech_fixture.dart'; /// The run every run-scoped hit names. Its recorded sequences are deliberately /// not its row indexes (plan 5, build 2, plan 1), exactly like the unit @@ -786,6 +787,8 @@ class _FixtureSink implements WebSocketSink { /// `?terminal=float` opens them at launch (tool/perf/terminal_fixture.dart); /// `?windows=N` opens N floating probe windows whose content opens every /// kind of popup and a pushed page (tool/capture/window_probe.dart); +/// `?speech=fake` reads replies aloud with a scripted engine, one word per +/// `?speechStep=` ms (tool/probe_speech_fixture.dart); /// `?theme=dark` / `?theme=light` pins the platform brightness the app's /// `ThemeMode.system` follows (omitted: the browser's color scheme). /// @@ -806,6 +809,7 @@ Future main() async { return; } configurePerfFixture(params); + installSpeechFixture(params); if (params['files'] == '1') { probeMissingRememberedFolder = params['filesMissing'] == '1'; installFilesFixture();