6367cce68a
An assistant reply gets Read aloud beside Copy; while it reads it becomes Pause/Resume and Stop, one reply at a time app-wide. The spoken word is highlighted in the rendered reply (across multi-part replies) and the transcript scrolls only when that word leaves the viewport. Reading stops when its chat closes or is swapped, another reply starts, or the app goes to the background. Markdown is spoken as sentences; code blocks are announced as "Code block.". Speech goes through a SpeechEngine seam with a flutter_tts implementation (Android, macOS, web; hidden on Linux) and a fake for tests. Settings has a Read aloud section: a voice picker grouped by language (device language first, a sample per voice), a rate slider and reset, saved per device. flutter_tts has no Swift package, so macOS/iOS gain the CocoaPods integration (Podfiles with the Runner deployment targets); Android declares the TTS_SERVICE query. The capture fixture gains ?speech=fake.
172 lines
6.0 KiB
Dart
172 lines
6.0 KiB
Dart
import '../ui/transcript/chat_markdown.dart';
|
|
import '../ui/transcript/chat_markdown_cache.dart';
|
|
|
|
typedef SpeechRange = ({int start, int end});
|
|
|
|
/// Where one spoken character came from: an offset into one markdown part.
|
|
typedef SpeechSource = ({int part, int offset});
|
|
|
|
/// A reply as spoken text, with the markdown source of every spoken character.
|
|
/// A reply can have several text parts; each is rendered by its own view, so
|
|
/// sources name the part as well as the offset within its trimmed text.
|
|
class SpeechDocument {
|
|
const SpeechDocument({
|
|
required this.text,
|
|
required this.parts,
|
|
required this.sources,
|
|
});
|
|
|
|
final String text;
|
|
|
|
/// The trimmed markdown of each part, in reading order.
|
|
final List<String> parts;
|
|
final List<SpeechSource> sources;
|
|
|
|
/// The index of the part whose markdown is [markdown], or -1.
|
|
int partOf(String markdown) => parts.indexOf(markdown.trim());
|
|
|
|
/// The source range in [part] that the spoken [range] covers, or null when
|
|
/// the spoken text there came from another part.
|
|
SpeechRange? sourceRange(int part, SpeechRange range) {
|
|
int? start;
|
|
int? end;
|
|
final last = range.end.clamp(0, sources.length);
|
|
for (var index = range.start.clamp(0, last); index < last; index += 1) {
|
|
final source = sources[index];
|
|
if (source.part != part) continue;
|
|
if (start == null || source.offset < start) start = source.offset;
|
|
if (end == null || source.offset + 1 > end) end = source.offset + 1;
|
|
}
|
|
return start == null ? null : (start: start, end: end!);
|
|
}
|
|
}
|
|
|
|
SpeechDocument markdownToSpeech(String markdown) =>
|
|
speechDocumentFor([markdown]);
|
|
|
|
/// Converts reply parts to natural speech: markup is dropped, blocks become
|
|
/// sentences, and a fenced code block is announced as "Code block." rather
|
|
/// than read out. Visible characters are located in order in the source, so
|
|
/// repeated words map to their own occurrence.
|
|
SpeechDocument speechDocumentFor(List<String> markdownParts) {
|
|
final parts = [
|
|
for (final part in markdownParts)
|
|
if (part.trim().isNotEmpty) part.trim(),
|
|
];
|
|
final builder = _SpeechBuilder();
|
|
for (var index = 0; index < parts.length; index += 1) {
|
|
builder.addPart(index, parts[index]);
|
|
}
|
|
return SpeechDocument(
|
|
text: builder.spoken.toString(),
|
|
parts: parts,
|
|
sources: builder.sources,
|
|
);
|
|
}
|
|
|
|
class _SpeechBuilder {
|
|
final spoken = StringBuffer();
|
|
final sources = <SpeechSource>[];
|
|
String _markdown = '';
|
|
var _part = 0;
|
|
var _cursor = 0;
|
|
|
|
void addPart(int part, String markdown) {
|
|
_part = part;
|
|
_markdown = markdown;
|
|
_cursor = 0;
|
|
for (final block in chatMarkdownCache.blocksFor(markdown)) {
|
|
switch (block) {
|
|
case MarkdownHeading(text: final text):
|
|
_visible(text);
|
|
case MarkdownParagraph(text: final text):
|
|
_visible(text);
|
|
case MarkdownQuote(lines: final lines):
|
|
lines.where((line) => line.trim().isNotEmpty).forEach(_visible);
|
|
case MarkdownList(items: final items):
|
|
items.forEach(_visible);
|
|
case MarkdownTable(header: final header, rows: final rows):
|
|
_phrase('Table.', _cursor);
|
|
[...header, ...rows.expand((row) => row)].forEach(_visible);
|
|
case MarkdownCode(code: final code):
|
|
_skipCode(code);
|
|
case MarkdownRule():
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Ends the previous sentence so blocks are not read as one run-on line.
|
|
void _sentenceBreak(int at) {
|
|
if (spoken.isEmpty) return;
|
|
if (!RegExp(r'[.!?:;]$').hasMatch(spoken.toString())) _write('.', at);
|
|
_write(' ', at);
|
|
}
|
|
|
|
void _write(String value, int at) {
|
|
spoken.write(value);
|
|
for (var index = 0; index < value.length; index += 1) {
|
|
sources.add((part: _part, offset: at));
|
|
}
|
|
}
|
|
|
|
void _phrase(String value, int at) {
|
|
_sentenceBreak(at);
|
|
_write(value, at);
|
|
}
|
|
|
|
void _visible(String blockText) {
|
|
final value = _plain(blockText).trim();
|
|
if (value.isEmpty) return;
|
|
final start = _markdown.indexOf(blockText, _cursor);
|
|
final blockStart = start < 0 ? _cursor : start;
|
|
final blockEnd = start < 0 ? _markdown.length : start + blockText.length;
|
|
_sentenceBreak(blockStart);
|
|
var search = blockStart;
|
|
for (var index = 0; index < value.length; index += 1) {
|
|
final character = value[index];
|
|
final found = search >= blockEnd
|
|
? -1
|
|
: _markdown.indexOf(character, search);
|
|
final inBlock = found >= 0 && found < blockEnd;
|
|
spoken.write(character);
|
|
sources.add((part: _part, offset: inBlock ? found : search));
|
|
if (inBlock) search = found + 1;
|
|
}
|
|
_cursor = start < 0 ? search : blockEnd;
|
|
}
|
|
|
|
void _skipCode(String code) {
|
|
final span = fencedCodeSpan(_markdown, _cursor, code);
|
|
_phrase('Code block.', span.start);
|
|
_cursor = span.end;
|
|
}
|
|
}
|
|
|
|
/// The source range of the fenced code block holding [code] that starts at or
|
|
/// after [cursor] (``` or ~~~ fences); an unclosed fence runs to its code's end.
|
|
SpeechRange fencedCodeSpan(String source, int cursor, String code) {
|
|
int find(String marker) => source.indexOf(marker, cursor);
|
|
final fences = [find('```'), find('~~~')].where((index) => index >= 0);
|
|
if (fences.isEmpty) return (start: cursor, end: cursor);
|
|
final opening = fences.reduce((a, b) => a < b ? a : b);
|
|
final marker = source.substring(opening, opening + 3);
|
|
final closing = source.indexOf(marker, opening + 3);
|
|
if (closing >= 0) return (start: opening, end: closing + 3);
|
|
final codeStart = source.indexOf(code, opening + 3);
|
|
return (
|
|
start: opening,
|
|
end: codeStart < 0 ? opening + 3 : codeStart + code.length,
|
|
);
|
|
}
|
|
|
|
String _plain(String value) => parseInline(value).map(_inlinePlain).join();
|
|
|
|
String _inlinePlain(MarkdownInline span) => switch (span) {
|
|
MarkdownPlain(text: final text) => text,
|
|
MarkdownInlineCode(text: final text) => text,
|
|
MarkdownStrong(spans: final spans) => spans.map(_inlinePlain).join(),
|
|
MarkdownEmphasis(spans: final spans) => spans.map(_inlinePlain).join(),
|
|
MarkdownLink(spans: final spans) => spans.map(_inlinePlain).join(),
|
|
};
|