Files
pipelines/app/lib/features/chat/ui/transcript/chat_markdown_speech.dart
T
alan 6367cce68a Read chat replies aloud with word highlighting
An assistant reply gets Read aloud beside Copy; while it reads it becomes
Pause/Resume and Stop, one reply at a time app-wide. The spoken word is
highlighted in the rendered reply (across multi-part replies) and the
transcript scrolls only when that word leaves the viewport. Reading stops
when its chat closes or is swapped, another reply starts, or the app goes
to the background. Markdown is spoken as sentences; code blocks are
announced as "Code block.".

Speech goes through a SpeechEngine seam with a flutter_tts implementation
(Android, macOS, web; hidden on Linux) and a fake for tests. Settings has
a Read aloud section: a voice picker grouped by language (device language
first, a sample per voice), a rate slider and reset, saved per device.

flutter_tts has no Swift package, so macOS/iOS gain the CocoaPods
integration (Podfiles with the Runner deployment targets); Android
declares the TTS_SERVICE query. The capture fixture gains ?speech=fake.
2026-11-13 01:29:04 +00:00

182 lines
5.6 KiB
Dart

import 'dart:async';
import 'package:flutter/material.dart';
import 'package:flutter/rendering.dart';
import '../../speech/speech.dart';
/// Space kept between a revealed word and the transcript's edge.
const double _revealMargin = 24;
/// The range of [text] (one part of reply [messageId]) that [speech] is
/// speaking now, or null.
SpeechRange? spokenSourceIn(
SpeechController speech,
String? messageId,
String text,
) {
final document = speech.document;
final word = speech.word.value;
if (messageId == null || speech.messageId != messageId) return null;
if (document == null || word == null) return null;
final part = document.partOf(text);
if (part < 0) return null;
final range = document.sourceRange(part, word);
if (range == null) return null;
// Document parts are trimmed; the view's offsets include the lead.
final lead = text.length - text.trimLeft().length;
return (start: range.start + lead, end: range.end + lead);
}
/// [text] as spans whose characters with source [offsets] inside [range] are
/// filled with [fill]; plain text when nothing in it is spoken.
TextSpan spokenTextSpan(
String text,
List<int> offsets,
SpeechRange? range,
Color fill,
) {
if (range == null || offsets.isEmpty) return TextSpan(text: text);
bool spoken(int index) =>
offsets[index] >= range.start && offsets[index] < range.end;
final children = <InlineSpan>[];
var runStart = 0;
for (var index = 1; index <= text.length; index += 1) {
if (index < text.length && spoken(index) == spoken(runStart)) continue;
children.add(
TextSpan(
text: text.substring(runStart, index),
style: spoken(runStart) ? TextStyle(backgroundColor: fill) : null,
),
);
runStart = index;
}
return TextSpan(children: children);
}
/// Scrolls [scroll] just enough to show the spoken [range] when it has left
/// the viewport; a word already in view never moves the transcript.
void revealSpokenWord(
ScrollPosition scroll,
List<MarkdownSpokenSegment> segments,
SpeechRange range, {
required bool reduceMotion,
}) {
if (!scroll.hasContentDimensions) return;
final segment = segments
.where(
(item) => item.sourceStart < range.end && item.sourceEnd > range.start,
)
.firstOrNull;
final paragraph = _paragraphIn(segment?.key.currentContext);
if (segment == null || paragraph == null) return;
final boxes = paragraph.getBoxesForSelection(
TextSelection(
baseOffset: segment.renderedStart,
extentOffset: segment.renderedEnd,
),
);
final viewport = RenderAbstractViewport.maybeOf(paragraph);
if (boxes.isEmpty || viewport == null) return;
final rect = boxes.first.toRect().inflate(_revealMargin);
final leading = viewport.getOffsetToReveal(paragraph, 0, rect: rect).offset;
final trailing = viewport.getOffsetToReveal(paragraph, 1, rect: rect).offset;
final low = leading < trailing ? leading : trailing;
final high = leading < trailing ? trailing : leading;
final target = scroll.pixels
.clamp(low, high)
.clamp(scroll.minScrollExtent, scroll.maxScrollExtent);
if ((target - scroll.pixels).abs() < 1) return;
if (reduceMotion) {
scroll.jumpTo(target);
return;
}
unawaited(
scroll.animateTo(
target,
duration: const Duration(milliseconds: 240),
curve: Curves.easeOut,
),
);
}
/// The paragraph a keyed [Text] draws; selectable text wraps it in a mouse
/// region, so the keyed element's own render object is not it.
RenderParagraph? _paragraphIn(BuildContext? context) {
RenderParagraph? found;
void visit(RenderObject object) {
if (found != null) return;
if (object is RenderParagraph) {
found = object;
} else {
object.visitChildren(visit);
}
}
final root = context?.findRenderObject();
if (root != null) visit(root);
return found;
}
/// Resolves visible inline leaves in order against the original block source.
/// Searching from the previous match skips emphasis/link/code delimiters and
/// keeps repeated words tied to their own occurrence.
class MarkdownSourceCursor {
MarkdownSourceCursor(this.source, int textStart, this.key)
: _cursor = textStart;
final String source;
/// The paragraph these leaves render into.
final GlobalKey key;
var _cursor = 0;
var visibleOffset = 0;
List<int> consume(String text) {
final offsets = <int>[];
for (var index = 0; index < text.length; index += 1) {
_skipLinkDestination();
final character = text.substring(index, index + 1);
final searchStart = _cursor.clamp(0, source.length);
final found = searchStart >= source.length
? -1
: source.indexOf(character, searchStart);
final offset = found < 0 ? searchStart : found;
offsets.add(offset);
_cursor = found < 0 ? searchStart + 1 : found + 1;
}
return offsets;
}
void _skipLinkDestination() {
if (_cursor >= source.length || source[_cursor] != ']') return;
if (_cursor + 1 >= source.length || source[_cursor + 1] != '(') return;
var depth = 0;
for (var index = _cursor + 1; index < source.length; index += 1) {
if (source[index] == '(') depth += 1;
if (source[index] == ')' && --depth == 0) {
_cursor = index + 1;
return;
}
}
}
}
/// One inline leaf: its source range and its character range in [key]'s
/// rendered paragraph.
class MarkdownSpokenSegment {
const MarkdownSpokenSegment(
this.key,
this.sourceStart,
this.sourceEnd,
this.renderedStart,
this.renderedEnd,
);
final GlobalKey key;
final int sourceStart;
final int sourceEnd;
final int renderedStart;
final int renderedEnd;
}