diff --git a/app/lib/features/chat/conversation/chat_row_widgets.dart b/app/lib/features/chat/conversation/chat_row_widgets.dart new file mode 100644 index 00000000..6e105164 --- /dev/null +++ b/app/lib/features/chat/conversation/chat_row_widgets.dart @@ -0,0 +1,74 @@ +import 'package:flutter/widgets.dart'; + +/// Reuses a built widget while the content and inputs it was built from are +/// unchanged: one per transcript row, or per piece of conversation chrome. +/// +/// Every streamed delta rebuilds the conversation, and each build makes a new +/// message widget for every row. Handing Flutter the identical widget instance +/// for an unchanged row lets it skip that row's whole subtree, so during +/// streaming only the live turn rebuilds. Inherited changes (theme, platform +/// style, text scale) still reach the reused subtree through its own +/// dependencies. +class ChatRowWidgets { + final Map _held = {}; + Set _seen = {}; + + /// The widget held for [key] when [row] and [inputs] deep-equal the ones it + /// was built from, else a fresh [build] that is held instead. + /// + /// [row] is decoded JSON (maps, lists and scalars); [inputs] are the other + /// values the row's widget depends on, compared the same way. + Widget widgetFor( + String key, + Object? row, + Object? inputs, + Widget Function() build, + ) { + _seen.add(key); + final held = _held[key]; + if (held != null && + chatJsonEquals(held.row, row) && + chatJsonEquals(held.inputs, inputs)) { + return held.widget; + } + final widget = build(); + _held[key] = (row: row, inputs: inputs, widget: widget); + return widget; + } + + /// Drops rows the last build did not ask for. Call once after each build. + void prune() { + _held.removeWhere((key, _) => !_seen.contains(key)); + _seen = {}; + } + + void clear() { + _held.clear(); + _seen = {}; + } +} + +/// Structural equality for decoded JSON. Strings shared between snapshots are +/// identical, so an unchanged transcript compares in near-constant time per +/// field. +bool chatJsonEquals(Object? a, Object? b) { + if (identical(a, b)) return true; + if (a is Map && b is Map) { + if (a.length != b.length) return false; + for (final entry in a.entries) { + if (!b.containsKey(entry.key) || + !chatJsonEquals(entry.value, b[entry.key])) { + return false; + } + } + return true; + } + if (a is List && b is List) { + if (a.length != b.length) return false; + for (var index = 0; index < a.length; index++) { + if (!chatJsonEquals(a[index], b[index])) return false; + } + return true; + } + return a == b; +} diff --git a/app/lib/features/chat/conversation/chat_thread_view.dart b/app/lib/features/chat/conversation/chat_thread_view.dart index 654fb1a8..4abe59c5 100644 --- a/app/lib/features/chat/conversation/chat_thread_view.dart +++ b/app/lib/features/chat/conversation/chat_thread_view.dart @@ -26,6 +26,7 @@ import 'chat_mode_control.dart'; import 'chat_model_control.dart'; import 'chat_models.dart'; import 'chat_row_keys.dart'; +import 'chat_row_widgets.dart'; import 'chat_scroll.dart'; import 'chat_server_queue.dart'; import 'chat_thread_controller.dart'; @@ -136,6 +137,11 @@ class _ChatThreadViewState extends State { late ChatThreadController _controller = _attach(); late ChatComposerController _composer = _attachComposer(); late ChatRowKeys _rowKeys = ChatRowKeys(); + final _rowWidgets = ChatRowWidgets(); + + /// The floating bar and composer, reused while their inputs hold so a + /// streamed delta does not rebuild the conversation's chrome. + final _chrome = ChatRowWidgets(); /// Sent images' bytes for this server, cached across rebuilds and scrolling. late ChatAttachmentImages _images = ChatAttachmentImages(widget.gateway); @@ -402,6 +408,8 @@ class _ChatThreadViewState extends State { _controller = _attach(); _composer = _attachComposer(); _rowKeys = ChatRowKeys(); + _rowWidgets.clear(); + _chrome.clear(); _dismissals = ChatNoticeDismissals(); _reconnecting = false; _adoptCachedTranscript(); @@ -757,8 +765,13 @@ class _ChatThreadViewState extends State { // The transcript fills the page; the conversation's bar floats over its // top and the frosted composer over its bottom, and history scrolls // behind both. - final column = FloatingBarOverlay( - bar: ChatComposerBar( + // Every input of the composer comes from this widget's configuration, + // the composer, the images and the running state. + final composerBar = _chrome.widgetFor( + 'composer', + _controller.running, + [widget, _composer, _images], + () => ChatComposerBar( key: ValueKey('chat-composer-${widget.sessionId}'), composer: _composer, running: _controller.running, @@ -775,6 +788,9 @@ class _ChatThreadViewState extends State { syncToken: widget.syncToken, ), ), + ); + final column = FloatingBarOverlay( + bar: composerBar, child: SegmentedBarOverlay( bar: _header(context, inShellBar), child: NotificationListener( @@ -787,6 +803,24 @@ class _ChatThreadViewState extends State { // window; the margins beside it still scroll the list. final side = chatReadingInset(constraints.maxWidth); final media = MediaQuery.paddingOf(context); + final transcript = [ + for (var index = 0; index < rows.length; index++) + Padding( + key: ValueKey(keys[index]), + padding: const EdgeInsets.symmetric(horizontal: 16), + child: _transcriptRow( + rows[index], + keys[index], + onDismissCompaction: + compactionNotice != null && + chatNoticeActivityCondition(rows[index]) == + compactionNotice.condition + ? () => _dismiss(compactionNotice) + : null, + ), + ), + ]; + _rowWidgets.prune(); return Stack( children: [ // At rest the first item starts below the floating bar @@ -861,24 +895,7 @@ class _ChatThreadViewState extends State { padding: EdgeInsets.all(16), child: Text('No messages in this chat yet.'), ), - for (var index = 0; index < rows.length; index++) - Padding( - key: ValueKey(keys[index]), - padding: const EdgeInsets.symmetric( - horizontal: 16, - ), - child: _transcriptRow( - rows[index], - onDismissCompaction: - compactionNotice != null && - chatNoticeActivityCondition( - rows[index], - ) == - compactionNotice.condition - ? () => _dismiss(compactionNotice) - : null, - ), - ), + ...transcript, // Messages the server holds queued (one sent before // a reload, or from another device) follow the // transcript: none has reached the harness yet. @@ -952,15 +969,29 @@ class _ChatThreadViewState extends State { /// the model and mode controls. Widget _header(BuildContext context, bool inShellBar) { final session = widget.session; + final canGoBack = inShellBar + ? ModalRoute.of(context)?.impliesAppBarDismissal ?? false + : true; + final answeredBy = latestReplyModel(_controller.messages); + return _chrome.widgetFor('header', answeredBy, [ + widget, + inShellBar, + canGoBack, + ], () => _buildHeader(session, inShellBar, canGoBack, answeredBy)); + } + + Widget _buildHeader( + JsonMap? session, + bool inShellBar, + bool canGoBack, + String answeredBy, + ) { final hasModel = session != null && supportsChatModels('${session['backend'] ?? ''}'); // The mode control has nothing to show (or change) without both facts. final hasMode = '${session?['mode'] ?? ''}'.trim().isNotEmpty && '${session?['mode_name'] ?? ''}'.trim().isNotEmpty; - final canGoBack = inShellBar - ? ModalRoute.of(context)?.impliesAppBarDismissal ?? false - : true; final directory = '${session?['directory'] ?? ''}'.trim(); final folder = directory .split(RegExp(r'[/\\]')) @@ -991,7 +1022,7 @@ class _ChatThreadViewState extends State { sessionId: widget.sessionId, session: session, onChanged: widget.onModelChanged, - answeredBy: latestReplyModel(_controller.messages), + answeredBy: answeredBy, ), ), if (hasMode) @@ -1061,7 +1092,11 @@ class _ChatThreadViewState extends State { }); } - Widget _transcriptRow(JsonMap row, {VoidCallback? onDismissCompaction}) { + Widget _transcriptRow( + JsonMap row, + String key, { + VoidCallback? onDismissCompaction, + }) { final kind = '${row['kind'] ?? ''}'; if (kind == 'activity') { return _ChatActivityLine(row: row, onDismiss: onDismissCompaction); @@ -1090,20 +1125,32 @@ class _ChatThreadViewState extends State { final delivery = message != null ? _composer.deliveries.stateFor(message) : null; - return ChatMessageView( - row: row, - textBuilder: chatMarkdownText, - footer: [ - ...chips, - if (delivery != null) - ChatDeliveryIndicator( - state: delivery, - cupertino: - chatControlsStyleOf(context) == ChatControlsStyle.cupertino, - onRemove: _removeAction('${message?['operation_id'] ?? ''}'), - ), - ], - images: _images, + final cupertino = + chatControlsStyleOf(context) == ChatControlsStyle.cupertino; + final onRemove = _removeAction('${message?['operation_id'] ?? ''}'); + // An unchanged finished row hands back its previous widget, so a streamed + // delta rebuilds only the live turn. Transcript snapshots are copied on + // write, never mutated, so a deep-equal row is the same content. Permission + // chips carry objects that compare by identity, so a row with chips is + // always rebuilt. + return _rowWidgets.widgetFor( + key, + row, + [delivery, cupertino, onRemove != null, _images, _controller, _composer], + () => ChatMessageView( + row: row, + textBuilder: chatMarkdownText, + footer: [ + ...chips, + if (delivery != null) + ChatDeliveryIndicator( + state: delivery, + cupertino: cupertino, + onRemove: onRemove, + ), + ], + images: _images, + ), ); } diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_cache.dart b/app/lib/features/chat/ui/transcript/chat_markdown_cache.dart new file mode 100644 index 00000000..a591b402 --- /dev/null +++ b/app/lib/features/chat/ui/transcript/chat_markdown_cache.dart @@ -0,0 +1,49 @@ +import 'dart:collection'; + +import 'chat_markdown.dart'; + +/// Parsed markdown blocks shared by every markdown view, keyed by the text. +/// +/// A lazily built transcript disposes rows scrolled out of view, so scrolling +/// back would parse the same answer again. Blocks are immutable and depend on +/// the text alone, so one parse serves every later view of that text. Bounded +/// by entries and by held characters, evicting the least recently used first, +/// so a long streamed reply's intermediate texts cannot grow it without limit. +class ChatMarkdownCache { + ChatMarkdownCache({this.maxEntries = 400, this.maxChars = 1000000}); + + final int maxEntries; + final int maxChars; + + /// Least recently used first. + final LinkedHashMap> _blocks = LinkedHashMap(); + int _chars = 0; + + /// How many texts were actually parsed (cache misses). + int parses = 0; + + List blocksFor(String text) { + final held = _blocks.remove(text); + if (held != null) return _blocks[text] = held; + final blocks = List.unmodifiable(parseMarkdown(text)); + parses++; + if (text.length > maxChars) return blocks; + _blocks[text] = blocks; + _chars += text.length; + while (_blocks.length > maxEntries || _chars > maxChars) { + final oldest = _blocks.keys.first; + _blocks.remove(oldest); + _chars -= oldest.length; + } + return blocks; + } + + void clear() { + _blocks.clear(); + _chars = 0; + parses = 0; + } +} + +/// The one cache the transcript's markdown views share. +final chatMarkdownCache = ChatMarkdownCache(); diff --git a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart index 9119ce7d..5f5b15d8 100644 --- a/app/lib/features/chat/ui/transcript/chat_markdown_view.dart +++ b/app/lib/features/chat/ui/transcript/chat_markdown_view.dart @@ -6,6 +6,7 @@ import 'package:flutter/semantics.dart'; import '../../../../ui/adaptive/host.dart'; import '../sessions/chat_adaptive.dart'; import 'chat_markdown.dart'; +import 'chat_markdown_cache.dart'; import 'chat_message_bubble.dart'; /// Renders one assistant text part with native text widgets and the host's own @@ -68,7 +69,7 @@ class _ChatMarkdownViewState extends State { final cupertino = chatControlsStyleOf(context) == ChatControlsStyle.cupertino; final gap = chatParagraphGap(context); - final blocks = parseMarkdown(widget.text); + final blocks = chatMarkdownCache.blocksFor(widget.text); return _rendered = _selectable( context, Column( diff --git a/app/notes/chat-stream-perf.md b/app/notes/chat-stream-perf.md new file mode 100644 index 00000000..5d765f9b --- /dev/null +++ b/app/notes/chat-stream-perf.md @@ -0,0 +1,58 @@ +# Chat thread streaming performance + +Measurement-led round on the chat thread while replies stream and while long +transcripts scroll. No behaviour change. + +## Changes + +- `ChatRowWidgets` (`lib/features/chat/conversation/chat_row_widgets.dart`) + hands back a row's previous `ChatMessageView` while the row deep-equals the + one it was built from (and delivery state, platform style, remove action, + images, controller and composer are unchanged). Snapshots are copy-on-write, + so Flutter skips every finished row's subtree. Rows with permission chips + always rebuild, because those objects compare by identity. +- The composer bar and the floating header are reused the same way while the + thread widget, running state and latest reply model hold. +- `ChatMarkdownCache` (`lib/features/chat/ui/transcript/chat_markdown_cache.dart`) + shares parsed blocks by text, bounded to 400 entries and 1M characters, so a + row scrolled back into view is not parsed again. + +Already in place and left unchanged: `ListView(children:)` lays out lazily +(SliverChildListDelegate) with a `RepaintBoundary` and a stable key for each row. +Thumbnails decode at display size (`ResizeImage`). Activity timers run only on +the streaming turn. + +## Numbers + +Widget harness `test/features/chat/perf/chat_stream_rebuild_perf_test.dart` +streams 200 deltas into a 300-message transcript (debug VM, so compare the +numbers with each other only): + +| | before | after | +| --- | --- | --- | +| ChatMessageView builds | 856 | 200 | +| all widget builds | 83,888 | 20,018 | +| markdown parses while streaming | 200 | 200 (live turn only) | +| frame time per delta (mean / p90) | 7.3 / 10.8 ms | 4.8 / 5.7 ms | +| markdown parses, 80 scroll drags | 43 | 0 | + +Release web probe `node tool/chat_perf_probe.mjs 300 200` (built from +`tool/chat_composer_probe_entry.dart`; headless Chrome renders in software). +Three runs of each, Flutter FrameTiming: + +| | before | after | +| --- | --- | --- | +| streaming build mean | 9.1–10.5 ms | 6.0–6.4 ms | +| streaming build p90 | 14.6–17.4 ms | 7.2–8.5 ms | +| streaming total frame mean | 67–72 ms | 45–50 ms | +| scrolling build mean | 2.5–2.8 ms | 1.6–2.8 ms (within noise) | + +## Findings left open + +- Opening a 300-message transcript builds every row once: jumping to the end of + a top-anchored `SliverList` has to lay out each row in between (~150 parses, + about 0.9 s in the debug harness). A bottom-anchored (reversed) list would + avoid that, but it changes the scroll-restore and follow logic, so it needs a + separate change. +- The fixture logs a `_PlatformViewPlaceholderBox.getTransformTo` null-check + error on web with a long transcript. It also happens before this change. diff --git a/app/test/features/chat/conversation/chat_row_widgets_test.dart b/app/test/features/chat/conversation/chat_row_widgets_test.dart new file mode 100644 index 00000000..93065079 --- /dev/null +++ b/app/test/features/chat/conversation/chat_row_widgets_test.dart @@ -0,0 +1,70 @@ +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/features/chat/conversation/chat_row_widgets.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_cache.dart'; + +void main() { + test('chatJsonEquals compares decoded JSON structurally', () { + expect( + chatJsonEquals( + { + 'a': [ + 1, + 'x', + {'b': true}, + ], + }, + { + 'a': [ + 1, + 'x', + {'b': true}, + ], + }, + ), + isTrue, + ); + expect(chatJsonEquals({'a': 1}, {'a': 2}), isFalse); + expect(chatJsonEquals({'a': 1}, {'b': 1}), isFalse); + expect(chatJsonEquals([1, 2], [1, 2, 3]), isFalse); + expect(chatJsonEquals({'a': null}, {}), isFalse); + }); + + test('ChatRowWidgets reuses a widget until its row or inputs change', () { + final rows = ChatRowWidgets(); + var builds = 0; + Widget build() { + builds++; + return SizedBox(width: builds.toDouble()); + } + + final first = rows.widgetFor('k', {'text': 'a'}, [1], build); + expect(rows.widgetFor('k', {'text': 'a'}, [1], build), same(first)); + expect(builds, 1); + expect(rows.widgetFor('k', {'text': 'b'}, [1], build), isNot(same(first))); + final second = rows.widgetFor('k', {'text': 'b'}, [2], build); + expect(builds, 3); + rows.prune(); + // A row the last build did not ask for is forgotten. + rows.prune(); + expect(rows.widgetFor('k', {'text': 'b'}, [2], build), isNot(same(second))); + expect(builds, 4); + }); + + test('ChatMarkdownCache parses a text once and stays bounded', () { + final cache = ChatMarkdownCache(maxEntries: 2, maxChars: 100); + final blocks = cache.blocksFor('# One'); + expect(cache.blocksFor('# One'), same(blocks)); + expect(cache.parses, 1); + cache.blocksFor('two'); + cache.blocksFor('three'); + // The least recently used text was evicted and parses again. + expect(cache.blocksFor('# One'), isNot(same(blocks))); + expect(cache.parses, 4); + // A text larger than the character budget is parsed but never held. + final long = 'x' * 200; + cache.blocksFor(long); + cache.blocksFor(long); + expect(cache.parses, 6); + }); +} diff --git a/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart b/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart new file mode 100644 index 00000000..c6e30703 --- /dev/null +++ b/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart @@ -0,0 +1,159 @@ +import 'package:flutter/material.dart'; +import 'package:flutter/widgets.dart' as widgets; +import 'package:flutter_test/flutter_test.dart'; +import 'package:pipelines_app/core/api_gateway.dart'; +import 'package:pipelines_app/features/chat/conversation/chat_thread_view.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_cache.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_markdown_view.dart'; +import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart'; + +import '../conversation/chat_thread_fixtures.dart'; + +/// Measurement harness: streams 200 text deltas into a 300-message transcript +/// and counts message builds, markdown builds/parses and per-delta frame time; +/// then scrolls the long transcript up and back down. +const _messages = 300; +const _deltas = 200; + +String _answer(int index) => + '## Answer $index\n\nSome **bold** text with `code` and a ' + '[link](https://example.com/$index).\n\n' + '```dart\n' + '${List.generate(30, (line) => 'final v$line = $index * $line;').join('\n')}' + '\n```\n\n' + '- one\n- two\n- three\n'; + +List _transcript() => [ + for (var index = 0; index < _messages; index++) + // The newest message is the operator's, so the reply opens a new turn. + index.isOdd + ? chatMessage( + 'u$index', + 'user', + parts: [ + {'type': 'text', 'text': 'Question $index'}, + ], + ) + : chatMessage( + 'a$index', + 'assistant', + parts: [ + {'type': 'text', 'id': 't$index', 'text': _answer(index)}, + ], + ), +]; + +Future _open(WidgetTester tester) async { + tester.view.physicalSize = const Size(1000, 1600); + tester.view.devicePixelRatio = 1; + addTearDown(tester.view.reset); + final gateway = FakeChatGateway(); + addTearDown(gateway.close); + gateway.history = (_) async => chatPage(_transcript()); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ChatThreadView(gateway: gateway, sessionId: 's', onBack: () {}), + ), + ), + ); + final watch = Stopwatch()..start(); + await tester.pumpAndSettle(); + watch.stop(); + // ignore: avoid_print + print( + 'PERF open: parses=${chatMarkdownCache.parses} ' + 'settleMs=${watch.elapsedMilliseconds}', + ); + return gateway; +} + +String _stats(List samples) { + final sorted = [...samples]..sort(); + final mean = sorted.reduce((a, b) => a + b) / sorted.length; + return 'meanUs=${mean.round()} p50Us=${sorted[sorted.length ~/ 2]} ' + 'p90Us=${sorted[sorted.length * 9 ~/ 10]} maxUs=${sorted.last}'; +} + +void main() { + setUp(chatMarkdownCache.clear); + testWidgets('streaming deltas rebuild only the live turn', (tester) async { + final gateway = await _open(tester); + var messageBuilds = 0; + var markdownBuilds = 0; + var totalBuilds = 0; + widgets.debugOnRebuildDirtyWidget = (element, _) { + final widget = element.widget; + if (widget is ChatMessageView) messageBuilds++; + if (widget is ChatMarkdownView) markdownBuilds++; + totalBuilds++; + }; + addTearDown(() => widgets.debugOnRebuildDirtyWidget = null); + final parsesBefore = chatMarkdownCache.parses; + final perDelta = []; + final watch = Stopwatch(); + for (var index = 0; index < _deltas; index++) { + gateway.emitEvent( + chatTextFrame( + index == 0 ? 'Live reply ' : 'word$index ', + messageId: 'live', + ), + ); + await tester.idle(); + watch + ..reset() + ..start(); + await tester.pump(); + watch.stop(); + perDelta.add(watch.elapsedMicroseconds); + } + widgets.debugOnRebuildDirtyWidget = null; + final parses = chatMarkdownCache.parses - parsesBefore; + // ignore: avoid_print + print( + 'PERF stream: deltas=$_deltas messages=$_messages ' + 'messageBuilds=$messageBuilds markdownBuilds=$markdownBuilds ' + 'markdownParses=$parses totalBuilds=$totalBuilds frame ${_stats(perDelta)}', + ); + expect( + find.textContaining('word${_deltas - 1}', findRichText: true), + findsOneWidget, + ); + // Only the live turn rebuilds per delta (plus slack for the turn that + // starts the stream); every finished message is left alone. + expect(messageBuilds, lessThanOrEqualTo(_deltas + 10)); + expect(parses, lessThanOrEqualTo(_deltas + 10)); + }); + + testWidgets('scrolling a long transcript reuses parsed markdown', ( + tester, + ) async { + await _open(tester); + final list = find.byKey(const ValueKey('chat-thread-s')); + final parsesBefore = chatMarkdownCache.parses; + var parsesUp = 0; + final frames = []; + final watch = Stopwatch(); + // Up 40 drags and back down 40: the way back revisits parsed rows. + for (final dy in [...List.filled(40, 600.0), ...List.filled(40, -600.0)]) { + await tester.drag(list, Offset(0, dy)); + watch + ..reset() + ..start(); + await tester.pump(const Duration(milliseconds: 16)); + watch.stop(); + frames.add(watch.elapsedMicroseconds); + if (frames.length == 40) { + parsesUp = chatMarkdownCache.parses - parsesBefore; + } + } + await tester.pumpAndSettle(); + final parses = chatMarkdownCache.parses - parsesBefore; + // ignore: avoid_print + print( + 'PERF scroll: drags=${frames.length} markdownParses=$parses ' + '(up=$parsesUp down=${parses - parsesUp}) ' + 'frame ${_stats(frames)}', + ); + }); +} diff --git a/app/tool/chat_composer_probe_entry.dart b/app/tool/chat_composer_probe_entry.dart index 93360aca..e762a1e6 100644 --- a/app/tool/chat_composer_probe_entry.dart +++ b/app/tool/chat_composer_probe_entry.dart @@ -7,6 +7,8 @@ import 'package:pipelines_app/core/api_gateway.dart'; import 'package:pipelines_app/features/chat/conversation/chat_draft_store.dart'; import 'package:pipelines_app/features/chat/conversation/chat_thread_view.dart'; +import 'chat_perf_fixture.dart'; + /// A live-looking fixture conversation with a deterministic composer. /// /// A successful prompt starts a running turn (so the primary control becomes @@ -43,6 +45,8 @@ class _ProbeGateway implements ApiGateway { return { 'chat_history': { 'messages': [ + // `?perf=`: a long transcript for tool/chat_perf_probe.mjs. + ...chatPerfTranscript(chatPerfMessages), // A long history scrolls behind the floating header. if (_probeLong || _query['long'] == '1') for (var i = 0; i < 12; i++) @@ -62,28 +66,31 @@ class _ProbeGateway implements ApiGateway { // text parts and a second folded message, each with markdown // paragraphs. if (_query['paras'] == '1') ..._chunkedReply, - { - 'id': 'probe-user', - 'role': 'user', - 'parts': [ - { - 'type': 'text', - 'text': 'Show me the composer as an operator would use it.', - }, - ], - }, - { - 'id': 'probe-assistant', - 'role': 'assistant', - 'parts': [ - { - 'type': 'text', - 'text': - 'The composer sits below the conversation with a ' - 'multiline field and one primary control.', - }, - ], - }, + if (chatPerfMessages == 0) ...[ + { + 'id': 'probe-user', + 'role': 'user', + 'parts': [ + { + 'type': 'text', + 'text': + 'Show me the composer as an operator would use it.', + }, + ], + }, + { + 'id': 'probe-assistant', + 'role': 'assistant', + 'parts': [ + { + 'type': 'text', + 'text': + 'The composer sits below the conversation with a ' + 'multiline field and one primary control.', + }, + ], + }, + ], ], 'next_cursor': '', 'has_more': false, @@ -124,6 +131,8 @@ class _ProbeGateway implements ApiGateway { 'step': {'status': 'ended', 'turn_id': 'probe-turn', 'intermediate': false}, }); + void emit(JsonMap body) => _emit(body); + void _emit(JsonMap body) => _frames.add({ 'type': 'chat_event', 'chat_event': {'session_id': _session, ...body}, @@ -200,7 +209,8 @@ const _probeStyle = String.fromEnvironment( /// Browser overrides: `?platform=cupertino|material`, `?theme=dark|light`, /// `?long=1`, `?paras=1` (a chunked multi-paragraph reply), `?tasks=`, `?mode=0` (no permission mode recorded) and -/// `?model=0` (a backend without models). +/// `?model=0` (a backend without models). `?perf=` and `?stream=` are +/// the long-transcript performance fixture (see chat_perf_fixture.dart). /// /// Delivery states: `?send=` types and sends that text once the thread /// has loaded (`hold` never replies, `queued` queues, `refuse` is refused), and @@ -244,6 +254,7 @@ void main() { ), newOperationId: gateway.nextOperation, ); + startChatPerf(gateway.emit); if (_query['send'] != null) { Timer( const Duration(milliseconds: 1200), diff --git a/app/tool/chat_perf_fixture.dart b/app/tool/chat_perf_fixture.dart new file mode 100644 index 00000000..28330e70 --- /dev/null +++ b/app/tool/chat_perf_fixture.dart @@ -0,0 +1,82 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +import 'package:pipelines_app/core/api_gateway.dart'; + +/// Long-transcript performance fixture for `tool/chat_composer_probe_entry.dart`. +/// +/// `?perf=` opens a transcript of n messages whose answers carry headings, +/// inline markdown, a 30-line code block and a list. `?stream=` then streams +/// n text deltas into a new reply, one every 40 ms, starting 6 s after load. +/// With either, every frame's build and raster durations are printed to the +/// console as `PERF_FRAME ` for +/// `tool/chat_perf_probe.mjs`. +int get chatPerfMessages => + int.tryParse(Uri.base.queryParameters['perf'] ?? '') ?? 0; +int get chatPerfDeltas => + int.tryParse(Uri.base.queryParameters['stream'] ?? '') ?? 0; + +String _answer(int index) => + '## Answer $index\n\nSome **bold** text with `code` and a ' + '[link](https://example.com/$index).\n\n' + '```dart\n' + '${List.generate(30, (line) => 'final v$line = $index * $line;').join('\n')}' + '\n```\n\n' + '- one\n- two\n- three\n'; + +/// The fixture history; the newest message is the operator's, so a streamed +/// reply opens a new turn. +List chatPerfTranscript(int count) => [ + for (var index = 0; index < count; index++) + (count - index).isOdd + ? { + 'id': 'perf-u$index', + 'role': 'user', + 'parts': [ + {'type': 'text', 'text': 'Question $index'}, + ], + } + : { + 'id': 'perf-a$index', + 'role': 'assistant', + 'parts': [ + {'type': 'text', 'id': 'perf-t$index', 'text': _answer(index)}, + ], + }, +]; + +/// Starts the frame report and, when asked, the streamed reply through [emit]. +void startChatPerf(void Function(JsonMap body) emit) { + if (chatPerfMessages == 0 && chatPerfDeltas == 0) return; + WidgetsFlutterBinding.ensureInitialized().addTimingsCallback((timings) { + for (final timing in timings) { + // ignore: avoid_print + print( + 'PERF_FRAME ${timing.buildDuration.inMicroseconds} ' + '${timing.rasterDuration.inMicroseconds} ' + '${timing.totalSpan.inMicroseconds}', + ); + } + }); + final deltas = chatPerfDeltas; + if (deltas == 0) return; + Timer(const Duration(milliseconds: 6000), () { + // ignore: avoid_print + print('PERF_STREAM_START'); + var sent = 0; + Timer.periodic(const Duration(milliseconds: 40), (timer) { + emit({ + 'message_id': 'perf-live', + 'kind': 'text', + 'part': {'id': 'perf-live-text', 'type': 'text'}, + 'delta': sent == 0 ? 'Live reply ' : 'word$sent ', + }); + if (++sent >= deltas) { + timer.cancel(); + // ignore: avoid_print + print('PERF_STREAM_END'); + } + }); + }); +} diff --git a/app/tool/chat_perf_probe.mjs b/app/tool/chat_perf_probe.mjs new file mode 100644 index 00000000..e2ae50df --- /dev/null +++ b/app/tool/chat_perf_probe.mjs @@ -0,0 +1,108 @@ +// Release-web frame timings for a long chat transcript, over the capture kit. +// +// Build the fixture once per variant, from app/: +// flutter build web --release --target tool/chat_composer_probe_entry.dart \ +// --output build/chat-perf +// then: +// node tool/chat_perf_probe.mjs build/chat-perf [messages=300] [deltas=200] +// +// Two phases, each on a fresh page: wheel-scroll the transcript up and back +// down, then stream deltas into a new reply. Flutter's own FrameTiming (build, +// raster, total) arrives as console lines from tool/chat_perf_fixture.dart. +// Prints one JSON report. Headless Chrome rasterises in software, so raster +// numbers compare variants rather than predict a device. +import { setTimeout as delay } from 'node:timers/promises'; +import { openBrowser } from './capture/browser.mjs'; +import { flutterPage } from './capture/page.mjs'; +import { serveBuild } from './capture/serve.mjs'; + +const [dir, messagesArg = '300', deltasArg = '200'] = process.argv.slice(2); +if (!dir) { + console.error('usage: node tool/chat_perf_probe.mjs [messages] [deltas]'); + process.exit(2); +} +const messages = Number(messagesArg); +const deltas = Number(deltasArg); + +function stats(frames) { + const pick = (i) => frames.map((f) => f[i]).sort((a, b) => a - b); + const summary = (values) => values.length === 0 ? null : { + meanMs: +(values.reduce((a, b) => a + b, 0) / values.length / 1000).toFixed(2), + p50Ms: +(values[Math.floor(values.length / 2)] / 1000).toFixed(2), + p90Ms: +(values[Math.floor(values.length * 0.9)] / 1000).toFixed(2), + maxMs: +(values[values.length - 1] / 1000).toFixed(2), + }; + const total = pick(2); + return { + frames: frames.length, + build: summary(pick(0)), + raster: summary(pick(1)), + total: summary(total), + over16ms: total.filter((t) => t > 16667).length, + }; +} + +async function phase(origin, query, drive) { + const browser = await openBrowser(); + try { + const page = flutterPage(browser, (m) => console.error(m)); + let recording = false; + let frames = []; + const marks = []; + browser.on('Runtime.consoleAPICalled', (p) => { + const text = p.args.map((a) => a.value ?? '').join(' '); + if (text.startsWith('PERF_FRAME ')) { + if (recording) frames.push(text.split(' ').slice(1).map(Number)); + } else if (text.startsWith('PERF_')) { + marks.push(text); + } + }); + const opened = Date.now(); + try { + await page.open(`${origin}/?${query}`, { width: 880, height: 1000 }); + } catch (error) { + throw new Error(`${error.message}; page errors: ${JSON.stringify(page.errors.slice(0, 5))}`); + } + const openMs = Date.now() - opened; + await delay(1000); + const control = { + page, + marks, + start: () => { frames = []; recording = true; }, + stop: () => { recording = false; return frames; }, + }; + const result = await drive(control); + return { openMs, ...result, errors: page.errors.slice(0, 5) }; + } finally { + await browser.close(); + } +} + +const server = await serveBuild(dir); +try { + const scroll = await phase(server.origin, `perf=${messages}`, async ({ page, start, stop }) => { + start(); + for (const dy of [...Array(60).fill(-400), ...Array(60).fill(400)]) { + await page.scroll(dy); + await delay(50); + } + await delay(500); + return { scroll: stats(stop()) }; + }); + const stream = await phase( + server.origin, + `perf=${messages}&stream=${deltas}`, + async ({ start, stop, marks }) => { + const begin = Date.now() + 15000; + while (!marks.includes('PERF_STREAM_START') && Date.now() < begin) await delay(5); + start(); + const end = Date.now() + deltas * 40 + 15000; + while (!marks.includes('PERF_STREAM_END') && Date.now() < end) await delay(100); + await delay(500); + return { streamed: marks.includes('PERF_STREAM_END'), stream: stats(stop()) }; + }, + ); + console.log(JSON.stringify({ build: dir, messages, deltas, scroll, stream }, null, 2)); +} finally { + await server.close(); +}