diff --git a/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart b/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart index c6e30703..e269f614 100644 --- a/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart +++ b/app/test/features/chat/perf/chat_stream_rebuild_perf_test.dart @@ -9,9 +9,11 @@ import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart import '../conversation/chat_thread_fixtures.dart'; -/// Measurement harness: streams 200 text deltas into a 300-message transcript -/// and counts message builds, markdown builds/parses and per-delta frame time; -/// then scrolls the long transcript up and back down. +/// Measurement harness: opens 300- and 1000-message transcripts and counts the +/// rows built and parsed until the newest message shows; streams 200 text +/// deltas into a 300-message transcript and counts message builds, markdown +/// builds/parses and per-delta frame time; then scrolls the long transcript up +/// and back down. const _messages = 300; const _deltas = 200; @@ -23,8 +25,8 @@ String _answer(int index) => '\n```\n\n' '- one\n- two\n- three\n'; -List _transcript() => [ - for (var index = 0; index < _messages; index++) +List _transcript([int count = _messages]) => [ + for (var index = 0; index < count; index++) // The newest message is the operator's, so the reply opens a new turn. index.isOdd ? chatMessage( @@ -77,6 +79,92 @@ String _stats(List samples) { void main() { setUp(chatMarkdownCache.clear); + for (final count in [300, 1000]) { + testWidgets('opening a $count-message transcript builds what shows', ( + tester, + ) async { + tester.view.physicalSize = const Size(1000, 1600); + tester.view.devicePixelRatio = 1; + addTearDown(tester.view.reset); + final gateway = FakeChatGateway(); + addTearDown(gateway.close); + gateway.history = (_) async => chatPage(_transcript(count)); + var messageBuilds = 0; + widgets.debugOnRebuildDirtyWidget = (element, _) { + if (element.widget is ChatMessageView) messageBuilds++; + }; + addTearDown(() => widgets.debugOnRebuildDirtyWidget = null); + // The newest message is the operator's question. + final newestId = 'u${count - 1}'; + final newest = find.byWidgetPredicate( + (widget) => + widget is ChatMessageView && + (widget.row['key'] == newestId || + widget.row['message']?['id'] == newestId), + ); + final watch = Stopwatch()..start(); + await tester.pumpWidget( + MaterialApp( + home: Scaffold( + body: ChatThreadView( + gateway: gateway, + sessionId: 's', + onBack: () {}, + ), + ), + ), + ); + // From the first frame that builds a message row to the frame that shows + // the newest one: the open cost the operator waits through. + int? firstRowFrame; + int? shownFrame; + var openUs = 0; + var maxFrameUs = 0; + bool newestShows() { + if (newest.evaluate().isEmpty) return false; + final rect = tester.getRect(newest); + return rect.bottom > 0 && rect.top < 1600; + } + + final frame = Stopwatch(); + for (var index = 1; index <= 300 && shownFrame == null; index++) { + final rowsBefore = messageBuilds; + frame + ..reset() + ..start(); + await tester.pump(const Duration(milliseconds: 16)); + frame.stop(); + if (firstRowFrame == null && messageBuilds > rowsBefore) { + firstRowFrame = index; + } + if (firstRowFrame != null) { + openUs += frame.elapsedMicroseconds; + if (frame.elapsedMicroseconds > maxFrameUs) { + maxFrameUs = frame.elapsedMicroseconds; + } + } + if (newestShows()) shownFrame = index; + } + final rowsAtShow = messageBuilds; + final parsesAtShow = chatMarkdownCache.parses; + await tester.pumpAndSettle(); + watch.stop(); + widgets.debugOnRebuildDirtyWidget = null; + // ignore: avoid_print + print( + 'PERF open$count: rowsBuiltAtShow=$rowsAtShow ' + 'parsesAtShow=$parsesAtShow ' + 'framesToShow=${shownFrame == null ? null : shownFrame - firstRowFrame! + 1} ' + 'openMs=${(openUs / 1000).toStringAsFixed(1)} ' + 'maxFrameMs=${(maxFrameUs / 1000).toStringAsFixed(1)} ' + 'settled rows=$messageBuilds parses=${chatMarkdownCache.parses} ' + 'wallMs=${watch.elapsedMilliseconds}', + ); + expect(shownFrame, isNotNull); + expect(tester.getRect(newest).bottom, lessThanOrEqualTo(1600)); + }); + } + testWidgets('streaming deltas rebuild only the live turn', (tester) async { final gateway = await _open(tester); var messageBuilds = 0; diff --git a/app/tool/chat_composer_probe_entry.dart b/app/tool/chat_composer_probe_entry.dart index e762a1e6..1d6b7219 100644 --- a/app/tool/chat_composer_probe_entry.dart +++ b/app/tool/chat_composer_probe_entry.dart @@ -42,6 +42,7 @@ class _ProbeGateway implements ApiGateway { ], }; case 'chat_messages': + markChatPerfHistory(); return { 'chat_history': { 'messages': [ diff --git a/app/tool/chat_perf_fixture.dart b/app/tool/chat_perf_fixture.dart index 28330e70..63b70b1d 100644 --- a/app/tool/chat_perf_fixture.dart +++ b/app/tool/chat_perf_fixture.dart @@ -11,7 +11,7 @@ import 'package:pipelines_app/core/api_gateway.dart'; /// n text deltas into a new reply, one every 40 ms, starting 6 s after load. /// With either, every frame's build and raster durations are printed to the /// console as `PERF_FRAME ` for -/// `tool/chat_perf_probe.mjs`. +/// `tool/chat_perf_probe.mjs`, and the history reply as `PERF_HISTORY`. int get chatPerfMessages => int.tryParse(Uri.base.queryParameters['perf'] ?? '') ?? 0; int get chatPerfDeltas => @@ -46,6 +46,14 @@ List chatPerfTranscript(int count) => [ }, ]; +/// Marks the history reply for the probe's open phase: the frames after it are +/// the cost of opening the transcript. +void markChatPerfHistory() { + if (chatPerfMessages == 0) return; + // ignore: avoid_print + print('PERF_HISTORY'); +} + /// Starts the frame report and, when asked, the streamed reply through [emit]. void startChatPerf(void Function(JsonMap body) emit) { if (chatPerfMessages == 0 && chatPerfDeltas == 0) return; diff --git a/app/tool/chat_perf_probe.mjs b/app/tool/chat_perf_probe.mjs index e2ae50df..ed02383e 100644 --- a/app/tool/chat_perf_probe.mjs +++ b/app/tool/chat_perf_probe.mjs @@ -6,8 +6,9 @@ // then: // node tool/chat_perf_probe.mjs build/chat-perf [messages=300] [deltas=200] // -// Two phases, each on a fresh page: wheel-scroll the transcript up and back -// down, then stream deltas into a new reply. Flutter's own FrameTiming (build, +// Three phases, each on a fresh page: open the transcript (the frames after the +// history reply until the page is quiet), wheel-scroll it up and back down, +// then stream deltas into a new reply. Flutter's own FrameTiming (build, // raster, total) arrives as console lines from tool/chat_perf_fixture.dart. // Prints one JSON report. Headless Chrome rasterises in software, so raster // numbers compare variants rather than predict a device. @@ -49,12 +50,19 @@ async function phase(origin, query, drive) { let recording = false; let frames = []; const marks = []; + // Every frame since navigation, for the open phase. + const allFrames = []; + let historyAt = null; browser.on('Runtime.consoleAPICalled', (p) => { const text = p.args.map((a) => a.value ?? '').join(' '); if (text.startsWith('PERF_FRAME ')) { if (recording) frames.push(text.split(' ').slice(1).map(Number)); } else if (text.startsWith('PERF_')) { marks.push(text); + if (text === 'PERF_HISTORY') historyAt = { at: Date.now(), frame: allFrames.length }; + } + if (text.startsWith('PERF_FRAME ')) { + allFrames.push({ at: Date.now(), f: text.split(' ').slice(1).map(Number) }); } }); const opened = Date.now(); @@ -70,6 +78,7 @@ async function phase(origin, query, drive) { marks, start: () => { frames = []; recording = true; }, stop: () => { recording = false; return frames; }, + opened: () => ({ historyAt, allFrames }), }; const result = await drive(control); return { openMs, ...result, errors: page.errors.slice(0, 5) }; @@ -80,6 +89,30 @@ async function phase(origin, query, drive) { const server = await serveBuild(dir); try { + const open = await phase(server.origin, `perf=${messages}`, async ({ opened }) => { + // The page is quiet once no frame has arrived for a second. + const end = Date.now() + 20000; + for (;;) { + const { historyAt, allFrames } = opened(); + const last = allFrames.at(-1)?.at ?? 0; + if (historyAt && Date.now() - Math.max(last, historyAt.at) > 1000) break; + if (Date.now() > end) break; + await delay(100); + } + const { historyAt, allFrames } = opened(); + if (!historyAt) return { open: null }; + const after = allFrames.slice(historyAt.frame); + const sum = (i) => +(after.reduce((a, x) => a + x.f[i], 0) / 1000).toFixed(1); + return { + open: { + ...stats(after.map((x) => x.f)), + buildSumMs: sum(0), + totalSumMs: sum(2), + // Wall time from the history reply to the last frame of the burst. + settleMs: after.length ? after.at(-1).at - historyAt.at : 0, + }, + }; + }); const scroll = await phase(server.origin, `perf=${messages}`, async ({ page, start, stop }) => { start(); for (const dy of [...Array(60).fill(-400), ...Array(60).fill(400)]) { @@ -102,7 +135,7 @@ try { return { streamed: marks.includes('PERF_STREAM_END'), stream: stats(stop()) }; }, ); - console.log(JSON.stringify({ build: dir, messages, deltas, scroll, stream }, null, 2)); + console.log(JSON.stringify({ build: dir, messages, deltas, open, scroll, stream }, null, 2)); } finally { await server.close(); }