Measure chat transcript open cost in the perf harness and web probe

This commit is contained in:
2026-11-02 00:15:40 +00:00
parent 52e31fc4ec
commit b80c52b3e1
4 changed files with 139 additions and 9 deletions
@@ -9,9 +9,11 @@ import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart
import '../conversation/chat_thread_fixtures.dart';
/// Measurement harness: streams 200 text deltas into a 300-message transcript
/// and counts message builds, markdown builds/parses and per-delta frame time;
/// then scrolls the long transcript up and back down.
/// Measurement harness: opens 300- and 1000-message transcripts and counts the
/// rows built and parsed until the newest message shows; streams 200 text
/// deltas into a 300-message transcript and counts message builds, markdown
/// builds/parses and per-delta frame time; then scrolls the long transcript up
/// and back down.
const _messages = 300;
const _deltas = 200;
@@ -23,8 +25,8 @@ String _answer(int index) =>
'\n```\n\n'
'- one\n- two\n- three\n';
List<JsonMap> _transcript() => [
for (var index = 0; index < _messages; index++)
List<JsonMap> _transcript([int count = _messages]) => [
for (var index = 0; index < count; index++)
// The newest message is the operator's, so the reply opens a new turn.
index.isOdd
? chatMessage(
@@ -77,6 +79,92 @@ String _stats(List<int> samples) {
void main() {
setUp(chatMarkdownCache.clear);
for (final count in [300, 1000]) {
testWidgets('opening a $count-message transcript builds what shows', (
tester,
) async {
tester.view.physicalSize = const Size(1000, 1600);
tester.view.devicePixelRatio = 1;
addTearDown(tester.view.reset);
final gateway = FakeChatGateway();
addTearDown(gateway.close);
gateway.history = (_) async => chatPage(_transcript(count));
var messageBuilds = 0;
widgets.debugOnRebuildDirtyWidget = (element, _) {
if (element.widget is ChatMessageView) messageBuilds++;
};
addTearDown(() => widgets.debugOnRebuildDirtyWidget = null);
// The newest message is the operator's question.
final newestId = 'u${count - 1}';
final newest = find.byWidgetPredicate(
(widget) =>
widget is ChatMessageView &&
(widget.row['key'] == newestId ||
widget.row['message']?['id'] == newestId),
);
final watch = Stopwatch()..start();
await tester.pumpWidget(
MaterialApp(
home: Scaffold(
body: ChatThreadView(
gateway: gateway,
sessionId: 's',
onBack: () {},
),
),
),
);
// From the first frame that builds a message row to the frame that shows
// the newest one: the open cost the operator waits through.
int? firstRowFrame;
int? shownFrame;
var openUs = 0;
var maxFrameUs = 0;
bool newestShows() {
if (newest.evaluate().isEmpty) return false;
final rect = tester.getRect(newest);
return rect.bottom > 0 && rect.top < 1600;
}
final frame = Stopwatch();
for (var index = 1; index <= 300 && shownFrame == null; index++) {
final rowsBefore = messageBuilds;
frame
..reset()
..start();
await tester.pump(const Duration(milliseconds: 16));
frame.stop();
if (firstRowFrame == null && messageBuilds > rowsBefore) {
firstRowFrame = index;
}
if (firstRowFrame != null) {
openUs += frame.elapsedMicroseconds;
if (frame.elapsedMicroseconds > maxFrameUs) {
maxFrameUs = frame.elapsedMicroseconds;
}
}
if (newestShows()) shownFrame = index;
}
final rowsAtShow = messageBuilds;
final parsesAtShow = chatMarkdownCache.parses;
await tester.pumpAndSettle();
watch.stop();
widgets.debugOnRebuildDirtyWidget = null;
// ignore: avoid_print
print(
'PERF open$count: rowsBuiltAtShow=$rowsAtShow '
'parsesAtShow=$parsesAtShow '
'framesToShow=${shownFrame == null ? null : shownFrame - firstRowFrame! + 1} '
'openMs=${(openUs / 1000).toStringAsFixed(1)} '
'maxFrameMs=${(maxFrameUs / 1000).toStringAsFixed(1)} '
'settled rows=$messageBuilds parses=${chatMarkdownCache.parses} '
'wallMs=${watch.elapsedMilliseconds}',
);
expect(shownFrame, isNotNull);
expect(tester.getRect(newest).bottom, lessThanOrEqualTo(1600));
});
}
testWidgets('streaming deltas rebuild only the live turn', (tester) async {
final gateway = await _open(tester);
var messageBuilds = 0;
+1
View File
@@ -42,6 +42,7 @@ class _ProbeGateway implements ApiGateway {
],
};
case 'chat_messages':
markChatPerfHistory();
return {
'chat_history': {
'messages': [
+9 -1
View File
@@ -11,7 +11,7 @@ import 'package:pipelines_app/core/api_gateway.dart';
/// n text deltas into a new reply, one every 40 ms, starting 6 s after load.
/// With either, every frame's build and raster durations are printed to the
/// console as `PERF_FRAME <build us> <raster us> <total us>` for
/// `tool/chat_perf_probe.mjs`.
/// `tool/chat_perf_probe.mjs`, and the history reply as `PERF_HISTORY`.
int get chatPerfMessages =>
int.tryParse(Uri.base.queryParameters['perf'] ?? '') ?? 0;
int get chatPerfDeltas =>
@@ -46,6 +46,14 @@ List<JsonMap> chatPerfTranscript(int count) => [
},
];
/// Marks the history reply for the probe's open phase: the frames after it are
/// the cost of opening the transcript.
void markChatPerfHistory() {
if (chatPerfMessages == 0) return;
// ignore: avoid_print
print('PERF_HISTORY');
}
/// Starts the frame report and, when asked, the streamed reply through [emit].
void startChatPerf(void Function(JsonMap body) emit) {
if (chatPerfMessages == 0 && chatPerfDeltas == 0) return;
+36 -3
View File
@@ -6,8 +6,9 @@
// then:
// node tool/chat_perf_probe.mjs build/chat-perf [messages=300] [deltas=200]
//
// Two phases, each on a fresh page: wheel-scroll the transcript up and back
// down, then stream deltas into a new reply. Flutter's own FrameTiming (build,
// Three phases, each on a fresh page: open the transcript (the frames after the
// history reply until the page is quiet), wheel-scroll it up and back down,
// then stream deltas into a new reply. Flutter's own FrameTiming (build,
// raster, total) arrives as console lines from tool/chat_perf_fixture.dart.
// Prints one JSON report. Headless Chrome rasterises in software, so raster
// numbers compare variants rather than predict a device.
@@ -49,12 +50,19 @@ async function phase(origin, query, drive) {
let recording = false;
let frames = [];
const marks = [];
// Every frame since navigation, for the open phase.
const allFrames = [];
let historyAt = null;
browser.on('Runtime.consoleAPICalled', (p) => {
const text = p.args.map((a) => a.value ?? '').join(' ');
if (text.startsWith('PERF_FRAME ')) {
if (recording) frames.push(text.split(' ').slice(1).map(Number));
} else if (text.startsWith('PERF_')) {
marks.push(text);
if (text === 'PERF_HISTORY') historyAt = { at: Date.now(), frame: allFrames.length };
}
if (text.startsWith('PERF_FRAME ')) {
allFrames.push({ at: Date.now(), f: text.split(' ').slice(1).map(Number) });
}
});
const opened = Date.now();
@@ -70,6 +78,7 @@ async function phase(origin, query, drive) {
marks,
start: () => { frames = []; recording = true; },
stop: () => { recording = false; return frames; },
opened: () => ({ historyAt, allFrames }),
};
const result = await drive(control);
return { openMs, ...result, errors: page.errors.slice(0, 5) };
@@ -80,6 +89,30 @@ async function phase(origin, query, drive) {
const server = await serveBuild(dir);
try {
const open = await phase(server.origin, `perf=${messages}`, async ({ opened }) => {
// The page is quiet once no frame has arrived for a second.
const end = Date.now() + 20000;
for (;;) {
const { historyAt, allFrames } = opened();
const last = allFrames.at(-1)?.at ?? 0;
if (historyAt && Date.now() - Math.max(last, historyAt.at) > 1000) break;
if (Date.now() > end) break;
await delay(100);
}
const { historyAt, allFrames } = opened();
if (!historyAt) return { open: null };
const after = allFrames.slice(historyAt.frame);
const sum = (i) => +(after.reduce((a, x) => a + x.f[i], 0) / 1000).toFixed(1);
return {
open: {
...stats(after.map((x) => x.f)),
buildSumMs: sum(0),
totalSumMs: sum(2),
// Wall time from the history reply to the last frame of the burst.
settleMs: after.length ? after.at(-1).at - historyAt.at : 0,
},
};
});
const scroll = await phase(server.origin, `perf=${messages}`, async ({ page, start, stop }) => {
start();
for (const dy of [...Array(60).fill(-400), ...Array(60).fill(400)]) {
@@ -102,7 +135,7 @@ try {
return { streamed: marks.includes('PERF_STREAM_END'), stream: stats(stop()) };
},
);
console.log(JSON.stringify({ build: dir, messages, deltas, scroll, stream }, null, 2));
console.log(JSON.stringify({ build: dir, messages, deltas, open, scroll, stream }, null, 2));
} finally {
await server.close();
}