Measure chat transcript open cost in the perf harness and web probe
This commit is contained in:
@@ -9,9 +9,11 @@ import 'package:pipelines_app/features/chat/ui/transcript/chat_message_view.dart
|
||||
|
||||
import '../conversation/chat_thread_fixtures.dart';
|
||||
|
||||
/// Measurement harness: streams 200 text deltas into a 300-message transcript
|
||||
/// and counts message builds, markdown builds/parses and per-delta frame time;
|
||||
/// then scrolls the long transcript up and back down.
|
||||
/// Measurement harness: opens 300- and 1000-message transcripts and counts the
|
||||
/// rows built and parsed until the newest message shows; streams 200 text
|
||||
/// deltas into a 300-message transcript and counts message builds, markdown
|
||||
/// builds/parses and per-delta frame time; then scrolls the long transcript up
|
||||
/// and back down.
|
||||
const _messages = 300;
|
||||
const _deltas = 200;
|
||||
|
||||
@@ -23,8 +25,8 @@ String _answer(int index) =>
|
||||
'\n```\n\n'
|
||||
'- one\n- two\n- three\n';
|
||||
|
||||
List<JsonMap> _transcript() => [
|
||||
for (var index = 0; index < _messages; index++)
|
||||
List<JsonMap> _transcript([int count = _messages]) => [
|
||||
for (var index = 0; index < count; index++)
|
||||
// The newest message is the operator's, so the reply opens a new turn.
|
||||
index.isOdd
|
||||
? chatMessage(
|
||||
@@ -77,6 +79,92 @@ String _stats(List<int> samples) {
|
||||
|
||||
void main() {
|
||||
setUp(chatMarkdownCache.clear);
|
||||
for (final count in [300, 1000]) {
|
||||
testWidgets('opening a $count-message transcript builds what shows', (
|
||||
tester,
|
||||
) async {
|
||||
tester.view.physicalSize = const Size(1000, 1600);
|
||||
tester.view.devicePixelRatio = 1;
|
||||
addTearDown(tester.view.reset);
|
||||
final gateway = FakeChatGateway();
|
||||
addTearDown(gateway.close);
|
||||
gateway.history = (_) async => chatPage(_transcript(count));
|
||||
var messageBuilds = 0;
|
||||
widgets.debugOnRebuildDirtyWidget = (element, _) {
|
||||
if (element.widget is ChatMessageView) messageBuilds++;
|
||||
};
|
||||
addTearDown(() => widgets.debugOnRebuildDirtyWidget = null);
|
||||
// The newest message is the operator's question.
|
||||
final newestId = 'u${count - 1}';
|
||||
final newest = find.byWidgetPredicate(
|
||||
(widget) =>
|
||||
widget is ChatMessageView &&
|
||||
(widget.row['key'] == newestId ||
|
||||
widget.row['message']?['id'] == newestId),
|
||||
);
|
||||
final watch = Stopwatch()..start();
|
||||
await tester.pumpWidget(
|
||||
MaterialApp(
|
||||
home: Scaffold(
|
||||
body: ChatThreadView(
|
||||
gateway: gateway,
|
||||
sessionId: 's',
|
||||
onBack: () {},
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
// From the first frame that builds a message row to the frame that shows
|
||||
// the newest one: the open cost the operator waits through.
|
||||
int? firstRowFrame;
|
||||
int? shownFrame;
|
||||
var openUs = 0;
|
||||
var maxFrameUs = 0;
|
||||
bool newestShows() {
|
||||
if (newest.evaluate().isEmpty) return false;
|
||||
final rect = tester.getRect(newest);
|
||||
return rect.bottom > 0 && rect.top < 1600;
|
||||
}
|
||||
|
||||
final frame = Stopwatch();
|
||||
for (var index = 1; index <= 300 && shownFrame == null; index++) {
|
||||
final rowsBefore = messageBuilds;
|
||||
frame
|
||||
..reset()
|
||||
..start();
|
||||
await tester.pump(const Duration(milliseconds: 16));
|
||||
frame.stop();
|
||||
if (firstRowFrame == null && messageBuilds > rowsBefore) {
|
||||
firstRowFrame = index;
|
||||
}
|
||||
if (firstRowFrame != null) {
|
||||
openUs += frame.elapsedMicroseconds;
|
||||
if (frame.elapsedMicroseconds > maxFrameUs) {
|
||||
maxFrameUs = frame.elapsedMicroseconds;
|
||||
}
|
||||
}
|
||||
if (newestShows()) shownFrame = index;
|
||||
}
|
||||
final rowsAtShow = messageBuilds;
|
||||
final parsesAtShow = chatMarkdownCache.parses;
|
||||
await tester.pumpAndSettle();
|
||||
watch.stop();
|
||||
widgets.debugOnRebuildDirtyWidget = null;
|
||||
// ignore: avoid_print
|
||||
print(
|
||||
'PERF open$count: rowsBuiltAtShow=$rowsAtShow '
|
||||
'parsesAtShow=$parsesAtShow '
|
||||
'framesToShow=${shownFrame == null ? null : shownFrame - firstRowFrame! + 1} '
|
||||
'openMs=${(openUs / 1000).toStringAsFixed(1)} '
|
||||
'maxFrameMs=${(maxFrameUs / 1000).toStringAsFixed(1)} '
|
||||
'settled rows=$messageBuilds parses=${chatMarkdownCache.parses} '
|
||||
'wallMs=${watch.elapsedMilliseconds}',
|
||||
);
|
||||
expect(shownFrame, isNotNull);
|
||||
expect(tester.getRect(newest).bottom, lessThanOrEqualTo(1600));
|
||||
});
|
||||
}
|
||||
|
||||
testWidgets('streaming deltas rebuild only the live turn', (tester) async {
|
||||
final gateway = await _open(tester);
|
||||
var messageBuilds = 0;
|
||||
|
||||
@@ -42,6 +42,7 @@ class _ProbeGateway implements ApiGateway {
|
||||
],
|
||||
};
|
||||
case 'chat_messages':
|
||||
markChatPerfHistory();
|
||||
return {
|
||||
'chat_history': {
|
||||
'messages': [
|
||||
|
||||
@@ -11,7 +11,7 @@ import 'package:pipelines_app/core/api_gateway.dart';
|
||||
/// n text deltas into a new reply, one every 40 ms, starting 6 s after load.
|
||||
/// With either, every frame's build and raster durations are printed to the
|
||||
/// console as `PERF_FRAME <build us> <raster us> <total us>` for
|
||||
/// `tool/chat_perf_probe.mjs`.
|
||||
/// `tool/chat_perf_probe.mjs`, and the history reply as `PERF_HISTORY`.
|
||||
int get chatPerfMessages =>
|
||||
int.tryParse(Uri.base.queryParameters['perf'] ?? '') ?? 0;
|
||||
int get chatPerfDeltas =>
|
||||
@@ -46,6 +46,14 @@ List<JsonMap> chatPerfTranscript(int count) => [
|
||||
},
|
||||
];
|
||||
|
||||
/// Marks the history reply for the probe's open phase: the frames after it are
|
||||
/// the cost of opening the transcript.
|
||||
void markChatPerfHistory() {
|
||||
if (chatPerfMessages == 0) return;
|
||||
// ignore: avoid_print
|
||||
print('PERF_HISTORY');
|
||||
}
|
||||
|
||||
/// Starts the frame report and, when asked, the streamed reply through [emit].
|
||||
void startChatPerf(void Function(JsonMap body) emit) {
|
||||
if (chatPerfMessages == 0 && chatPerfDeltas == 0) return;
|
||||
|
||||
@@ -6,8 +6,9 @@
|
||||
// then:
|
||||
// node tool/chat_perf_probe.mjs build/chat-perf [messages=300] [deltas=200]
|
||||
//
|
||||
// Two phases, each on a fresh page: wheel-scroll the transcript up and back
|
||||
// down, then stream deltas into a new reply. Flutter's own FrameTiming (build,
|
||||
// Three phases, each on a fresh page: open the transcript (the frames after the
|
||||
// history reply until the page is quiet), wheel-scroll it up and back down,
|
||||
// then stream deltas into a new reply. Flutter's own FrameTiming (build,
|
||||
// raster, total) arrives as console lines from tool/chat_perf_fixture.dart.
|
||||
// Prints one JSON report. Headless Chrome rasterises in software, so raster
|
||||
// numbers compare variants rather than predict a device.
|
||||
@@ -49,12 +50,19 @@ async function phase(origin, query, drive) {
|
||||
let recording = false;
|
||||
let frames = [];
|
||||
const marks = [];
|
||||
// Every frame since navigation, for the open phase.
|
||||
const allFrames = [];
|
||||
let historyAt = null;
|
||||
browser.on('Runtime.consoleAPICalled', (p) => {
|
||||
const text = p.args.map((a) => a.value ?? '').join(' ');
|
||||
if (text.startsWith('PERF_FRAME ')) {
|
||||
if (recording) frames.push(text.split(' ').slice(1).map(Number));
|
||||
} else if (text.startsWith('PERF_')) {
|
||||
marks.push(text);
|
||||
if (text === 'PERF_HISTORY') historyAt = { at: Date.now(), frame: allFrames.length };
|
||||
}
|
||||
if (text.startsWith('PERF_FRAME ')) {
|
||||
allFrames.push({ at: Date.now(), f: text.split(' ').slice(1).map(Number) });
|
||||
}
|
||||
});
|
||||
const opened = Date.now();
|
||||
@@ -70,6 +78,7 @@ async function phase(origin, query, drive) {
|
||||
marks,
|
||||
start: () => { frames = []; recording = true; },
|
||||
stop: () => { recording = false; return frames; },
|
||||
opened: () => ({ historyAt, allFrames }),
|
||||
};
|
||||
const result = await drive(control);
|
||||
return { openMs, ...result, errors: page.errors.slice(0, 5) };
|
||||
@@ -80,6 +89,30 @@ async function phase(origin, query, drive) {
|
||||
|
||||
const server = await serveBuild(dir);
|
||||
try {
|
||||
const open = await phase(server.origin, `perf=${messages}`, async ({ opened }) => {
|
||||
// The page is quiet once no frame has arrived for a second.
|
||||
const end = Date.now() + 20000;
|
||||
for (;;) {
|
||||
const { historyAt, allFrames } = opened();
|
||||
const last = allFrames.at(-1)?.at ?? 0;
|
||||
if (historyAt && Date.now() - Math.max(last, historyAt.at) > 1000) break;
|
||||
if (Date.now() > end) break;
|
||||
await delay(100);
|
||||
}
|
||||
const { historyAt, allFrames } = opened();
|
||||
if (!historyAt) return { open: null };
|
||||
const after = allFrames.slice(historyAt.frame);
|
||||
const sum = (i) => +(after.reduce((a, x) => a + x.f[i], 0) / 1000).toFixed(1);
|
||||
return {
|
||||
open: {
|
||||
...stats(after.map((x) => x.f)),
|
||||
buildSumMs: sum(0),
|
||||
totalSumMs: sum(2),
|
||||
// Wall time from the history reply to the last frame of the burst.
|
||||
settleMs: after.length ? after.at(-1).at - historyAt.at : 0,
|
||||
},
|
||||
};
|
||||
});
|
||||
const scroll = await phase(server.origin, `perf=${messages}`, async ({ page, start, stop }) => {
|
||||
start();
|
||||
for (const dy of [...Array(60).fill(-400), ...Array(60).fill(400)]) {
|
||||
@@ -102,7 +135,7 @@ try {
|
||||
return { streamed: marks.includes('PERF_STREAM_END'), stream: stats(stop()) };
|
||||
},
|
||||
);
|
||||
console.log(JSON.stringify({ build: dir, messages, deltas, scroll, stream }, null, 2));
|
||||
console.log(JSON.stringify({ build: dir, messages, deltas, open, scroll, stream }, null, 2));
|
||||
} finally {
|
||||
await server.close();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user