Files
pipelines/app/tool/chat_perf_probe.mjs
T

139 lines
5.5 KiB
JavaScript

// Release-web frame timings for a long chat transcript, over the capture kit.
//
// Build the fixture once per variant, from app/:
// flutter build web --release --target tool/chat_composer_probe_entry.dart \
// --output build/chat-perf
// then:
// node tool/chat_perf_probe.mjs build/chat-perf [messages=300] [deltas=200] [repeats=3]
//
// Three phases, each on a fresh page: open the transcript (the frames that do
// work after the history reply), wheel-scroll it up and back down,
// then stream deltas into a new reply. Flutter's own FrameTiming (build,
// raster, total) arrives as console lines from tool/chat_perf_fixture.dart.
// Prints one JSON report. Headless Chrome rasterises in software, so raster
// numbers compare variants rather than predict a device.
import { setTimeout as delay } from 'node:timers/promises';
import { openBrowser } from './capture/browser.mjs';
import { flutterPage } from './capture/page.mjs';
import { serveBuild } from './capture/serve.mjs';
const [dir, messagesArg = '300', deltasArg = '200', repeatsArg = '3'] = process.argv.slice(2);
if (!dir) {
console.error('usage: node tool/chat_perf_probe.mjs <build dir> [messages] [deltas] [repeats]');
process.exit(2);
}
const messages = Number(messagesArg);
const deltas = Number(deltasArg);
const repeats = Number(repeatsArg);
function stats(frames) {
const pick = (i) => frames.map((f) => f[i]).sort((a, b) => a - b);
const summary = (values) => values.length === 0 ? null : {
meanMs: +(values.reduce((a, b) => a + b, 0) / values.length / 1000).toFixed(2),
p50Ms: +(values[Math.floor(values.length / 2)] / 1000).toFixed(2),
p90Ms: +(values[Math.floor(values.length * 0.9)] / 1000).toFixed(2),
maxMs: +(values[values.length - 1] / 1000).toFixed(2),
};
const total = pick(2);
return {
frames: frames.length,
build: summary(pick(0)),
raster: summary(pick(1)),
total: summary(total),
over16ms: total.filter((t) => t > 16667).length,
};
}
async function phase(origin, query, drive) {
const browser = await openBrowser();
try {
const page = flutterPage(browser, (m) => console.error(m));
let recording = false;
let frames = [];
const marks = [];
// Every frame since navigation, for the open phase.
const allFrames = [];
let historyAt = null;
browser.on('Runtime.consoleAPICalled', (p) => {
const text = p.args.map((a) => a.value ?? '').join(' ');
if (text.startsWith('PERF_FRAME ')) {
if (recording) frames.push(text.split(' ').slice(1).map(Number));
} else if (text.startsWith('PERF_')) {
marks.push(text);
if (text === 'PERF_HISTORY') historyAt = { frame: allFrames.length };
}
if (text.startsWith('PERF_FRAME ')) {
allFrames.push({ f: text.split(' ').slice(1).map(Number) });
}
});
const opened = Date.now();
try {
await page.open(`${origin}/?${query}`, { width: 560, height: 1000 });
} catch (error) {
throw new Error(`${error.message}; page errors: ${JSON.stringify(page.errors.slice(0, 5))}`);
}
const openMs = Date.now() - opened;
await delay(1000);
const control = {
page,
marks,
start: () => { frames = []; recording = true; },
stop: () => { recording = false; return frames; },
opened: () => ({ historyAt, allFrames }),
};
const result = await drive(control);
return { openMs, ...result, errors: page.errors.slice(0, 5) };
} finally {
await browser.close();
}
}
const server = await serveBuild(dir);
try {
const results = [];
for (let repeat = 0; repeat < repeats; repeat++) {
const open = await phase(server.origin, `perf=${messages}`, async ({ opened }) => {
// The fixture flushes the timings 2.5 s after the history reply.
const end = Date.now() + 20000;
while (!opened().historyAt && Date.now() < end) await delay(100);
await delay(3500);
const { historyAt, allFrames } = opened();
if (!historyAt) return { open: null };
// Frames that did work after the history reply: the cost of opening. The
// fixture's flush frames do none.
const after = allFrames.slice(historyAt.frame).map((x) => x.f).filter((f) => f[0] >= 1000);
const sum = (i) => +(after.reduce((a, f) => a + f[i], 0) / 1000).toFixed(1);
return { open: { ...stats(after), buildSumMs: sum(0), totalSumMs: sum(2) } };
});
const scroll = await phase(server.origin, `perf=${messages}`, async ({ page, start, stop }) => {
start();
for (const dy of [...Array(60).fill(-400), ...Array(60).fill(400)]) {
await page.scroll(dy);
await delay(50);
}
await delay(500);
return { scroll: stats(stop()) };
});
const stream = await phase(
server.origin,
`perf=${messages}&stream=${deltas}`,
async ({ start, stop, marks }) => {
const begin = Date.now() + 15000;
while (!marks.includes('PERF_STREAM_START') && Date.now() < begin) await delay(5);
start();
const end = Date.now() + deltas * 40 + 15000;
while (!marks.includes('PERF_STREAM_END') && Date.now() < end) await delay(100);
await delay(500);
return { streamed: marks.includes('PERF_STREAM_END'), stream: stats(stop()) };
},
);
results.push({ repeat: repeat + 1, open, scroll, stream });
console.log(JSON.stringify({ repeat: repeat + 1, messages, deltas, open, scroll, stream }));
}
console.log(JSON.stringify({ build: dir, messages, deltas, repeats, results }, null, 2));
} finally {
await server.close();
}
// A keep-alive connection can hold the server open after close.
process.exit(0);