139 lines
5.5 KiB
JavaScript
139 lines
5.5 KiB
JavaScript
// Release-web frame timings for a long chat transcript, over the capture kit.
|
|
//
|
|
// Build the fixture once per variant, from app/:
|
|
// flutter build web --release --target tool/chat_composer_probe_entry.dart \
|
|
// --output build/chat-perf
|
|
// then:
|
|
// node tool/chat_perf_probe.mjs build/chat-perf [messages=300] [deltas=200] [repeats=3]
|
|
//
|
|
// Three phases, each on a fresh page: open the transcript (the frames that do
|
|
// work after the history reply), wheel-scroll it up and back down,
|
|
// then stream deltas into a new reply. Flutter's own FrameTiming (build,
|
|
// raster, total) arrives as console lines from tool/chat_perf_fixture.dart.
|
|
// Prints one JSON report. Headless Chrome rasterises in software, so raster
|
|
// numbers compare variants rather than predict a device.
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { openBrowser } from './capture/browser.mjs';
|
|
import { flutterPage } from './capture/page.mjs';
|
|
import { serveBuild } from './capture/serve.mjs';
|
|
|
|
const [dir, messagesArg = '300', deltasArg = '200', repeatsArg = '3'] = process.argv.slice(2);
|
|
if (!dir) {
|
|
console.error('usage: node tool/chat_perf_probe.mjs <build dir> [messages] [deltas] [repeats]');
|
|
process.exit(2);
|
|
}
|
|
const messages = Number(messagesArg);
|
|
const deltas = Number(deltasArg);
|
|
const repeats = Number(repeatsArg);
|
|
|
|
function stats(frames) {
|
|
const pick = (i) => frames.map((f) => f[i]).sort((a, b) => a - b);
|
|
const summary = (values) => values.length === 0 ? null : {
|
|
meanMs: +(values.reduce((a, b) => a + b, 0) / values.length / 1000).toFixed(2),
|
|
p50Ms: +(values[Math.floor(values.length / 2)] / 1000).toFixed(2),
|
|
p90Ms: +(values[Math.floor(values.length * 0.9)] / 1000).toFixed(2),
|
|
maxMs: +(values[values.length - 1] / 1000).toFixed(2),
|
|
};
|
|
const total = pick(2);
|
|
return {
|
|
frames: frames.length,
|
|
build: summary(pick(0)),
|
|
raster: summary(pick(1)),
|
|
total: summary(total),
|
|
over16ms: total.filter((t) => t > 16667).length,
|
|
};
|
|
}
|
|
|
|
async function phase(origin, query, drive) {
|
|
const browser = await openBrowser();
|
|
try {
|
|
const page = flutterPage(browser, (m) => console.error(m));
|
|
let recording = false;
|
|
let frames = [];
|
|
const marks = [];
|
|
// Every frame since navigation, for the open phase.
|
|
const allFrames = [];
|
|
let historyAt = null;
|
|
browser.on('Runtime.consoleAPICalled', (p) => {
|
|
const text = p.args.map((a) => a.value ?? '').join(' ');
|
|
if (text.startsWith('PERF_FRAME ')) {
|
|
if (recording) frames.push(text.split(' ').slice(1).map(Number));
|
|
} else if (text.startsWith('PERF_')) {
|
|
marks.push(text);
|
|
if (text === 'PERF_HISTORY') historyAt = { frame: allFrames.length };
|
|
}
|
|
if (text.startsWith('PERF_FRAME ')) {
|
|
allFrames.push({ f: text.split(' ').slice(1).map(Number) });
|
|
}
|
|
});
|
|
const opened = Date.now();
|
|
try {
|
|
await page.open(`${origin}/?${query}`, { width: 560, height: 1000 });
|
|
} catch (error) {
|
|
throw new Error(`${error.message}; page errors: ${JSON.stringify(page.errors.slice(0, 5))}`);
|
|
}
|
|
const openMs = Date.now() - opened;
|
|
await delay(1000);
|
|
const control = {
|
|
page,
|
|
marks,
|
|
start: () => { frames = []; recording = true; },
|
|
stop: () => { recording = false; return frames; },
|
|
opened: () => ({ historyAt, allFrames }),
|
|
};
|
|
const result = await drive(control);
|
|
return { openMs, ...result, errors: page.errors.slice(0, 5) };
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
const server = await serveBuild(dir);
|
|
try {
|
|
const results = [];
|
|
for (let repeat = 0; repeat < repeats; repeat++) {
|
|
const open = await phase(server.origin, `perf=${messages}`, async ({ opened }) => {
|
|
// The fixture flushes the timings 2.5 s after the history reply.
|
|
const end = Date.now() + 20000;
|
|
while (!opened().historyAt && Date.now() < end) await delay(100);
|
|
await delay(3500);
|
|
const { historyAt, allFrames } = opened();
|
|
if (!historyAt) return { open: null };
|
|
// Frames that did work after the history reply: the cost of opening. The
|
|
// fixture's flush frames do none.
|
|
const after = allFrames.slice(historyAt.frame).map((x) => x.f).filter((f) => f[0] >= 1000);
|
|
const sum = (i) => +(after.reduce((a, f) => a + f[i], 0) / 1000).toFixed(1);
|
|
return { open: { ...stats(after), buildSumMs: sum(0), totalSumMs: sum(2) } };
|
|
});
|
|
const scroll = await phase(server.origin, `perf=${messages}`, async ({ page, start, stop }) => {
|
|
start();
|
|
for (const dy of [...Array(60).fill(-400), ...Array(60).fill(400)]) {
|
|
await page.scroll(dy);
|
|
await delay(50);
|
|
}
|
|
await delay(500);
|
|
return { scroll: stats(stop()) };
|
|
});
|
|
const stream = await phase(
|
|
server.origin,
|
|
`perf=${messages}&stream=${deltas}`,
|
|
async ({ start, stop, marks }) => {
|
|
const begin = Date.now() + 15000;
|
|
while (!marks.includes('PERF_STREAM_START') && Date.now() < begin) await delay(5);
|
|
start();
|
|
const end = Date.now() + deltas * 40 + 15000;
|
|
while (!marks.includes('PERF_STREAM_END') && Date.now() < end) await delay(100);
|
|
await delay(500);
|
|
return { streamed: marks.includes('PERF_STREAM_END'), stream: stats(stop()) };
|
|
},
|
|
);
|
|
results.push({ repeat: repeat + 1, open, scroll, stream });
|
|
console.log(JSON.stringify({ repeat: repeat + 1, messages, deltas, open, scroll, stream }));
|
|
}
|
|
console.log(JSON.stringify({ build: dir, messages, deltas, repeats, results }, null, 2));
|
|
} finally {
|
|
await server.close();
|
|
}
|
|
// A keep-alive connection can hold the server open after close.
|
|
process.exit(0);
|