Files
pipelines/app/tool/perf/interaction_bench.mjs
alan 6eb26b6d17 Bench desktop windows and chat, with an option to keep the GPU
interaction_bench.mjs gains 1280x800 scenarios (dragging, resizing, hiding
and floating chat windows, a probe window drag, chat scroll, typing, chat
switch, the terminal panel), key, drag and wheel steps, a per-scenario size
and --gpu, which keeps WebGL so raster time reads as on a desktop.
2026-11-11 01:41:44 +00:00

234 lines
14 KiB
JavaScript

#!/usr/bin/env node
// Interaction bench over the full-app fixture (tool/search_selection_probe_entry.dart,
// built release). For each scenario it loads a route with `?perf=1`, runs the
// setup steps, then records Flutter's FrameTiming (tool/perf/frame_log.dart)
// for the measured steps until the page settles, and prints per-scenario
// medians: frames, build sum / max, raster sum / max and the longest frame.
// Startup reports the first rasterised frame (ms since navigation) and the
// main.dart.js / canvaskit fetch times.
//
// --cpuprofile records a V8 CPU profile of the measured steps (first rep),
// writes `<scenario>.cpuprofile` and prints the functions with the most
// inclusive and self time; use a `flutter build web --profile` build so the
// names are readable; add --source-maps to attribute time to Dart files
// (tool/perf/cpu_attribution.mjs). --frames prints every measured frame's
// build and raster time; --shots writes a screenshot after each scenario.
//
// node tool/perf/interaction_bench.mjs --build build/probe-app --out /tmp/bench \
// [--repeat 5] [--only name,name] [--size 560x900] [--shots] [--cpuprofile] [--frames] [--gpu]
//
// Clicks are points (CSS px at 560x900, Material, light), so Flutter's
// semantics tree stays off as for most users. Headless Chrome without a GPU:
// CanvasKit rasterises on the CPU, so compare builds with each other rather
// than predicting a device. --gpu keeps the GPU (WebGL), which is what a
// desktop user has: use it to judge raster time.
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { parseArgs } from 'node:util';
import { setTimeout as delay } from 'node:timers/promises';
import { openBrowser } from '../capture/browser.mjs';
import { serveBuild } from '../capture/serve.mjs';
import { attribute, loadSourceMap } from './cpu_attribution.mjs';
const { values: opts } = parseArgs({
options: {
build: { type: 'string' }, out: { type: 'string' }, repeat: { type: 'string', default: '5' },
only: { type: 'string' }, cpuprofile: { type: 'boolean' }, frames: { type: 'boolean' }, shots: { type: 'boolean' }, gpu: { type: 'boolean' }, size: { type: 'string', default: '560x900' },
},
});
if (!opts.build || !opts.out) throw new Error('usage: --build <dir> --out <dir>');
// Bottom tab bar and top-bar points at 560x900 (Material).
const TAB = { first: [337, 854], second: [396, 854], third: [455, 854], fourth: [514, 854] };
const TOP_RIGHT = [524, 32];
const FIRST_ROW = [280, 138];
// A rich two-chat fixture at 1280x800, and the steps that put the second
// chat in a floating window (More actions, Add chat pane, the chat, Alt+Y).
const DESKTOP_CHAT = { size: '1280x800', route: '/chat/perf-chat-0', q: { chats: '2', header: '1', rich: '1' } };
const FLOAT_CHAT = [{ click: [1240, 40] }, { click: [1198, 96] }, { click: [650, 245] }, { key: 'Alt+y' }, { wait: 800 }];
// `setup` runs unrecorded after load; `act` is recorded. A step is
// `{ click: [x, y] }`, `{ type: 'text' }` (one character per 120 ms), `{ wait }`,
// `{ key: 'Alt+y' }`, `{ drag: [x, y, dx, dy] }` (40 moves, 16 ms apart) or
// `{ wheel: [x, y, deltaY, count] }` (one notch per 16 ms). A scenario's `size`
// overrides --size.
const SCENARIOS = [
{ name: 'startup', route: '/runs', act: [] },
{ name: 'tab:runs->pipelines', route: '/runs', act: [{ click: TAB.first }] },
{ name: 'tab:pipelines->runs', route: '/pipelines', act: [{ click: TAB.second }] },
{ name: 'tab:runs->search', route: '/runs', act: [{ click: TAB.fourth }] },
{ name: 'tab:runs->chat', route: '/runs', q: { chats: '30' }, act: [{ click: TAB.third }] },
{ name: 'tab:runs->chat (150 chats)', route: '/runs', q: { chats: '150' }, act: [{ click: TAB.third }] },
{ name: 'startup (150 chats)', route: '/runs', q: { chats: '150' }, act: [] },
{ name: 'runs(400)->run detail', route: '/runs', q: { runs: '400' }, act: [{ click: FIRST_ROW }] },
{ name: 'runs->run detail', route: '/runs', act: [{ click: FIRST_ROW }] },
{ name: 'run tab:summary->timeline', route: '/runs/7', act: [{ click: TAB.second }] },
{ name: 'run tab:timeline(400)', route: '/runs/7', q: { steps: '400' }, act: [{ click: TAB.second }] },
{ name: 'run tab:->artifacts', route: '/runs/7', act: [{ click: TAB.third }] },
{ name: 'run tab:->changes', route: '/runs/7', act: [{ click: TAB.fourth }] },
{ name: 'timeline->visit page', route: '/runs/7', setup: [{ click: TAB.second }], act: [{ click: [280, 150] }] },
{ name: 'run tab:timeline->summary', route: '/runs/7', setup: [{ click: TAB.second }], act: [{ click: TAB.first }] },
{ name: 'run detail with graph preview', route: '/runs', q: { graph: '1' }, act: [{ click: FIRST_ROW }] },
{ name: 'run graph full screen', route: '/runs/7', q: { graph: '1' }, act: [{ click: [512, 152] }] },
{ name: 'artifact viewer open', route: '/runs/7', setup: [{ click: TAB.third }], act: [{ click: [280, 376] }] },
{ name: 'search type "release" (200 hits)', route: '/search', q: { hits: '200' },
setup: [{ click: [200, 32] }], act: [{ type: 'release' }] },
{ name: 'pipelines->pipeline detail', route: '/pipelines', act: [{ click: [280, 155] }] },
{ name: 'pipeline tab:->graph', route: '/pipelines/24', act: [{ click: TAB.third }] },
{ name: 'pipeline tab:->source', route: '/pipelines/24', act: [{ click: TAB.fourth }] },
{ name: 'chat list->open chat', route: '/chat', q: { chats: '30' }, act: [{ click: FIRST_ROW }] },
{ name: 'pipelines->settings', route: '/pipelines', act: [{ click: TOP_RIGHT }] },
// Desktop (1280x800): floating windows and the chat beside its list.
{ name: 'desktop: drag a chat window', ...DESKTOP_CHAT, setup: FLOAT_CHAT,
act: [{ drag: [300, 116, 500, 120] }] },
{ name: 'desktop: resize a chat window', ...DESKTOP_CHAT, setup: FLOAT_CHAT,
act: [{ drag: [591, 647, 300, 80] }] },
{ name: 'desktop: hide and show chat windows', ...DESKTOP_CHAT, setup: FLOAT_CHAT,
act: [{ key: 'Alt+y' }, { wait: 600 }, { key: 'Alt+y' }] },
{ name: 'desktop: float a chat', ...DESKTOP_CHAT, setup: FLOAT_CHAT.slice(0, 3),
act: [{ key: 'Alt+y' }] },
{ name: 'desktop: drag a probe window', size: '1280x800', route: '/runs', q: { windows: '3' },
act: [{ drag: [250, 164, 500, 120] }] },
{ name: 'desktop: scroll a rich chat', ...DESKTOP_CHAT,
act: [{ wheel: [860, 400, -240, 30] }] },
{ name: 'desktop: type in the composer', ...DESKTOP_CHAT, setup: [{ click: [860, 748] }],
act: [{ type: 'hello world, typing' }] },
{ name: 'desktop: switch chat', ...DESKTOP_CHAT, act: [{ click: [220, 260] }] },
{ name: 'desktop: open the terminal panel', ...DESKTOP_CHAT, q: { chats: '2', header: '1', terminals: '1' },
act: [{ key: 'Alt+t' }] },
];
const median = (xs) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]; };
const r1 = (x) => Math.round(x * 10) / 10;
const lookup = opts.cpuprofile ? await loadSourceMap(opts.build) : null;
const server = await serveBuild(opts.build);
const browser = await openBrowser({ gpu: Boolean(opts.gpu) });
const { send, on } = browser;
const evaluate = async (expression) => (await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true })).result?.value;
let frames = [];
let firstFrame = null;
const errors = [];
on('Runtime.consoleAPICalled', (p) => {
const text = p.args.map((a) => a.value ?? '').join(' ');
if (text.startsWith('PERF_FRAME ')) frames.push(text.split(' ').slice(1).map(Number));
else if (text.startsWith('PERF_FIRST_FRAME ')) firstFrame = Number(text.split(' ')[1]);
else if (p.type === 'error') errors.push(text.slice(0, 200));
});
on('Runtime.exceptionThrown', (p) => errors.push(String(p.exceptionDetails?.exception?.description ?? p.exceptionDetails?.text).slice(0, 200)));
async function tap([x, y]) {
for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) {
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 });
}
}
async function run(steps) {
for (const step of steps ?? []) {
if (step.click) { await tap(step.click); await delay(1200); }
if (step.type) for (const ch of step.type) { await send('Input.insertText', { text: ch }); await delay(120); }
if (step.wait) await delay(step.wait);
if (step.key) { await key(step.key); await delay(800); }
if (step.drag) { await drag(step.drag); await delay(400); }
if (step.wheel) {
const [x, y, deltaY, count] = step.wheel;
for (let i = 0; i < count; i++) {
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX: 0, deltaY });
await delay(16);
}
await delay(400);
}
}
}
const MODIFIERS = { Alt: 1, Control: 2, Meta: 4, Shift: 8 };
async function key(chord) {
const parts = chord.split('+');
const name = parts.pop();
const modifiers = parts.reduce((bits, part) => bits | MODIFIERS[part], 0);
const code = /^[a-z]$/i.test(name) ? `Key${name.toUpperCase()}` : name;
const base = { modifiers, key: name, code, windowsVirtualKeyCode: name.toUpperCase().charCodeAt(0) };
for (const part of parts) await send('Input.dispatchKeyEvent', { type: 'rawKeyDown', key: part, code: `${part}Left`, modifiers });
await send('Input.dispatchKeyEvent', { type: 'rawKeyDown', ...base });
await send('Input.dispatchKeyEvent', { type: 'keyUp', ...base });
for (const part of parts) await send('Input.dispatchKeyEvent', { type: 'keyUp', key: part, code: `${part}Left`, modifiers: 0 });
}
async function drag([x, y, dx, dy]) {
const steps = 40;
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x, y, button: 'none' });
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 });
for (let i = 1; i <= steps; i++) {
await send('Input.dispatchMouseEvent', { type: 'mouseMoved', x: x + (dx * i) / steps, y: y + (dy * i) / steps, button: 'left', buttons: 1 });
await delay(16);
}
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: x + dx, y: y + dy, button: 'left', clickCount: 1 });
}
// The engine hands timings over in batches; two late frames flush them.
async function settle() {
await delay(1500);
for (let i = 0; i < 2; i++) { await evaluate('window.__perfKick?.()'); await delay(250); }
}
await mkdir(opts.out, { recursive: true });
const results = {};
try {
await send('Page.enable');
await send('Runtime.enable');
await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value: 'light' }] });
const only = opts.only?.split(',');
for (const s of SCENARIOS.filter((s) => !only || only.includes(s.name))) {
const [w, h] = (s.size ?? opts.size).split('x').map(Number);
await send('Emulation.setDeviceMetricsOverride', { width: w, height: h, deviceScaleFactor: 1, mobile: false });
const reps = [];
for (let rep = 0; rep < Number(opts.repeat); rep++) {
const url = new URL('/', server.origin);
url.searchParams.set('route', s.route);
url.searchParams.set('theme', 'light');
url.searchParams.set('perf', '1');
for (const [k, v] of Object.entries(s.q ?? {})) url.searchParams.set(k, v);
frames = []; firstFrame = null;
await send('Page.navigate', { url: url.href });
for (let i = 0; i < 300 && firstFrame === null; i++) await delay(50);
await delay(2000);
const startup = s.name.startsWith('startup');
const loads = startup ? await evaluate(`Object.fromEntries(performance.getEntriesByType('resource')
.filter((e) => /main\\.dart\\.js|canvaskit\\.wasm/.test(e.name))
.map((e) => [e.name.split('/').pop(), Math.round(e.responseEnd)]))`) : null;
await run(s.setup);
await settle();
const startFrames = startup ? 0 : frames.length;
const profiling = opts.cpuprofile && rep === 0;
if (profiling) {
await send('Profiler.enable');
await send('Profiler.setSamplingInterval', { interval: 100 });
await send('Profiler.start');
}
await run(s.act);
await settle();
if (profiling) {
const { profile } = await send('Profiler.stop');
await writeFile(join(opts.out, `${s.name.replace(/[^\w-]+/g, '_')}.cpuprofile`), JSON.stringify(profile));
console.log(s.name, attribute(profile, lookup, { filter: (k) => k.startsWith('app:') || !k.includes(' ') }));
}
if (opts.shots && rep === 0) {
const png = (await send('Page.captureScreenshot', { format: 'png' })).data;
await writeFile(join(opts.out, `${s.name.replace(/[^\w-]+/g, '_')}.png`), Buffer.from(png, 'base64'));
}
const f = frames.slice(startFrames).filter((x) => x[0] + x[1] > 300);
if (opts.frames) console.log(` ${s.name} rep ${rep} build/raster ms:`, f.map((x) => `${r1(x[0] / 1000)}/${r1(x[1] / 1000)}`).join(' '));
const sum = (i) => f.reduce((a, x) => a + x[i], 0) / 1000;
const max = (i) => f.reduce((a, x) => Math.max(a, x[i]), 0) / 1000;
reps.push({ frames: f.length, buildSum: sum(0), buildMax: max(0), rasterSum: sum(1), rasterMax: max(1),
longest: max(2), over16: f.filter((x) => x[2] > 16667).length, firstFrame, ...loads });
}
const keys = Object.keys(reps[0]).filter((k) => typeof reps[0][k] === 'number');
results[s.name] = Object.fromEntries(keys.map((k) => [k, r1(median(reps.map((r) => r[k])))]));
results[s.name].reps = reps.length;
console.log(s.name.padEnd(36), JSON.stringify(results[s.name]));
}
if (errors.length) console.log('page errors (first 5):', [...new Set(errors)].slice(0, 5));
await writeFile(join(opts.out, 'interaction-bench.json'), JSON.stringify({ build: opts.build, size: opts.size, gpu: Boolean(opts.gpu), results }, null, 2));
} finally {
await browser.close().catch(() => {});
await server.close();
}