58c72538d5
tool/perf/chat_4k_bench.mjs opens the full-app fixture at 3840x2160 with 12 rich chats and measures opening, scrolling, streaming, switching and typing (frame timings plus a source-mapped CPU profile). The capture kit's --gpu mode now uses ANGLE on Vulkan on Linux. notes/perf-4k-chat.md records the baseline and the hotspots it found. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
253 lines
11 KiB
JavaScript
253 lines
11 KiB
JavaScript
#!/usr/bin/env node
|
|
// Chat performance at 4K over the full-app fixture (tool/search_selection_probe_entry.dart),
|
|
// never a live server. Build it as a profile build with source maps, from app/:
|
|
//
|
|
// flutter build web --profile --source-maps \
|
|
// --target tool/search_selection_probe_entry.dart --output build/probe-profile
|
|
// node tool/perf/chat_4k_bench.mjs --build build/probe-profile --out /tmp/chat4k [--gpu] [--repeat 3]
|
|
//
|
|
// The window is 3840x2160 CSS px at device scale 1 (a 4K monitor at 100%);
|
|
// --size and --dpr change it (1920x1080 at --dpr 2 is the same panel at 200%).
|
|
// Data: 12 chats of 150 rich messages (code, wide tables, images, agent
|
|
// reports, links, turn activity, shown items and turn diffs).
|
|
//
|
|
// A discovery load (semantics on) finds the chat rows, the composer and the
|
|
// transcript; the measured loads keep semantics off, as for most users, and
|
|
// drive the app with CDP pointer, wheel and text input. Each scenario runs on a
|
|
// fresh load. Flutter's FrameTiming comes from tool/perf/frame_log.dart; the
|
|
// first repetition also records a V8 CPU profile, attributed to Dart files
|
|
// through the source map (tool/perf/cpu_attribution.mjs).
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { parseArgs } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { openBrowser } from '../capture/browser.mjs';
|
|
import { flutterPage } from '../capture/page.mjs';
|
|
import { serveBuild } from '../capture/serve.mjs';
|
|
import { attribute, loadSourceMap } from './cpu_attribution.mjs';
|
|
|
|
const { values: opts } = parseArgs({
|
|
options: {
|
|
build: { type: 'string' }, out: { type: 'string' }, repeat: { type: 'string', default: '3' },
|
|
gpu: { type: 'boolean' }, size: { type: 'string', default: '3840x2160' }, dpr: { type: 'string', default: '1' },
|
|
only: { type: 'string' }, live: { type: 'string', default: '200' },
|
|
},
|
|
});
|
|
if (!opts.build || !opts.out) throw new Error('usage: --build <dir> --out <dir> [--gpu]');
|
|
const [WIDTH, HEIGHT] = opts.size.split('x').map(Number);
|
|
const DPR = Number(opts.dpr);
|
|
const DATA = {
|
|
chats: '12', msgs: '150', rich: '1', links: '1', turnactivity: '1',
|
|
timestamps: '1', shown: '1', turndiffs: '1', theme: 'light',
|
|
};
|
|
|
|
const r1 = (x) => Math.round(x * 10) / 10;
|
|
|
|
/** Self (leaf) ms per mapped function: where the CPU actually was. */
|
|
function selfTime(profile, lookup, n) {
|
|
const byId = new Map(profile.nodes.map((node) => [node.id, node]));
|
|
const self = new Map();
|
|
profile.samples.forEach((id, i) => {
|
|
const dt = (profile.timeDeltas[i + 1] ?? 0) / 1000;
|
|
const f = byId.get(id).callFrame;
|
|
if (['(idle)', '(program)', '(garbage collector)'].includes(f.functionName)) {
|
|
if (f.functionName === '(garbage collector)') self.set('(gc)', (self.get('(gc)') ?? 0) + dt);
|
|
return;
|
|
}
|
|
const m = f.url.endsWith('main.dart.js') ? lookup(f.lineNumber, f.columnNumber) : null;
|
|
const key = m ? `${m.source}:${m.line} ${f.functionName}` : `(${f.url.split('/').pop() || 'native'}) ${f.functionName}`;
|
|
self.set(key, (self.get(key) ?? 0) + dt);
|
|
});
|
|
return [...self].sort((a, b) => b[1] - a[1]).slice(0, n).map(([k, v]) => `${r1(v)} ${k}`);
|
|
}
|
|
const median = (xs) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]; };
|
|
const pct = (xs, p) => { const s = [...xs].sort((a, b) => a - b); return s.length ? s[Math.min(s.length - 1, Math.floor(s.length * p))] : 0; };
|
|
|
|
function summarize(frames) {
|
|
const ms = (us) => us / 1000;
|
|
const build = frames.map((f) => ms(f[0]));
|
|
const raster = frames.map((f) => ms(f[1]));
|
|
const total = frames.map((f) => ms(f[2]));
|
|
return {
|
|
frames: frames.length,
|
|
buildP50: r1(pct(build, 0.5)), buildP90: r1(pct(build, 0.9)), buildMax: r1(Math.max(0, ...build)),
|
|
rasterP50: r1(pct(raster, 0.5)), rasterP90: r1(pct(raster, 0.9)), rasterMax: r1(Math.max(0, ...raster)),
|
|
totalP90: r1(pct(total, 0.9)), totalMax: r1(Math.max(0, ...total)),
|
|
over16: total.filter((t) => t > 16.7).length, over33: total.filter((t) => t > 33.4).length,
|
|
buildSum: r1(build.reduce((a, b) => a + b, 0)),
|
|
};
|
|
}
|
|
|
|
function url(origin, extra = {}) {
|
|
const u = new URL('/', origin);
|
|
u.searchParams.set('route', '/chat/perf-chat-0');
|
|
for (const [k, v] of Object.entries({ ...DATA, perf: '1', ...extra })) u.searchParams.set(k, v);
|
|
return u.href;
|
|
}
|
|
|
|
const server = await serveBuild(opts.build);
|
|
const lookup = await loadSourceMap(opts.build);
|
|
await mkdir(opts.out, { recursive: true });
|
|
|
|
// ---- Discovery: where the rows, the composer and the transcript are --------
|
|
async function discover() {
|
|
const browser = await openBrowser({ gpu: Boolean(opts.gpu) });
|
|
try {
|
|
const page = flutterPage(browser, () => {});
|
|
await page.open(url(server.origin, { perf: '0' }), { width: WIDTH, height: HEIGHT });
|
|
await delay(2500);
|
|
const labels = await page.labels();
|
|
const center = (n) => [n.rect[0] + n.rect[2] / 2, n.rect[1] + n.rect[3] / 2];
|
|
const rows = labels.filter((n) => /^OpenCode harness\nFixture chat \d+/.test(n.label))
|
|
.sort((a, b) => a.rect[1] - b.rect[1]).map(center);
|
|
const composer = labels.find((n) => n.role === 'textarea' || n.role === 'input' || /^Work on/.test(n.label));
|
|
const pane = labels.find((n) => n.key === 'list-detail-body');
|
|
if (rows.length < 6 || !composer || !pane) {
|
|
await writeFile(join(opts.out, 'discovery-labels.json'), JSON.stringify(labels, null, 2));
|
|
throw new Error(`discovery failed: rows=${rows.length} composer=${Boolean(composer)} pane=${Boolean(pane)}`);
|
|
}
|
|
await writeFile(join(opts.out, 'layout.png'), await page.shot());
|
|
const transcript = [pane.rect[0] + pane.rect[2] / 2, pane.rect[1] + pane.rect[3] * 0.45];
|
|
return { rows, composer: center(composer), transcript };
|
|
} finally {
|
|
await browser.close().catch(() => {});
|
|
}
|
|
}
|
|
|
|
// ---- Measured loads -------------------------------------------------------
|
|
async function measure(name, extra, drive) {
|
|
const runs = [];
|
|
let report = null;
|
|
for (let rep = 0; rep < Number(opts.repeat); rep++) {
|
|
const browser = await openBrowser({ gpu: Boolean(opts.gpu) });
|
|
try {
|
|
const { send, on } = browser;
|
|
const evaluate = async (expression) => (await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true })).result?.value;
|
|
let frames = [];
|
|
let first = null;
|
|
const errors = [];
|
|
on('Runtime.consoleAPICalled', (p) => {
|
|
const text = p.args.map((a) => a.value ?? '').join(' ');
|
|
if (text.startsWith('PERF_FRAME ')) frames.push(text.split(' ').slice(1).map(Number));
|
|
else if (text.startsWith('PERF_FIRST_FRAME ')) first = Number(text.split(' ')[1]);
|
|
else if (p.type === 'error') errors.push(text.slice(0, 160));
|
|
});
|
|
await send('Page.enable');
|
|
await send('Runtime.enable');
|
|
await send('Emulation.setDeviceMetricsOverride', { width: WIDTH, height: HEIGHT, deviceScaleFactor: DPR, mobile: false });
|
|
const io = {
|
|
send, evaluate,
|
|
async settle(ms = 1200) { await delay(ms); for (let i = 0; i < 2; i++) { await evaluate('window.__perfKick?.()'); await delay(250); } },
|
|
async tap([x, y]) {
|
|
for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 });
|
|
}
|
|
},
|
|
async wheel([x, y], deltaY, count) {
|
|
for (let i = 0; i < count; i++) {
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX: 0, deltaY });
|
|
await delay(16);
|
|
}
|
|
},
|
|
async type(text, gap = 60) { for (const ch of text) { await send('Input.insertText', { text: ch }); await delay(gap); } },
|
|
};
|
|
const profile = rep === 0;
|
|
if (profile && name === 'open') { await send('Profiler.enable'); await send('Profiler.setSamplingInterval', { interval: 200 }); await send('Profiler.start'); }
|
|
const navigated = Date.now();
|
|
await send('Page.navigate', { url: url(server.origin, extra) });
|
|
for (let i = 0; i < 400 && first === null; i++) await delay(50);
|
|
let result;
|
|
if (name === 'open') {
|
|
await io.settle(3000);
|
|
result = { firstFrameMs: first, wallMs: Date.now() - navigated, ...summarize(frames) };
|
|
} else {
|
|
await io.settle(3500);
|
|
frames = [];
|
|
if (profile) { await send('Profiler.enable'); await send('Profiler.setSamplingInterval', { interval: 200 }); await send('Profiler.start'); }
|
|
const started = Date.now();
|
|
await drive(io);
|
|
await io.settle();
|
|
result = { wallMs: Date.now() - started, ...summarize(frames) };
|
|
}
|
|
if (profile) {
|
|
const { profile: cpu } = await send('Profiler.stop');
|
|
await writeFile(join(opts.out, `${name}.cpuprofile`), JSON.stringify(cpu));
|
|
report = {
|
|
...attribute(cpu, lookup, { n: 25 }),
|
|
app: attribute(cpu, lookup, { n: 25, filter: (k) => k.startsWith('app:') }),
|
|
self: selfTime(cpu, lookup, 25),
|
|
};
|
|
}
|
|
if (errors.length) result.errors = [...new Set(errors)].slice(0, 3);
|
|
runs.push(result);
|
|
} finally {
|
|
await browser.close().catch(() => {});
|
|
}
|
|
}
|
|
const keys = Object.keys(runs[0]).filter((k) => typeof runs[0][k] === 'number');
|
|
const med = Object.fromEntries(keys.map((k) => [k, median(runs.map((r) => r[k]))]));
|
|
return { median: med, runs, cpu: report };
|
|
}
|
|
|
|
const points = await discover();
|
|
console.error('layout', JSON.stringify(points));
|
|
const SCENARIOS = {
|
|
open: { extra: {}, drive: null },
|
|
scroll: {
|
|
extra: {},
|
|
drive: async (io) => {
|
|
await io.wheel(points.transcript, -400, 60);
|
|
await delay(300);
|
|
await io.wheel(points.transcript, 400, 60);
|
|
},
|
|
},
|
|
stream: {
|
|
extra: { live: opts.live },
|
|
// The fixture starts streaming 3 s after the chat subscribes; the settle
|
|
// before drive() covers most of that, then every delta is 60 ms apart.
|
|
drive: async () => { await delay(Number(opts.live) * 60 + 1500); },
|
|
},
|
|
switch: {
|
|
extra: {},
|
|
drive: async (io) => {
|
|
for (const row of [1, 2, 3, 4, 5, 0].map((i) => points.rows[i])) {
|
|
await io.tap(row);
|
|
await delay(900);
|
|
}
|
|
},
|
|
},
|
|
type: {
|
|
extra: {},
|
|
drive: async (io) => {
|
|
await io.tap(points.composer);
|
|
await delay(300);
|
|
await io.type('Profiling the composer at 4K: does every keystroke rebuild the transcript?');
|
|
},
|
|
},
|
|
};
|
|
|
|
const results = {};
|
|
try {
|
|
const only = opts.only?.split(',');
|
|
for (const [name, s] of Object.entries(SCENARIOS)) {
|
|
if (only && !only.includes(name)) continue;
|
|
results[name] = await measure(name, s.extra, s.drive);
|
|
console.log(name, JSON.stringify(results[name].median));
|
|
if (results[name].cpu) {
|
|
console.log(` cpu busy ${results[name].cpu.busyMs} ms; files:`);
|
|
for (const line of results[name].cpu.files.slice(0, 10)) console.log(' ', line);
|
|
console.log(' app files (inclusive):');
|
|
for (const line of results[name].cpu.app.files.slice(0, 15)) console.log(' ', line);
|
|
console.log(' app functions (inclusive):');
|
|
for (const line of results[name].cpu.app.functions.slice(0, 20)) console.log(' ', line);
|
|
console.log(' self time:');
|
|
for (const line of results[name].cpu.self.slice(0, 20)) console.log(' ', line);
|
|
}
|
|
}
|
|
await writeFile(join(opts.out, 'chat-4k-bench.json'), JSON.stringify({
|
|
build: opts.build, size: opts.size, dpr: DPR, gpu: Boolean(opts.gpu), data: DATA, points, results,
|
|
}, null, 2));
|
|
} finally {
|
|
await server.close();
|
|
}
|