Files
pipelines/app/tool/perf/chat_4k_bench.mjs
T
alan 58c72538d5 Add a 4K chat performance bench and record its baseline
tool/perf/chat_4k_bench.mjs opens the full-app fixture at 3840x2160 with
12 rich chats and measures opening, scrolling, streaming, switching and
typing (frame timings plus a source-mapped CPU profile). The capture
kit's --gpu mode now uses ANGLE on Vulkan on Linux. notes/perf-4k-chat.md
records the baseline and the hotspots it found.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-22 16:45:22 +00:00

253 lines
11 KiB
JavaScript

#!/usr/bin/env node
// Chat performance at 4K over the full-app fixture (tool/search_selection_probe_entry.dart),
// never a live server. Build it as a profile build with source maps, from app/:
//
// flutter build web --profile --source-maps \
// --target tool/search_selection_probe_entry.dart --output build/probe-profile
// node tool/perf/chat_4k_bench.mjs --build build/probe-profile --out /tmp/chat4k [--gpu] [--repeat 3]
//
// The window is 3840x2160 CSS px at device scale 1 (a 4K monitor at 100%);
// --size and --dpr change it (1920x1080 at --dpr 2 is the same panel at 200%).
// Data: 12 chats of 150 rich messages (code, wide tables, images, agent
// reports, links, turn activity, shown items and turn diffs).
//
// A discovery load (semantics on) finds the chat rows, the composer and the
// transcript; the measured loads keep semantics off, as for most users, and
// drive the app with CDP pointer, wheel and text input. Each scenario runs on a
// fresh load. Flutter's FrameTiming comes from tool/perf/frame_log.dart; the
// first repetition also records a V8 CPU profile, attributed to Dart files
// through the source map (tool/perf/cpu_attribution.mjs).
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { parseArgs } from 'node:util';
import { setTimeout as delay } from 'node:timers/promises';
import { openBrowser } from '../capture/browser.mjs';
import { flutterPage } from '../capture/page.mjs';
import { serveBuild } from '../capture/serve.mjs';
import { attribute, loadSourceMap } from './cpu_attribution.mjs';
const { values: opts } = parseArgs({
options: {
build: { type: 'string' }, out: { type: 'string' }, repeat: { type: 'string', default: '3' },
gpu: { type: 'boolean' }, size: { type: 'string', default: '3840x2160' }, dpr: { type: 'string', default: '1' },
only: { type: 'string' }, live: { type: 'string', default: '200' },
},
});
if (!opts.build || !opts.out) throw new Error('usage: --build <dir> --out <dir> [--gpu]');
const [WIDTH, HEIGHT] = opts.size.split('x').map(Number);
const DPR = Number(opts.dpr);
const DATA = {
chats: '12', msgs: '150', rich: '1', links: '1', turnactivity: '1',
timestamps: '1', shown: '1', turndiffs: '1', theme: 'light',
};
const r1 = (x) => Math.round(x * 10) / 10;
/** Self (leaf) ms per mapped function: where the CPU actually was. */
function selfTime(profile, lookup, n) {
const byId = new Map(profile.nodes.map((node) => [node.id, node]));
const self = new Map();
profile.samples.forEach((id, i) => {
const dt = (profile.timeDeltas[i + 1] ?? 0) / 1000;
const f = byId.get(id).callFrame;
if (['(idle)', '(program)', '(garbage collector)'].includes(f.functionName)) {
if (f.functionName === '(garbage collector)') self.set('(gc)', (self.get('(gc)') ?? 0) + dt);
return;
}
const m = f.url.endsWith('main.dart.js') ? lookup(f.lineNumber, f.columnNumber) : null;
const key = m ? `${m.source}:${m.line} ${f.functionName}` : `(${f.url.split('/').pop() || 'native'}) ${f.functionName}`;
self.set(key, (self.get(key) ?? 0) + dt);
});
return [...self].sort((a, b) => b[1] - a[1]).slice(0, n).map(([k, v]) => `${r1(v)} ${k}`);
}
const median = (xs) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]; };
const pct = (xs, p) => { const s = [...xs].sort((a, b) => a - b); return s.length ? s[Math.min(s.length - 1, Math.floor(s.length * p))] : 0; };
function summarize(frames) {
const ms = (us) => us / 1000;
const build = frames.map((f) => ms(f[0]));
const raster = frames.map((f) => ms(f[1]));
const total = frames.map((f) => ms(f[2]));
return {
frames: frames.length,
buildP50: r1(pct(build, 0.5)), buildP90: r1(pct(build, 0.9)), buildMax: r1(Math.max(0, ...build)),
rasterP50: r1(pct(raster, 0.5)), rasterP90: r1(pct(raster, 0.9)), rasterMax: r1(Math.max(0, ...raster)),
totalP90: r1(pct(total, 0.9)), totalMax: r1(Math.max(0, ...total)),
over16: total.filter((t) => t > 16.7).length, over33: total.filter((t) => t > 33.4).length,
buildSum: r1(build.reduce((a, b) => a + b, 0)),
};
}
function url(origin, extra = {}) {
const u = new URL('/', origin);
u.searchParams.set('route', '/chat/perf-chat-0');
for (const [k, v] of Object.entries({ ...DATA, perf: '1', ...extra })) u.searchParams.set(k, v);
return u.href;
}
const server = await serveBuild(opts.build);
const lookup = await loadSourceMap(opts.build);
await mkdir(opts.out, { recursive: true });
// ---- Discovery: where the rows, the composer and the transcript are --------
async function discover() {
const browser = await openBrowser({ gpu: Boolean(opts.gpu) });
try {
const page = flutterPage(browser, () => {});
await page.open(url(server.origin, { perf: '0' }), { width: WIDTH, height: HEIGHT });
await delay(2500);
const labels = await page.labels();
const center = (n) => [n.rect[0] + n.rect[2] / 2, n.rect[1] + n.rect[3] / 2];
const rows = labels.filter((n) => /^OpenCode harness\nFixture chat \d+/.test(n.label))
.sort((a, b) => a.rect[1] - b.rect[1]).map(center);
const composer = labels.find((n) => n.role === 'textarea' || n.role === 'input' || /^Work on/.test(n.label));
const pane = labels.find((n) => n.key === 'list-detail-body');
if (rows.length < 6 || !composer || !pane) {
await writeFile(join(opts.out, 'discovery-labels.json'), JSON.stringify(labels, null, 2));
throw new Error(`discovery failed: rows=${rows.length} composer=${Boolean(composer)} pane=${Boolean(pane)}`);
}
await writeFile(join(opts.out, 'layout.png'), await page.shot());
const transcript = [pane.rect[0] + pane.rect[2] / 2, pane.rect[1] + pane.rect[3] * 0.45];
return { rows, composer: center(composer), transcript };
} finally {
await browser.close().catch(() => {});
}
}
// ---- Measured loads -------------------------------------------------------
async function measure(name, extra, drive) {
const runs = [];
let report = null;
for (let rep = 0; rep < Number(opts.repeat); rep++) {
const browser = await openBrowser({ gpu: Boolean(opts.gpu) });
try {
const { send, on } = browser;
const evaluate = async (expression) => (await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true })).result?.value;
let frames = [];
let first = null;
const errors = [];
on('Runtime.consoleAPICalled', (p) => {
const text = p.args.map((a) => a.value ?? '').join(' ');
if (text.startsWith('PERF_FRAME ')) frames.push(text.split(' ').slice(1).map(Number));
else if (text.startsWith('PERF_FIRST_FRAME ')) first = Number(text.split(' ')[1]);
else if (p.type === 'error') errors.push(text.slice(0, 160));
});
await send('Page.enable');
await send('Runtime.enable');
await send('Emulation.setDeviceMetricsOverride', { width: WIDTH, height: HEIGHT, deviceScaleFactor: DPR, mobile: false });
const io = {
send, evaluate,
async settle(ms = 1200) { await delay(ms); for (let i = 0; i < 2; i++) { await evaluate('window.__perfKick?.()'); await delay(250); } },
async tap([x, y]) {
for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) {
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 });
}
},
async wheel([x, y], deltaY, count) {
for (let i = 0; i < count; i++) {
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX: 0, deltaY });
await delay(16);
}
},
async type(text, gap = 60) { for (const ch of text) { await send('Input.insertText', { text: ch }); await delay(gap); } },
};
const profile = rep === 0;
if (profile && name === 'open') { await send('Profiler.enable'); await send('Profiler.setSamplingInterval', { interval: 200 }); await send('Profiler.start'); }
const navigated = Date.now();
await send('Page.navigate', { url: url(server.origin, extra) });
for (let i = 0; i < 400 && first === null; i++) await delay(50);
let result;
if (name === 'open') {
await io.settle(3000);
result = { firstFrameMs: first, wallMs: Date.now() - navigated, ...summarize(frames) };
} else {
await io.settle(3500);
frames = [];
if (profile) { await send('Profiler.enable'); await send('Profiler.setSamplingInterval', { interval: 200 }); await send('Profiler.start'); }
const started = Date.now();
await drive(io);
await io.settle();
result = { wallMs: Date.now() - started, ...summarize(frames) };
}
if (profile) {
const { profile: cpu } = await send('Profiler.stop');
await writeFile(join(opts.out, `${name}.cpuprofile`), JSON.stringify(cpu));
report = {
...attribute(cpu, lookup, { n: 25 }),
app: attribute(cpu, lookup, { n: 25, filter: (k) => k.startsWith('app:') }),
self: selfTime(cpu, lookup, 25),
};
}
if (errors.length) result.errors = [...new Set(errors)].slice(0, 3);
runs.push(result);
} finally {
await browser.close().catch(() => {});
}
}
const keys = Object.keys(runs[0]).filter((k) => typeof runs[0][k] === 'number');
const med = Object.fromEntries(keys.map((k) => [k, median(runs.map((r) => r[k]))]));
return { median: med, runs, cpu: report };
}
const points = await discover();
console.error('layout', JSON.stringify(points));
const SCENARIOS = {
open: { extra: {}, drive: null },
scroll: {
extra: {},
drive: async (io) => {
await io.wheel(points.transcript, -400, 60);
await delay(300);
await io.wheel(points.transcript, 400, 60);
},
},
stream: {
extra: { live: opts.live },
// The fixture starts streaming 3 s after the chat subscribes; the settle
// before drive() covers most of that, then every delta is 60 ms apart.
drive: async () => { await delay(Number(opts.live) * 60 + 1500); },
},
switch: {
extra: {},
drive: async (io) => {
for (const row of [1, 2, 3, 4, 5, 0].map((i) => points.rows[i])) {
await io.tap(row);
await delay(900);
}
},
},
type: {
extra: {},
drive: async (io) => {
await io.tap(points.composer);
await delay(300);
await io.type('Profiling the composer at 4K: does every keystroke rebuild the transcript?');
},
},
};
const results = {};
try {
const only = opts.only?.split(',');
for (const [name, s] of Object.entries(SCENARIOS)) {
if (only && !only.includes(name)) continue;
results[name] = await measure(name, s.extra, s.drive);
console.log(name, JSON.stringify(results[name].median));
if (results[name].cpu) {
console.log(` cpu busy ${results[name].cpu.busyMs} ms; files:`);
for (const line of results[name].cpu.files.slice(0, 10)) console.log(' ', line);
console.log(' app files (inclusive):');
for (const line of results[name].cpu.app.files.slice(0, 15)) console.log(' ', line);
console.log(' app functions (inclusive):');
for (const line of results[name].cpu.app.functions.slice(0, 20)) console.log(' ', line);
console.log(' self time:');
for (const line of results[name].cpu.self.slice(0, 20)) console.log(' ', line);
}
}
await writeFile(join(opts.out, 'chat-4k-bench.json'), JSON.stringify({
build: opts.build, size: opts.size, dpr: DPR, gpu: Boolean(opts.gpu), data: DATA, points, results,
}, null, 2));
} finally {
await server.close();
}