From 58c72538d5429882d3a3dad4065e780f1e607100 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Sun, 22 Nov 2026 16:45:22 +0000 Subject: [PATCH] Add a 4K chat performance bench and record its baseline tool/perf/chat_4k_bench.mjs opens the full-app fixture at 3840x2160 with 12 rich chats and measures opening, scrolling, streaming, switching and typing (frame timings plus a source-mapped CPU profile). The capture kit's --gpu mode now uses ANGLE on Vulkan on Linux. notes/perf-4k-chat.md records the baseline and the hotspots it found. Co-Authored-By: Claude Opus 5.5 --- app/notes/perf-4k-chat.md | 66 +++++++++ app/tool/capture/browser.mjs | 5 +- app/tool/perf/chat_4k_bench.mjs | 252 ++++++++++++++++++++++++++++++++ 3 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 app/notes/perf-4k-chat.md create mode 100644 app/tool/perf/chat_4k_bench.mjs diff --git a/app/notes/perf-4k-chat.md b/app/notes/perf-4k-chat.md new file mode 100644 index 00000000..207cba2f --- /dev/null +++ b/app/notes/perf-4k-chat.md @@ -0,0 +1,66 @@ +# Chat performance at 4K: baseline and hotspots (2026-11-22) + +`tool/perf/chat_4k_bench.mjs` drives the full-app fixture (profile build with +source maps, no server) at 3840x2160 CSS px, device scale 1, with the GPU +(`--gpu`: ANGLE on Vulkan, RTX 2080 Ti; `tool/capture/browser.mjs` now picks +Vulkan on Linux and Metal on macOS). Data: 12 chats x 150 rich messages +(code, wide tables, images, agent reports, links, turn activity, shown items, +turn diffs). Semantics stays off in the measured loads. + +```sh +flutter build web --profile --source-maps \ + --target tool/search_selection_probe_entry.dart --output build/probe-profile +node tool/perf/chat_4k_bench.mjs --build build/probe-profile --out /tmp/chat4k --gpu --repeat 3 +``` + +## Baseline (median of 3; frame = Flutter FrameTiming) + +| Scenario | Frames | Build p50 / p90 / max ms | >16 ms | >33 ms | Worst frame | +| --- | --- | --- | --- | --- | --- | +| open (first frame 1.4 s) | 67 | 1.9 / 9.5 / 117 | 7 | 5 | 152 ms | +| scroll 60 wheel ticks up, 60 down | 458 | 2.3 / 6.2 / 29 | 38 | 2 | 38 ms | +| stream 200 deltas (60 ms apart) | 911 | 2.9 / 6.4 / 9 | 1 | 0 | 17 ms | +| switch through 6 chats | 291 | 2.1 / 6.3 / 65 | 19 | 12 | 90 ms | +| type 74 characters | 100 | 5.2 / 5.9 / 11 | 1 | 0 | 19 ms | + +Raster is cheap with a GPU (p90 under 6 ms); the UI thread is the cost. +`printString` (the `PERF_FRAME` lines) and `_reportTaskEvent` (profile-build +timeline) in the self-time lists are measurement overhead. + +## Hotspots, by size + +1. **Running-turn composer border** (`composer_running_border.dart:116`, + `composer_running_arc.dart:135`): 602 ms inclusive over the 15 s stream, + the largest app cost. Every animation frame rebuilds the pill's outline + path, runs `computeMetrics()` and extracts the highlight; CanvasKit's + path measuring and its wire conversions (`fromWireType`, 301 ms self) + dominate. The outline and its metrics only change with size and shape: + cache them and extract per frame. Runs for every running turn, on every + device. +2. **Transcript selection ordering while scrolling**: 241 ms in one scroll. + `MultiSelectableSelectionContainerDelegate._flushAdditions` sorts every + selectable text piece of the transcript-wide selection area by screen + position (`_compareScreenOrder` -> `getTransformTo`) as rows scroll in. + Ordering by transcript row (a delegate that compares row indices, or one + selection container per message so the top level sorts messages, not text + runs) avoids the matrix work. Main cause of the 38 slow scroll frames. +3. **Table column measuring** (`chat_table_column_width.dart`, added + 2026-11-19 for wrapping tables): 142 ms (max + min intrinsic) in the + scroll, 61 ms switching chats, 21 ms opening. Each cell's paragraph is laid + out for its max width, its min width and then its real width. Caching per + cell text, or skipping the min pass when the max widths already fit, cuts + it. +4. **Per-delta transcript rebuild**: `transcriptRows` 41 ms, + `withChatShownItems` 28 ms (`_rowTime` 15 ms), `ChatRowKeys.forRows` + 16 ms, `sliver_animated_extent` layout 250 ms inclusive over the stream. + Every delta re-derives all rows of the transcript; only the live message + changed. Memoise the settled rows and recompute the tail. +5. **Spinner during streaming**: `CircularProgressIndicator`'s controller + getter looks up an inherited widget every frame (89 ms over the stream) on + top of the border animation; two 60 fps animations for one running turn. +6. **Switching chats**: 12 frames over 33 ms per 6 switches, worst 90 ms; + includes a first-use shader compile (`getShaderParameter` 27 ms) and the + new transcript's table measuring (item 3). +7. **Typing**: a `LayoutBuilder` subtree above the composer rebuilds on every + keystroke (166 ms over 74 keys, about 2 ms each), and the draft is written + to preferences per keystroke (3 ms total). Low priority. diff --git a/app/tool/capture/browser.mjs b/app/tool/capture/browser.mjs index c6f973ae..90c5f106 100644 --- a/app/tool/capture/browser.mjs +++ b/app/tool/capture/browser.mjs @@ -17,7 +17,10 @@ async function launch(chromePath, gpu) { '--headless=new', '--no-first-run', '--no-default-browser-check', // Without a GPU CanvasKit rasterises on the CPU; `gpu` keeps WebGL (Metal // through ANGLE on a Mac) for benches that measure raster time. - ...(gpu ? ['--use-angle=metal', '--enable-gpu-rasterization', '--ignore-gpu-blocklist'] : ['--disable-gpu']), + // Linux keeps the GPU through ANGLE on Vulkan. + ...(gpu ? [process.platform === 'darwin' ? '--use-angle=metal' : '--use-angle=vulkan', + ...(process.platform === 'darwin' ? [] : ['--enable-features=Vulkan']), + '--enable-gpu-rasterization', '--ignore-gpu-blocklist'] : ['--disable-gpu']), '--hide-scrollbars', '--mute-audio', '--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank', ], { stdio: ['ignore', 'ignore', 'pipe'] }); diff --git a/app/tool/perf/chat_4k_bench.mjs b/app/tool/perf/chat_4k_bench.mjs new file mode 100644 index 00000000..ad767a4a --- /dev/null +++ b/app/tool/perf/chat_4k_bench.mjs @@ -0,0 +1,252 @@ +#!/usr/bin/env node +// Chat performance at 4K over the full-app fixture (tool/search_selection_probe_entry.dart), +// never a live server. Build it as a profile build with source maps, from app/: +// +// flutter build web --profile --source-maps \ +// --target tool/search_selection_probe_entry.dart --output build/probe-profile +// node tool/perf/chat_4k_bench.mjs --build build/probe-profile --out /tmp/chat4k [--gpu] [--repeat 3] +// +// The window is 3840x2160 CSS px at device scale 1 (a 4K monitor at 100%); +// --size and --dpr change it (1920x1080 at --dpr 2 is the same panel at 200%). +// Data: 12 chats of 150 rich messages (code, wide tables, images, agent +// reports, links, turn activity, shown items and turn diffs). +// +// A discovery load (semantics on) finds the chat rows, the composer and the +// transcript; the measured loads keep semantics off, as for most users, and +// drive the app with CDP pointer, wheel and text input. Each scenario runs on a +// fresh load. Flutter's FrameTiming comes from tool/perf/frame_log.dart; the +// first repetition also records a V8 CPU profile, attributed to Dart files +// through the source map (tool/perf/cpu_attribution.mjs). +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { openBrowser } from '../capture/browser.mjs'; +import { flutterPage } from '../capture/page.mjs'; +import { serveBuild } from '../capture/serve.mjs'; +import { attribute, loadSourceMap } from './cpu_attribution.mjs'; + +const { values: opts } = parseArgs({ + options: { + build: { type: 'string' }, out: { type: 'string' }, repeat: { type: 'string', default: '3' }, + gpu: { type: 'boolean' }, size: { type: 'string', default: '3840x2160' }, dpr: { type: 'string', default: '1' }, + only: { type: 'string' }, live: { type: 'string', default: '200' }, + }, +}); +if (!opts.build || !opts.out) throw new Error('usage: --build --out [--gpu]'); +const [WIDTH, HEIGHT] = opts.size.split('x').map(Number); +const DPR = Number(opts.dpr); +const DATA = { + chats: '12', msgs: '150', rich: '1', links: '1', turnactivity: '1', + timestamps: '1', shown: '1', turndiffs: '1', theme: 'light', +}; + +const r1 = (x) => Math.round(x * 10) / 10; + +/** Self (leaf) ms per mapped function: where the CPU actually was. */ +function selfTime(profile, lookup, n) { + const byId = new Map(profile.nodes.map((node) => [node.id, node])); + const self = new Map(); + profile.samples.forEach((id, i) => { + const dt = (profile.timeDeltas[i + 1] ?? 0) / 1000; + const f = byId.get(id).callFrame; + if (['(idle)', '(program)', '(garbage collector)'].includes(f.functionName)) { + if (f.functionName === '(garbage collector)') self.set('(gc)', (self.get('(gc)') ?? 0) + dt); + return; + } + const m = f.url.endsWith('main.dart.js') ? lookup(f.lineNumber, f.columnNumber) : null; + const key = m ? `${m.source}:${m.line} ${f.functionName}` : `(${f.url.split('/').pop() || 'native'}) ${f.functionName}`; + self.set(key, (self.get(key) ?? 0) + dt); + }); + return [...self].sort((a, b) => b[1] - a[1]).slice(0, n).map(([k, v]) => `${r1(v)} ${k}`); +} +const median = (xs) => { const s = [...xs].sort((a, b) => a - b); return s[Math.floor(s.length / 2)]; }; +const pct = (xs, p) => { const s = [...xs].sort((a, b) => a - b); return s.length ? s[Math.min(s.length - 1, Math.floor(s.length * p))] : 0; }; + +function summarize(frames) { + const ms = (us) => us / 1000; + const build = frames.map((f) => ms(f[0])); + const raster = frames.map((f) => ms(f[1])); + const total = frames.map((f) => ms(f[2])); + return { + frames: frames.length, + buildP50: r1(pct(build, 0.5)), buildP90: r1(pct(build, 0.9)), buildMax: r1(Math.max(0, ...build)), + rasterP50: r1(pct(raster, 0.5)), rasterP90: r1(pct(raster, 0.9)), rasterMax: r1(Math.max(0, ...raster)), + totalP90: r1(pct(total, 0.9)), totalMax: r1(Math.max(0, ...total)), + over16: total.filter((t) => t > 16.7).length, over33: total.filter((t) => t > 33.4).length, + buildSum: r1(build.reduce((a, b) => a + b, 0)), + }; +} + +function url(origin, extra = {}) { + const u = new URL('/', origin); + u.searchParams.set('route', '/chat/perf-chat-0'); + for (const [k, v] of Object.entries({ ...DATA, perf: '1', ...extra })) u.searchParams.set(k, v); + return u.href; +} + +const server = await serveBuild(opts.build); +const lookup = await loadSourceMap(opts.build); +await mkdir(opts.out, { recursive: true }); + +// ---- Discovery: where the rows, the composer and the transcript are -------- +async function discover() { + const browser = await openBrowser({ gpu: Boolean(opts.gpu) }); + try { + const page = flutterPage(browser, () => {}); + await page.open(url(server.origin, { perf: '0' }), { width: WIDTH, height: HEIGHT }); + await delay(2500); + const labels = await page.labels(); + const center = (n) => [n.rect[0] + n.rect[2] / 2, n.rect[1] + n.rect[3] / 2]; + const rows = labels.filter((n) => /^OpenCode harness\nFixture chat \d+/.test(n.label)) + .sort((a, b) => a.rect[1] - b.rect[1]).map(center); + const composer = labels.find((n) => n.role === 'textarea' || n.role === 'input' || /^Work on/.test(n.label)); + const pane = labels.find((n) => n.key === 'list-detail-body'); + if (rows.length < 6 || !composer || !pane) { + await writeFile(join(opts.out, 'discovery-labels.json'), JSON.stringify(labels, null, 2)); + throw new Error(`discovery failed: rows=${rows.length} composer=${Boolean(composer)} pane=${Boolean(pane)}`); + } + await writeFile(join(opts.out, 'layout.png'), await page.shot()); + const transcript = [pane.rect[0] + pane.rect[2] / 2, pane.rect[1] + pane.rect[3] * 0.45]; + return { rows, composer: center(composer), transcript }; + } finally { + await browser.close().catch(() => {}); + } +} + +// ---- Measured loads ------------------------------------------------------- +async function measure(name, extra, drive) { + const runs = []; + let report = null; + for (let rep = 0; rep < Number(opts.repeat); rep++) { + const browser = await openBrowser({ gpu: Boolean(opts.gpu) }); + try { + const { send, on } = browser; + const evaluate = async (expression) => (await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true })).result?.value; + let frames = []; + let first = null; + const errors = []; + on('Runtime.consoleAPICalled', (p) => { + const text = p.args.map((a) => a.value ?? '').join(' '); + if (text.startsWith('PERF_FRAME ')) frames.push(text.split(' ').slice(1).map(Number)); + else if (text.startsWith('PERF_FIRST_FRAME ')) first = Number(text.split(' ')[1]); + else if (p.type === 'error') errors.push(text.slice(0, 160)); + }); + await send('Page.enable'); + await send('Runtime.enable'); + await send('Emulation.setDeviceMetricsOverride', { width: WIDTH, height: HEIGHT, deviceScaleFactor: DPR, mobile: false }); + const io = { + send, evaluate, + async settle(ms = 1200) { await delay(ms); for (let i = 0; i < 2; i++) { await evaluate('window.__perfKick?.()'); await delay(250); } }, + async tap([x, y]) { + for (const type of ['mouseMoved', 'mousePressed', 'mouseReleased']) { + await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 }); + } + }, + async wheel([x, y], deltaY, count) { + for (let i = 0; i < count; i++) { + await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x, y, deltaX: 0, deltaY }); + await delay(16); + } + }, + async type(text, gap = 60) { for (const ch of text) { await send('Input.insertText', { text: ch }); await delay(gap); } }, + }; + const profile = rep === 0; + if (profile && name === 'open') { await send('Profiler.enable'); await send('Profiler.setSamplingInterval', { interval: 200 }); await send('Profiler.start'); } + const navigated = Date.now(); + await send('Page.navigate', { url: url(server.origin, extra) }); + for (let i = 0; i < 400 && first === null; i++) await delay(50); + let result; + if (name === 'open') { + await io.settle(3000); + result = { firstFrameMs: first, wallMs: Date.now() - navigated, ...summarize(frames) }; + } else { + await io.settle(3500); + frames = []; + if (profile) { await send('Profiler.enable'); await send('Profiler.setSamplingInterval', { interval: 200 }); await send('Profiler.start'); } + const started = Date.now(); + await drive(io); + await io.settle(); + result = { wallMs: Date.now() - started, ...summarize(frames) }; + } + if (profile) { + const { profile: cpu } = await send('Profiler.stop'); + await writeFile(join(opts.out, `${name}.cpuprofile`), JSON.stringify(cpu)); + report = { + ...attribute(cpu, lookup, { n: 25 }), + app: attribute(cpu, lookup, { n: 25, filter: (k) => k.startsWith('app:') }), + self: selfTime(cpu, lookup, 25), + }; + } + if (errors.length) result.errors = [...new Set(errors)].slice(0, 3); + runs.push(result); + } finally { + await browser.close().catch(() => {}); + } + } + const keys = Object.keys(runs[0]).filter((k) => typeof runs[0][k] === 'number'); + const med = Object.fromEntries(keys.map((k) => [k, median(runs.map((r) => r[k]))])); + return { median: med, runs, cpu: report }; +} + +const points = await discover(); +console.error('layout', JSON.stringify(points)); +const SCENARIOS = { + open: { extra: {}, drive: null }, + scroll: { + extra: {}, + drive: async (io) => { + await io.wheel(points.transcript, -400, 60); + await delay(300); + await io.wheel(points.transcript, 400, 60); + }, + }, + stream: { + extra: { live: opts.live }, + // The fixture starts streaming 3 s after the chat subscribes; the settle + // before drive() covers most of that, then every delta is 60 ms apart. + drive: async () => { await delay(Number(opts.live) * 60 + 1500); }, + }, + switch: { + extra: {}, + drive: async (io) => { + for (const row of [1, 2, 3, 4, 5, 0].map((i) => points.rows[i])) { + await io.tap(row); + await delay(900); + } + }, + }, + type: { + extra: {}, + drive: async (io) => { + await io.tap(points.composer); + await delay(300); + await io.type('Profiling the composer at 4K: does every keystroke rebuild the transcript?'); + }, + }, +}; + +const results = {}; +try { + const only = opts.only?.split(','); + for (const [name, s] of Object.entries(SCENARIOS)) { + if (only && !only.includes(name)) continue; + results[name] = await measure(name, s.extra, s.drive); + console.log(name, JSON.stringify(results[name].median)); + if (results[name].cpu) { + console.log(` cpu busy ${results[name].cpu.busyMs} ms; files:`); + for (const line of results[name].cpu.files.slice(0, 10)) console.log(' ', line); + console.log(' app files (inclusive):'); + for (const line of results[name].cpu.app.files.slice(0, 15)) console.log(' ', line); + console.log(' app functions (inclusive):'); + for (const line of results[name].cpu.app.functions.slice(0, 20)) console.log(' ', line); + console.log(' self time:'); + for (const line of results[name].cpu.self.slice(0, 20)) console.log(' ', line); + } + } + await writeFile(join(opts.out, 'chat-4k-bench.json'), JSON.stringify({ + build: opts.build, size: opts.size, dpr: DPR, gpu: Boolean(opts.gpu), data: DATA, points, results, + }, null, 2)); +} finally { + await server.close(); +}