Make chat pane benchmark recording and metrics readable

This commit is contained in:
2026-11-06 22:10:50 +00:00
parent c936eb4e25
commit 283a08ba01
+205 -76
View File
@@ -1,82 +1,211 @@
#!/usr/bin/env node
// Repeated real pointer switches over the full-app rich-chat fixture.
// Uses the generic capture browser and Flutter frame timing hook; never a live server.
import {mkdir,writeFile} from 'node:fs/promises';
import {join} from 'node:path';
import {parseArgs} from 'node:util';
import {setTimeout as delay} from 'node:timers/promises';
import {openBrowser} from '../capture/browser.mjs';
import {serveBuild} from '../capture/serve.mjs';
import {flutterPage} from '../capture/page.mjs';
import {attribute,loadSourceMap} from './cpu_attribution.mjs';
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { parseArgs } from 'node:util';
import { setTimeout as delay } from 'node:timers/promises';
import { openBrowser } from '../capture/browser.mjs';
import { serveBuild } from '../capture/serve.mjs';
import { flutterPage } from '../capture/page.mjs';
import { attribute, loadSourceMap } from './cpu_attribution.mjs';
const {values:o}=parseArgs({options:{build:{type:'string'},out:{type:'string'},repeat:{type:'string',default:'3'},switches:{type:'string',default:'10'},cadence:{type:'string',default:'400'},platform:{type:'string',default:'material'},messages:{type:'string',default:'120'}}});
if(!o.build||!o.out)throw Error('--build and --out required');
const quantile=(xs,p)=>{const s=[...xs].sort((a,b)=>a-b);return s.length?s[Math.min(s.length-1,Math.floor(s.length*p))]:0;};
const round=x=>Math.round(x*100)/100;
await mkdir(o.out,{recursive:true});
const lookup=await loadSourceMap(o.build);
const server=await serveBuild(o.build);
const browser=await openBrowser();
const page=flutterPage(browser,()=>{});
const frames=[];
browser.on('Runtime.consoleAPICalled',p=>{const line=p.args.map(a=>a.value??'').join(' ');if(line.startsWith('PERF_FRAME '))frames.push(line.split(' ').slice(1).map(Number));});
const runs=[];
try{
await browser.send('Page.enable');
await browser.send('Runtime.enable');
await browser.send('Page.addScriptToEvaluateOnNewDocument',{source:'try { localStorage.clear(); sessionStorage.clear(); } catch (_) {}'});
for(let rep=0;rep<Number(o.repeat);rep++){
const url=new URL('/',server.origin);
for(const [k,v] of Object.entries({route:'/chat/perf-chat-0',platform:o.platform,theme:'light',chats:'30',msgs:o.messages,rich:'1',perf:'1'}))url.searchParams.set(k,v);
await page.open(url.href,{width:560,height:1000,theme:'light'});
await page.click('Add chat pane');await delay(250);
await page.click('Fixture chat 2');await delay(900);
// Warm both retained transcripts before recording.
for(let i=0;i<2;i++){
await page.click('eval:(()=>{const e=document.querySelector(\'[flt-semantics-identifier^="chat-peek-"]\');if(!e)return null;const r=e.getBoundingClientRect();return [r.x+r.width/2,r.y+r.height/2]})()');
await delay(500);
const { values: options } = parseArgs({
options: {
build: { type: 'string' },
out: { type: 'string' },
repeat: { type: 'string', default: '3' },
switches: { type: 'string', default: '10' },
cadence: { type: 'string', default: '400' },
platform: { type: 'string', default: 'material' },
messages: { type: 'string', default: '120' },
},
});
if (!options.build || !options.out) throw new Error('--build and --out required');
function quantile(values, proportion) {
const sorted = [...values].sort((a, b) => a - b);
return sorted.length
? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * proportion))]
: 0;
}
const round = (value) => Math.round(value * 100) / 100;
// The peek always identifies the other retained pane after each switch.
const peekTarget = `eval:(() => {
const element = document.querySelector('[flt-semantics-identifier^="chat-peek-"]');
if (!element) return null;
const rect = element.getBoundingClientRect();
return [rect.x + rect.width / 2, rect.y + rect.height / 2];
})()`;
await mkdir(options.out, { recursive: true });
const lookup = await loadSourceMap(options.build);
const server = await serveBuild(options.build);
const browser = await openBrowser();
const page = flutterPage(browser, () => {});
const frames = [];
const runs = [];
browser.on('Runtime.consoleAPICalled', (event) => {
const line = event.args.map((argument) => argument.value ?? '').join(' ');
if (line.startsWith('PERF_FRAME ')) {
frames.push(line.split(' ').slice(1).map(Number));
}
await browser.send('Profiler.enable');
await browser.send('Profiler.setSamplingInterval',{interval:100});
await page.evaluate(`(()=>{window.__paneFrames=[];window.__paneLongTasks=[];window.__paneRecording=true;
window.__paneObserver=new PerformanceObserver(l=>{if(window.__paneRecording)window.__paneLongTasks.push(...l.getEntries().map(e=>e.duration));});
window.__paneObserver.observe({type:'longtask',buffered:false});
let last=performance.now();const tick=t=>{if(!window.__paneRecording)return;window.__paneFrames.push(t-last);last=t;requestAnimationFrame(tick);};requestAnimationFrame(tick);return true;})()`);
frames.length=0;
await browser.send('Profiler.start');
const start=performance.now();
for(let i=0;i<Number(o.switches);i++){
await page.click('eval:(()=>{const e=document.querySelector(\'[flt-semantics-identifier^="chat-peek-"]\');if(!e)return null;const r=e.getBoundingClientRect();return [r.x+r.width/2,r.y+r.height/2]})()');
await delay(Number(o.cadence));
});
try {
await browser.send('Page.enable');
await browser.send('Runtime.enable');
await browser.send('Page.addScriptToEvaluateOnNewDocument', {
source: 'try { localStorage.clear(); sessionStorage.clear(); } catch (_) {}',
});
for (let repetition = 0; repetition < Number(options.repeat); repetition++) {
const url = new URL('/', server.origin);
const fixture = {
route: '/chat/perf-chat-0',
platform: options.platform,
theme: 'light',
chats: '30',
msgs: options.messages,
rich: '1',
perf: '1',
};
for (const [key, value] of Object.entries(fixture)) {
url.searchParams.set(key, value);
}
await page.open(url.href, { width: 560, height: 1000, theme: 'light' });
await page.click('Add chat pane');
await delay(250);
await page.click('Fixture chat 2');
await delay(900);
// Warm both retained transcripts before recording.
for (let index = 0; index < 2; index++) {
await page.click(peekTarget);
await delay(500);
}
await browser.send('Profiler.enable');
await browser.send('Profiler.setSamplingInterval', { interval: 100 });
await page.evaluate(`(() => {
window.__paneFrames = [];
window.__paneLongTasks = [];
window.__paneRecording = true;
window.__paneObserver = new PerformanceObserver((list) => {
if (window.__paneRecording) {
window.__paneLongTasks.push(...list.getEntries().map((entry) => entry.duration));
}
});
window.__paneObserver.observe({ type: 'longtask', buffered: false });
let last = performance.now();
const tick = (time) => {
if (!window.__paneRecording) return;
window.__paneFrames.push(time - last);
last = time;
requestAnimationFrame(tick);
};
requestAnimationFrame(tick);
return true;
})()`);
frames.length = 0;
await browser.send('Profiler.start');
const start = performance.now();
for (let index = 0; index < Number(options.switches); index++) {
await page.click(peekTarget);
await delay(Number(options.cadence));
}
await delay(300);
const elapsed = performance.now() - start;
const raf = await page.evaluate(`
window.__paneRecording = false;
window.__paneObserver.disconnect();
({ deltas: window.__paneFrames, longTasks: window.__paneLongTasks })
`);
const { profile } = await browser.send('Profiler.stop');
await writeFile(
join(options.out, `run-${repetition + 1}.cpuprofile`),
JSON.stringify(profile),
);
const cpu = attribute(profile, lookup, { n: 15 });
await writeFile(
join(options.out, `run-${repetition + 1}-cpu.json`),
JSON.stringify(cpu, null, 2),
);
// Flush timing batches without counting navigation/setup frames.
await delay(500);
for (let index = 0; index < 2; index++) {
await page.evaluate('window.__perfKick?.();true');
await delay(150);
}
const useful = frames.filter((frame) => frame[0] + frame[1] > 300);
const sum = (index) => round(
useful.reduce((total, frame) => total + frame[index], 0) / 1000,
);
const max = (index) => round(
Math.max(0, ...useful.map((frame) => frame[index])) / 1000,
);
const p95 = (index) => round(quantile(
useful.map((frame) => frame[index] / 1000),
.95,
));
const result = {
rep: repetition + 1,
elapsedMs: round(elapsed),
switches: Number(options.switches),
cadenceMs: Number(options.cadence),
frames: useful.length,
buildSumMs: sum(0),
buildMaxMs: max(0),
buildP95Ms: p95(0),
rasterSumMs: sum(1),
rasterMaxMs: max(1),
rasterP95Ms: p95(1),
totalP95Ms: p95(2),
over16: useful.filter((frame) => frame[2] > 16667).length,
rafP95Ms: round(quantile(raf.deltas, .95)),
rafMaxMs: round(Math.max(0, ...raf.deltas)),
longTasks: raf.longTasks.length,
longTaskMs: round(raf.longTasks.reduce((total, duration) => total + duration, 0)),
cpu,
};
runs.push(result);
await writeFile(
join(options.out, `run-${repetition + 1}-frames.json`),
JSON.stringify({ flutter: useful, raf }, null, 2),
);
if (repetition === 0) {
await writeFile(join(options.out, 'final.png'), await page.shot());
}
console.log(JSON.stringify({ ...result, cpu: undefined }));
}
await delay(300);
const elapsed=performance.now()-start;
const raf=await page.evaluate('window.__paneRecording=false;window.__paneObserver.disconnect();({deltas:window.__paneFrames,longTasks:window.__paneLongTasks})');
const {profile}=await browser.send('Profiler.stop');
await writeFile(join(o.out,`run-${rep+1}.cpuprofile`),JSON.stringify(profile));
const cpu=attribute(profile,lookup,{n:15});
await writeFile(join(o.out,`run-${rep+1}-cpu.json`),JSON.stringify(cpu,null,2));
// Flush timing batches without counting navigation/setup frames.
await delay(500);
for(let i=0;i<2;i++){await page.evaluate('window.__perfKick?.();true');await delay(150);}
const useful=frames.filter(f=>f[0]+f[1]>300);
const sum=i=>round(useful.reduce((a,f)=>a+f[i],0)/1000);
const max=i=>round(Math.max(0,...useful.map(f=>f[i]))/1000);
const p95=i=>round(quantile(useful.map(f=>f[i]/1000),.95));
const result={rep:rep+1,elapsedMs:round(elapsed),switches:Number(o.switches),cadenceMs:Number(o.cadence),frames:useful.length,
buildSumMs:sum(0),buildMaxMs:max(0),buildP95Ms:p95(0),rasterSumMs:sum(1),rasterMaxMs:max(1),rasterP95Ms:p95(1),
totalP95Ms:p95(2),over16:useful.filter(f=>f[2]>16667).length,rafP95Ms:round(quantile(raf.deltas,.95)),rafMaxMs:round(Math.max(0,...raf.deltas)),
longTasks:raf.longTasks.length,longTaskMs:round(raf.longTasks.reduce((a,b)=>a+b,0)),cpu};
runs.push(result);
await writeFile(join(o.out,`run-${rep+1}-frames.json`),JSON.stringify({flutter:useful,raf},null,2));
if(rep===0)await writeFile(join(o.out,'final.png'),await page.shot());
console.log(JSON.stringify({...result,cpu:undefined}));
}
const metrics=['frames','buildSumMs','buildMaxMs','buildP95Ms','rasterSumMs','rasterP95Ms','totalP95Ms','over16','rafP95Ms','longTasks','longTaskMs'];
const medians=Object.fromEntries(metrics.map(k=>[k,round(quantile(runs.map(r=>r[k]),.5))]));
const report={build:o.build,platform:o.platform,messages:Number(o.messages),size:'560x1000',headlessCpuRaster:true,semanticsEnabled:true,runs,medians,errors:page.errors};
await writeFile(join(o.out,'report.json'),JSON.stringify(report,null,2));
if(page.errors.length)throw Error('Browser errors: '+page.errors.join('\n'));
console.log('MEDIANS',JSON.stringify(medians));
}finally{await browser.close();await server.close();}
const metrics = [
'frames', 'buildSumMs', 'buildMaxMs', 'buildP95Ms', 'rasterSumMs',
'rasterP95Ms', 'totalP95Ms', 'over16', 'rafP95Ms', 'longTasks', 'longTaskMs',
];
const medians = Object.fromEntries(metrics.map((metric) => [
metric,
round(quantile(runs.map((run) => run[metric]), .5)),
]));
const report = {
build: options.build,
platform: options.platform,
messages: Number(options.messages),
size: '560x1000',
headlessCpuRaster: true,
semanticsEnabled: true,
runs,
medians,
errors: page.errors,
};
await writeFile(join(options.out, 'report.json'), JSON.stringify(report, null, 2));
if (page.errors.length) throw new Error('Browser errors: ' + page.errors.join('\n'));
console.log('MEDIANS', JSON.stringify(medians));
} finally {
await browser.close();
await server.close();
}