Make chat pane benchmark recording and metrics readable
This commit is contained in:
@@ -1,82 +1,211 @@
|
||||
#!/usr/bin/env node
|
||||
// Repeated real pointer switches over the full-app rich-chat fixture.
|
||||
// Uses the generic capture browser and Flutter frame timing hook; never a live server.
|
||||
import {mkdir,writeFile} from 'node:fs/promises';
|
||||
import {join} from 'node:path';
|
||||
import {parseArgs} from 'node:util';
|
||||
import {setTimeout as delay} from 'node:timers/promises';
|
||||
import {openBrowser} from '../capture/browser.mjs';
|
||||
import {serveBuild} from '../capture/serve.mjs';
|
||||
import {flutterPage} from '../capture/page.mjs';
|
||||
import {attribute,loadSourceMap} from './cpu_attribution.mjs';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { openBrowser } from '../capture/browser.mjs';
|
||||
import { serveBuild } from '../capture/serve.mjs';
|
||||
import { flutterPage } from '../capture/page.mjs';
|
||||
import { attribute, loadSourceMap } from './cpu_attribution.mjs';
|
||||
|
||||
const {values:o}=parseArgs({options:{build:{type:'string'},out:{type:'string'},repeat:{type:'string',default:'3'},switches:{type:'string',default:'10'},cadence:{type:'string',default:'400'},platform:{type:'string',default:'material'},messages:{type:'string',default:'120'}}});
|
||||
if(!o.build||!o.out)throw Error('--build and --out required');
|
||||
const quantile=(xs,p)=>{const s=[...xs].sort((a,b)=>a-b);return s.length?s[Math.min(s.length-1,Math.floor(s.length*p))]:0;};
|
||||
const round=x=>Math.round(x*100)/100;
|
||||
await mkdir(o.out,{recursive:true});
|
||||
const lookup=await loadSourceMap(o.build);
|
||||
const server=await serveBuild(o.build);
|
||||
const browser=await openBrowser();
|
||||
const page=flutterPage(browser,()=>{});
|
||||
const frames=[];
|
||||
browser.on('Runtime.consoleAPICalled',p=>{const line=p.args.map(a=>a.value??'').join(' ');if(line.startsWith('PERF_FRAME '))frames.push(line.split(' ').slice(1).map(Number));});
|
||||
const runs=[];
|
||||
try{
|
||||
await browser.send('Page.enable');
|
||||
await browser.send('Runtime.enable');
|
||||
await browser.send('Page.addScriptToEvaluateOnNewDocument',{source:'try { localStorage.clear(); sessionStorage.clear(); } catch (_) {}'});
|
||||
for(let rep=0;rep<Number(o.repeat);rep++){
|
||||
const url=new URL('/',server.origin);
|
||||
for(const [k,v] of Object.entries({route:'/chat/perf-chat-0',platform:o.platform,theme:'light',chats:'30',msgs:o.messages,rich:'1',perf:'1'}))url.searchParams.set(k,v);
|
||||
await page.open(url.href,{width:560,height:1000,theme:'light'});
|
||||
await page.click('Add chat pane');await delay(250);
|
||||
await page.click('Fixture chat 2');await delay(900);
|
||||
// Warm both retained transcripts before recording.
|
||||
for(let i=0;i<2;i++){
|
||||
await page.click('eval:(()=>{const e=document.querySelector(\'[flt-semantics-identifier^="chat-peek-"]\');if(!e)return null;const r=e.getBoundingClientRect();return [r.x+r.width/2,r.y+r.height/2]})()');
|
||||
await delay(500);
|
||||
const { values: options } = parseArgs({
|
||||
options: {
|
||||
build: { type: 'string' },
|
||||
out: { type: 'string' },
|
||||
repeat: { type: 'string', default: '3' },
|
||||
switches: { type: 'string', default: '10' },
|
||||
cadence: { type: 'string', default: '400' },
|
||||
platform: { type: 'string', default: 'material' },
|
||||
messages: { type: 'string', default: '120' },
|
||||
},
|
||||
});
|
||||
if (!options.build || !options.out) throw new Error('--build and --out required');
|
||||
|
||||
function quantile(values, proportion) {
|
||||
const sorted = [...values].sort((a, b) => a - b);
|
||||
return sorted.length
|
||||
? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * proportion))]
|
||||
: 0;
|
||||
}
|
||||
const round = (value) => Math.round(value * 100) / 100;
|
||||
|
||||
// The peek always identifies the other retained pane after each switch.
|
||||
const peekTarget = `eval:(() => {
|
||||
const element = document.querySelector('[flt-semantics-identifier^="chat-peek-"]');
|
||||
if (!element) return null;
|
||||
const rect = element.getBoundingClientRect();
|
||||
return [rect.x + rect.width / 2, rect.y + rect.height / 2];
|
||||
})()`;
|
||||
|
||||
await mkdir(options.out, { recursive: true });
|
||||
const lookup = await loadSourceMap(options.build);
|
||||
const server = await serveBuild(options.build);
|
||||
const browser = await openBrowser();
|
||||
const page = flutterPage(browser, () => {});
|
||||
const frames = [];
|
||||
const runs = [];
|
||||
|
||||
browser.on('Runtime.consoleAPICalled', (event) => {
|
||||
const line = event.args.map((argument) => argument.value ?? '').join(' ');
|
||||
if (line.startsWith('PERF_FRAME ')) {
|
||||
frames.push(line.split(' ').slice(1).map(Number));
|
||||
}
|
||||
await browser.send('Profiler.enable');
|
||||
await browser.send('Profiler.setSamplingInterval',{interval:100});
|
||||
await page.evaluate(`(()=>{window.__paneFrames=[];window.__paneLongTasks=[];window.__paneRecording=true;
|
||||
window.__paneObserver=new PerformanceObserver(l=>{if(window.__paneRecording)window.__paneLongTasks.push(...l.getEntries().map(e=>e.duration));});
|
||||
window.__paneObserver.observe({type:'longtask',buffered:false});
|
||||
let last=performance.now();const tick=t=>{if(!window.__paneRecording)return;window.__paneFrames.push(t-last);last=t;requestAnimationFrame(tick);};requestAnimationFrame(tick);return true;})()`);
|
||||
frames.length=0;
|
||||
await browser.send('Profiler.start');
|
||||
const start=performance.now();
|
||||
for(let i=0;i<Number(o.switches);i++){
|
||||
await page.click('eval:(()=>{const e=document.querySelector(\'[flt-semantics-identifier^="chat-peek-"]\');if(!e)return null;const r=e.getBoundingClientRect();return [r.x+r.width/2,r.y+r.height/2]})()');
|
||||
await delay(Number(o.cadence));
|
||||
});
|
||||
|
||||
try {
|
||||
await browser.send('Page.enable');
|
||||
await browser.send('Runtime.enable');
|
||||
await browser.send('Page.addScriptToEvaluateOnNewDocument', {
|
||||
source: 'try { localStorage.clear(); sessionStorage.clear(); } catch (_) {}',
|
||||
});
|
||||
|
||||
for (let repetition = 0; repetition < Number(options.repeat); repetition++) {
|
||||
const url = new URL('/', server.origin);
|
||||
const fixture = {
|
||||
route: '/chat/perf-chat-0',
|
||||
platform: options.platform,
|
||||
theme: 'light',
|
||||
chats: '30',
|
||||
msgs: options.messages,
|
||||
rich: '1',
|
||||
perf: '1',
|
||||
};
|
||||
for (const [key, value] of Object.entries(fixture)) {
|
||||
url.searchParams.set(key, value);
|
||||
}
|
||||
await page.open(url.href, { width: 560, height: 1000, theme: 'light' });
|
||||
await page.click('Add chat pane');
|
||||
await delay(250);
|
||||
await page.click('Fixture chat 2');
|
||||
await delay(900);
|
||||
|
||||
// Warm both retained transcripts before recording.
|
||||
for (let index = 0; index < 2; index++) {
|
||||
await page.click(peekTarget);
|
||||
await delay(500);
|
||||
}
|
||||
|
||||
await browser.send('Profiler.enable');
|
||||
await browser.send('Profiler.setSamplingInterval', { interval: 100 });
|
||||
await page.evaluate(`(() => {
|
||||
window.__paneFrames = [];
|
||||
window.__paneLongTasks = [];
|
||||
window.__paneRecording = true;
|
||||
window.__paneObserver = new PerformanceObserver((list) => {
|
||||
if (window.__paneRecording) {
|
||||
window.__paneLongTasks.push(...list.getEntries().map((entry) => entry.duration));
|
||||
}
|
||||
});
|
||||
window.__paneObserver.observe({ type: 'longtask', buffered: false });
|
||||
let last = performance.now();
|
||||
const tick = (time) => {
|
||||
if (!window.__paneRecording) return;
|
||||
window.__paneFrames.push(time - last);
|
||||
last = time;
|
||||
requestAnimationFrame(tick);
|
||||
};
|
||||
requestAnimationFrame(tick);
|
||||
return true;
|
||||
})()`);
|
||||
frames.length = 0;
|
||||
await browser.send('Profiler.start');
|
||||
|
||||
const start = performance.now();
|
||||
for (let index = 0; index < Number(options.switches); index++) {
|
||||
await page.click(peekTarget);
|
||||
await delay(Number(options.cadence));
|
||||
}
|
||||
await delay(300);
|
||||
const elapsed = performance.now() - start;
|
||||
const raf = await page.evaluate(`
|
||||
window.__paneRecording = false;
|
||||
window.__paneObserver.disconnect();
|
||||
({ deltas: window.__paneFrames, longTasks: window.__paneLongTasks })
|
||||
`);
|
||||
const { profile } = await browser.send('Profiler.stop');
|
||||
await writeFile(
|
||||
join(options.out, `run-${repetition + 1}.cpuprofile`),
|
||||
JSON.stringify(profile),
|
||||
);
|
||||
const cpu = attribute(profile, lookup, { n: 15 });
|
||||
await writeFile(
|
||||
join(options.out, `run-${repetition + 1}-cpu.json`),
|
||||
JSON.stringify(cpu, null, 2),
|
||||
);
|
||||
|
||||
// Flush timing batches without counting navigation/setup frames.
|
||||
await delay(500);
|
||||
for (let index = 0; index < 2; index++) {
|
||||
await page.evaluate('window.__perfKick?.();true');
|
||||
await delay(150);
|
||||
}
|
||||
const useful = frames.filter((frame) => frame[0] + frame[1] > 300);
|
||||
const sum = (index) => round(
|
||||
useful.reduce((total, frame) => total + frame[index], 0) / 1000,
|
||||
);
|
||||
const max = (index) => round(
|
||||
Math.max(0, ...useful.map((frame) => frame[index])) / 1000,
|
||||
);
|
||||
const p95 = (index) => round(quantile(
|
||||
useful.map((frame) => frame[index] / 1000),
|
||||
.95,
|
||||
));
|
||||
const result = {
|
||||
rep: repetition + 1,
|
||||
elapsedMs: round(elapsed),
|
||||
switches: Number(options.switches),
|
||||
cadenceMs: Number(options.cadence),
|
||||
frames: useful.length,
|
||||
buildSumMs: sum(0),
|
||||
buildMaxMs: max(0),
|
||||
buildP95Ms: p95(0),
|
||||
rasterSumMs: sum(1),
|
||||
rasterMaxMs: max(1),
|
||||
rasterP95Ms: p95(1),
|
||||
totalP95Ms: p95(2),
|
||||
over16: useful.filter((frame) => frame[2] > 16667).length,
|
||||
rafP95Ms: round(quantile(raf.deltas, .95)),
|
||||
rafMaxMs: round(Math.max(0, ...raf.deltas)),
|
||||
longTasks: raf.longTasks.length,
|
||||
longTaskMs: round(raf.longTasks.reduce((total, duration) => total + duration, 0)),
|
||||
cpu,
|
||||
};
|
||||
runs.push(result);
|
||||
await writeFile(
|
||||
join(options.out, `run-${repetition + 1}-frames.json`),
|
||||
JSON.stringify({ flutter: useful, raf }, null, 2),
|
||||
);
|
||||
if (repetition === 0) {
|
||||
await writeFile(join(options.out, 'final.png'), await page.shot());
|
||||
}
|
||||
console.log(JSON.stringify({ ...result, cpu: undefined }));
|
||||
}
|
||||
await delay(300);
|
||||
const elapsed=performance.now()-start;
|
||||
const raf=await page.evaluate('window.__paneRecording=false;window.__paneObserver.disconnect();({deltas:window.__paneFrames,longTasks:window.__paneLongTasks})');
|
||||
const {profile}=await browser.send('Profiler.stop');
|
||||
await writeFile(join(o.out,`run-${rep+1}.cpuprofile`),JSON.stringify(profile));
|
||||
const cpu=attribute(profile,lookup,{n:15});
|
||||
await writeFile(join(o.out,`run-${rep+1}-cpu.json`),JSON.stringify(cpu,null,2));
|
||||
// Flush timing batches without counting navigation/setup frames.
|
||||
await delay(500);
|
||||
for(let i=0;i<2;i++){await page.evaluate('window.__perfKick?.();true');await delay(150);}
|
||||
const useful=frames.filter(f=>f[0]+f[1]>300);
|
||||
const sum=i=>round(useful.reduce((a,f)=>a+f[i],0)/1000);
|
||||
const max=i=>round(Math.max(0,...useful.map(f=>f[i]))/1000);
|
||||
const p95=i=>round(quantile(useful.map(f=>f[i]/1000),.95));
|
||||
const result={rep:rep+1,elapsedMs:round(elapsed),switches:Number(o.switches),cadenceMs:Number(o.cadence),frames:useful.length,
|
||||
buildSumMs:sum(0),buildMaxMs:max(0),buildP95Ms:p95(0),rasterSumMs:sum(1),rasterMaxMs:max(1),rasterP95Ms:p95(1),
|
||||
totalP95Ms:p95(2),over16:useful.filter(f=>f[2]>16667).length,rafP95Ms:round(quantile(raf.deltas,.95)),rafMaxMs:round(Math.max(0,...raf.deltas)),
|
||||
longTasks:raf.longTasks.length,longTaskMs:round(raf.longTasks.reduce((a,b)=>a+b,0)),cpu};
|
||||
runs.push(result);
|
||||
await writeFile(join(o.out,`run-${rep+1}-frames.json`),JSON.stringify({flutter:useful,raf},null,2));
|
||||
if(rep===0)await writeFile(join(o.out,'final.png'),await page.shot());
|
||||
console.log(JSON.stringify({...result,cpu:undefined}));
|
||||
}
|
||||
const metrics=['frames','buildSumMs','buildMaxMs','buildP95Ms','rasterSumMs','rasterP95Ms','totalP95Ms','over16','rafP95Ms','longTasks','longTaskMs'];
|
||||
const medians=Object.fromEntries(metrics.map(k=>[k,round(quantile(runs.map(r=>r[k]),.5))]));
|
||||
const report={build:o.build,platform:o.platform,messages:Number(o.messages),size:'560x1000',headlessCpuRaster:true,semanticsEnabled:true,runs,medians,errors:page.errors};
|
||||
await writeFile(join(o.out,'report.json'),JSON.stringify(report,null,2));
|
||||
if(page.errors.length)throw Error('Browser errors: '+page.errors.join('\n'));
|
||||
console.log('MEDIANS',JSON.stringify(medians));
|
||||
}finally{await browser.close();await server.close();}
|
||||
|
||||
const metrics = [
|
||||
'frames', 'buildSumMs', 'buildMaxMs', 'buildP95Ms', 'rasterSumMs',
|
||||
'rasterP95Ms', 'totalP95Ms', 'over16', 'rafP95Ms', 'longTasks', 'longTaskMs',
|
||||
];
|
||||
const medians = Object.fromEntries(metrics.map((metric) => [
|
||||
metric,
|
||||
round(quantile(runs.map((run) => run[metric]), .5)),
|
||||
]));
|
||||
const report = {
|
||||
build: options.build,
|
||||
platform: options.platform,
|
||||
messages: Number(options.messages),
|
||||
size: '560x1000',
|
||||
headlessCpuRaster: true,
|
||||
semanticsEnabled: true,
|
||||
runs,
|
||||
medians,
|
||||
errors: page.errors,
|
||||
};
|
||||
await writeFile(join(options.out, 'report.json'), JSON.stringify(report, null, 2));
|
||||
if (page.errors.length) throw new Error('Browser errors: ' + page.errors.join('\n'));
|
||||
console.log('MEDIANS', JSON.stringify(medians));
|
||||
} finally {
|
||||
await browser.close();
|
||||
await server.close();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user