From 283a08ba01410486bec7b8aae31747cf1f154303 Mon Sep 17 00:00:00 2001 From: Alan Silva Date: Fri, 6 Nov 2026 22:10:50 +0000 Subject: [PATCH] Make chat pane benchmark recording and metrics readable --- app/tool/perf/chat_pane_bench.mjs | 281 ++++++++++++++++++++++-------- 1 file changed, 205 insertions(+), 76 deletions(-) diff --git a/app/tool/perf/chat_pane_bench.mjs b/app/tool/perf/chat_pane_bench.mjs index d1b544c4..d1478090 100644 --- a/app/tool/perf/chat_pane_bench.mjs +++ b/app/tool/perf/chat_pane_bench.mjs @@ -1,82 +1,211 @@ #!/usr/bin/env node // Repeated real pointer switches over the full-app rich-chat fixture. // Uses the generic capture browser and Flutter frame timing hook; never a live server. -import {mkdir,writeFile} from 'node:fs/promises'; -import {join} from 'node:path'; -import {parseArgs} from 'node:util'; -import {setTimeout as delay} from 'node:timers/promises'; -import {openBrowser} from '../capture/browser.mjs'; -import {serveBuild} from '../capture/serve.mjs'; -import {flutterPage} from '../capture/page.mjs'; -import {attribute,loadSourceMap} from './cpu_attribution.mjs'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; +import { openBrowser } from '../capture/browser.mjs'; +import { serveBuild } from '../capture/serve.mjs'; +import { flutterPage } from '../capture/page.mjs'; +import { attribute, loadSourceMap } from './cpu_attribution.mjs'; -const {values:o}=parseArgs({options:{build:{type:'string'},out:{type:'string'},repeat:{type:'string',default:'3'},switches:{type:'string',default:'10'},cadence:{type:'string',default:'400'},platform:{type:'string',default:'material'},messages:{type:'string',default:'120'}}}); -if(!o.build||!o.out)throw Error('--build and --out required'); -const quantile=(xs,p)=>{const s=[...xs].sort((a,b)=>a-b);return s.length?s[Math.min(s.length-1,Math.floor(s.length*p))]:0;}; -const round=x=>Math.round(x*100)/100; -await mkdir(o.out,{recursive:true}); -const lookup=await loadSourceMap(o.build); -const server=await serveBuild(o.build); -const browser=await openBrowser(); -const page=flutterPage(browser,()=>{}); -const frames=[]; -browser.on('Runtime.consoleAPICalled',p=>{const line=p.args.map(a=>a.value??'').join(' ');if(line.startsWith('PERF_FRAME '))frames.push(line.split(' ').slice(1).map(Number));}); -const runs=[]; -try{ - await browser.send('Page.enable'); - await browser.send('Runtime.enable'); - await browser.send('Page.addScriptToEvaluateOnNewDocument',{source:'try { localStorage.clear(); sessionStorage.clear(); } catch (_) {}'}); - for(let rep=0;rep{const e=document.querySelector(\'[flt-semantics-identifier^="chat-peek-"]\');if(!e)return null;const r=e.getBoundingClientRect();return [r.x+r.width/2,r.y+r.height/2]})()'); - await delay(500); +const { values: options } = parseArgs({ + options: { + build: { type: 'string' }, + out: { type: 'string' }, + repeat: { type: 'string', default: '3' }, + switches: { type: 'string', default: '10' }, + cadence: { type: 'string', default: '400' }, + platform: { type: 'string', default: 'material' }, + messages: { type: 'string', default: '120' }, + }, +}); +if (!options.build || !options.out) throw new Error('--build and --out required'); + +function quantile(values, proportion) { + const sorted = [...values].sort((a, b) => a - b); + return sorted.length + ? sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * proportion))] + : 0; +} +const round = (value) => Math.round(value * 100) / 100; + +// The peek always identifies the other retained pane after each switch. +const peekTarget = `eval:(() => { + const element = document.querySelector('[flt-semantics-identifier^="chat-peek-"]'); + if (!element) return null; + const rect = element.getBoundingClientRect(); + return [rect.x + rect.width / 2, rect.y + rect.height / 2]; +})()`; + +await mkdir(options.out, { recursive: true }); +const lookup = await loadSourceMap(options.build); +const server = await serveBuild(options.build); +const browser = await openBrowser(); +const page = flutterPage(browser, () => {}); +const frames = []; +const runs = []; + +browser.on('Runtime.consoleAPICalled', (event) => { + const line = event.args.map((argument) => argument.value ?? '').join(' '); + if (line.startsWith('PERF_FRAME ')) { + frames.push(line.split(' ').slice(1).map(Number)); } - await browser.send('Profiler.enable'); - await browser.send('Profiler.setSamplingInterval',{interval:100}); - await page.evaluate(`(()=>{window.__paneFrames=[];window.__paneLongTasks=[];window.__paneRecording=true; - window.__paneObserver=new PerformanceObserver(l=>{if(window.__paneRecording)window.__paneLongTasks.push(...l.getEntries().map(e=>e.duration));}); - window.__paneObserver.observe({type:'longtask',buffered:false}); - let last=performance.now();const tick=t=>{if(!window.__paneRecording)return;window.__paneFrames.push(t-last);last=t;requestAnimationFrame(tick);};requestAnimationFrame(tick);return true;})()`); - frames.length=0; - await browser.send('Profiler.start'); - const start=performance.now(); - for(let i=0;i{const e=document.querySelector(\'[flt-semantics-identifier^="chat-peek-"]\');if(!e)return null;const r=e.getBoundingClientRect();return [r.x+r.width/2,r.y+r.height/2]})()'); - await delay(Number(o.cadence)); +}); + +try { + await browser.send('Page.enable'); + await browser.send('Runtime.enable'); + await browser.send('Page.addScriptToEvaluateOnNewDocument', { + source: 'try { localStorage.clear(); sessionStorage.clear(); } catch (_) {}', + }); + + for (let repetition = 0; repetition < Number(options.repeat); repetition++) { + const url = new URL('/', server.origin); + const fixture = { + route: '/chat/perf-chat-0', + platform: options.platform, + theme: 'light', + chats: '30', + msgs: options.messages, + rich: '1', + perf: '1', + }; + for (const [key, value] of Object.entries(fixture)) { + url.searchParams.set(key, value); + } + await page.open(url.href, { width: 560, height: 1000, theme: 'light' }); + await page.click('Add chat pane'); + await delay(250); + await page.click('Fixture chat 2'); + await delay(900); + + // Warm both retained transcripts before recording. + for (let index = 0; index < 2; index++) { + await page.click(peekTarget); + await delay(500); + } + + await browser.send('Profiler.enable'); + await browser.send('Profiler.setSamplingInterval', { interval: 100 }); + await page.evaluate(`(() => { + window.__paneFrames = []; + window.__paneLongTasks = []; + window.__paneRecording = true; + window.__paneObserver = new PerformanceObserver((list) => { + if (window.__paneRecording) { + window.__paneLongTasks.push(...list.getEntries().map((entry) => entry.duration)); + } + }); + window.__paneObserver.observe({ type: 'longtask', buffered: false }); + let last = performance.now(); + const tick = (time) => { + if (!window.__paneRecording) return; + window.__paneFrames.push(time - last); + last = time; + requestAnimationFrame(tick); + }; + requestAnimationFrame(tick); + return true; + })()`); + frames.length = 0; + await browser.send('Profiler.start'); + + const start = performance.now(); + for (let index = 0; index < Number(options.switches); index++) { + await page.click(peekTarget); + await delay(Number(options.cadence)); + } + await delay(300); + const elapsed = performance.now() - start; + const raf = await page.evaluate(` + window.__paneRecording = false; + window.__paneObserver.disconnect(); + ({ deltas: window.__paneFrames, longTasks: window.__paneLongTasks }) + `); + const { profile } = await browser.send('Profiler.stop'); + await writeFile( + join(options.out, `run-${repetition + 1}.cpuprofile`), + JSON.stringify(profile), + ); + const cpu = attribute(profile, lookup, { n: 15 }); + await writeFile( + join(options.out, `run-${repetition + 1}-cpu.json`), + JSON.stringify(cpu, null, 2), + ); + + // Flush timing batches without counting navigation/setup frames. + await delay(500); + for (let index = 0; index < 2; index++) { + await page.evaluate('window.__perfKick?.();true'); + await delay(150); + } + const useful = frames.filter((frame) => frame[0] + frame[1] > 300); + const sum = (index) => round( + useful.reduce((total, frame) => total + frame[index], 0) / 1000, + ); + const max = (index) => round( + Math.max(0, ...useful.map((frame) => frame[index])) / 1000, + ); + const p95 = (index) => round(quantile( + useful.map((frame) => frame[index] / 1000), + .95, + )); + const result = { + rep: repetition + 1, + elapsedMs: round(elapsed), + switches: Number(options.switches), + cadenceMs: Number(options.cadence), + frames: useful.length, + buildSumMs: sum(0), + buildMaxMs: max(0), + buildP95Ms: p95(0), + rasterSumMs: sum(1), + rasterMaxMs: max(1), + rasterP95Ms: p95(1), + totalP95Ms: p95(2), + over16: useful.filter((frame) => frame[2] > 16667).length, + rafP95Ms: round(quantile(raf.deltas, .95)), + rafMaxMs: round(Math.max(0, ...raf.deltas)), + longTasks: raf.longTasks.length, + longTaskMs: round(raf.longTasks.reduce((total, duration) => total + duration, 0)), + cpu, + }; + runs.push(result); + await writeFile( + join(options.out, `run-${repetition + 1}-frames.json`), + JSON.stringify({ flutter: useful, raf }, null, 2), + ); + if (repetition === 0) { + await writeFile(join(options.out, 'final.png'), await page.shot()); + } + console.log(JSON.stringify({ ...result, cpu: undefined })); } - await delay(300); - const elapsed=performance.now()-start; - const raf=await page.evaluate('window.__paneRecording=false;window.__paneObserver.disconnect();({deltas:window.__paneFrames,longTasks:window.__paneLongTasks})'); - const {profile}=await browser.send('Profiler.stop'); - await writeFile(join(o.out,`run-${rep+1}.cpuprofile`),JSON.stringify(profile)); - const cpu=attribute(profile,lookup,{n:15}); - await writeFile(join(o.out,`run-${rep+1}-cpu.json`),JSON.stringify(cpu,null,2)); - // Flush timing batches without counting navigation/setup frames. - await delay(500); - for(let i=0;i<2;i++){await page.evaluate('window.__perfKick?.();true');await delay(150);} - const useful=frames.filter(f=>f[0]+f[1]>300); - const sum=i=>round(useful.reduce((a,f)=>a+f[i],0)/1000); - const max=i=>round(Math.max(0,...useful.map(f=>f[i]))/1000); - const p95=i=>round(quantile(useful.map(f=>f[i]/1000),.95)); - const result={rep:rep+1,elapsedMs:round(elapsed),switches:Number(o.switches),cadenceMs:Number(o.cadence),frames:useful.length, - buildSumMs:sum(0),buildMaxMs:max(0),buildP95Ms:p95(0),rasterSumMs:sum(1),rasterMaxMs:max(1),rasterP95Ms:p95(1), - totalP95Ms:p95(2),over16:useful.filter(f=>f[2]>16667).length,rafP95Ms:round(quantile(raf.deltas,.95)),rafMaxMs:round(Math.max(0,...raf.deltas)), - longTasks:raf.longTasks.length,longTaskMs:round(raf.longTasks.reduce((a,b)=>a+b,0)),cpu}; - runs.push(result); - await writeFile(join(o.out,`run-${rep+1}-frames.json`),JSON.stringify({flutter:useful,raf},null,2)); - if(rep===0)await writeFile(join(o.out,'final.png'),await page.shot()); - console.log(JSON.stringify({...result,cpu:undefined})); - } - const metrics=['frames','buildSumMs','buildMaxMs','buildP95Ms','rasterSumMs','rasterP95Ms','totalP95Ms','over16','rafP95Ms','longTasks','longTaskMs']; - const medians=Object.fromEntries(metrics.map(k=>[k,round(quantile(runs.map(r=>r[k]),.5))])); - const report={build:o.build,platform:o.platform,messages:Number(o.messages),size:'560x1000',headlessCpuRaster:true,semanticsEnabled:true,runs,medians,errors:page.errors}; - await writeFile(join(o.out,'report.json'),JSON.stringify(report,null,2)); - if(page.errors.length)throw Error('Browser errors: '+page.errors.join('\n')); - console.log('MEDIANS',JSON.stringify(medians)); -}finally{await browser.close();await server.close();} + + const metrics = [ + 'frames', 'buildSumMs', 'buildMaxMs', 'buildP95Ms', 'rasterSumMs', + 'rasterP95Ms', 'totalP95Ms', 'over16', 'rafP95Ms', 'longTasks', 'longTaskMs', + ]; + const medians = Object.fromEntries(metrics.map((metric) => [ + metric, + round(quantile(runs.map((run) => run[metric]), .5)), + ])); + const report = { + build: options.build, + platform: options.platform, + messages: Number(options.messages), + size: '560x1000', + headlessCpuRaster: true, + semanticsEnabled: true, + runs, + medians, + errors: page.errors, + }; + await writeFile(join(options.out, 'report.json'), JSON.stringify(report, null, 2)); + if (page.errors.length) throw new Error('Browser errors: ' + page.errors.join('\n')); + console.log('MEDIANS', JSON.stringify(medians)); +} finally { + await browser.close(); + await server.close(); +}