Files
pipelines/app/tool/chat_back_latency_probe.mjs
T

123 lines
8.4 KiB
JavaScript

// Measures normal-app Back against a populated loopback chat, never operator data.
// PROBE_PLATFORM=android CPU_RATE=4 CHAT_PAIRS=100 node tool/chat_back_latency_probe.mjs <web-build> <out>
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, writeFile } from 'node:fs/promises';
import { once } from 'node:events';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { performance } from 'node:perf_hooks';
import { startFixture } from './runs_probe_server.mjs';
import { launchChrome, waitForPort, waitForPage, connect, sleep } from './pipelines_probe_cdp_lib.mjs';
import { axNodes, waitFor, capture } from './pipelines_probe_cdp_input.mjs';
const pairs = Number(process.env.CHAT_PAIRS ?? 100);
const width = Number(process.env.PROBE_WIDTH ?? 390);
const text = '# Recorded answer\n\n' +
Array.from({ length: 12 }, (_, i) => `Paragraph ${i}: This is recorded fixture content with **formatting**, an [inert link](https://example.test), and enough text to wrap naturally on a phone.\n`).join('\n') +
'\n```dart\n' + Array.from({ length: 25 }, (_, i) => `final value${i} = ${i};`).join('\n') + '\n```\n';
const turns = Array.from({ length: pairs }, (_, i) => [
{ id: `u${i}`, role: 'user', parts: [{ type: 'text', text: `Recorded question ${i}` }] },
{ id: `a${i}`, role: 'assistant', parts: [{ id: `p${i}`, type: 'text', text: process.env.CHAT_VARIANCE === '1' ? text.repeat(1 + i % 4) : text }] },
]).flat();
const folded = process.env.CHAT_FOLDED === '1';
const messages = folded ? [turns[0], ...turns.filter(m => m.role === 'assistant')] : turns;
const output = resolve(process.argv[3] ?? 'build/chat-back-latency');
await mkdir(output, { recursive: true });
const fixture = await startFixture({ root: resolve(process.argv[2]), chatMessages: messages });
const profile = await mkdtemp(join(tmpdir(), 'chat-back-latency-'));
const chrome = launchChrome('/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', profile, 'about:blank');
const log = { pairs, width, folded, roundTrip: process.env.SCROLL_ROUNDTRIP === '1', fromList: process.env.OPEN_FROM_LIST === '1', streaming: process.env.CHAT_STREAM === '1', cpuRate: Number(process.env.CPU_RATE ?? 1), host: process.env.PROBE_PLATFORM ?? 'cupertino', checks: [], captures: [] };
let c;
let streamTimer;
const requests = new Set();
try {
c = await connect((await waitForPage(await waitForPort(join(profile, 'DevToolsActivePort')))).webSocketDebuggerUrl);
await c.send('Page.enable');
if (log.host === 'android') await c.send('Emulation.setUserAgentOverride', {
userAgent: 'Mozilla/5.0 (Linux; Android 16; BrowserFixture) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Mobile Safari/537.36', platform: 'Linux armv8l',
});
if (log.host === 'material') await c.send('Emulation.setUserAgentOverride', {
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/155.0.0.0 Safari/537.36', platform: 'Win32',
});
await c.send('Emulation.setDeviceMetricsOverride', { width, height: 844, deviceScaleFactor: 1, mobile: true });
await c.send('Emulation.setCPUThrottlingRate', { rate: log.cpuRate });
const opened = performance.now();
await c.send('Page.navigate', { url: fixture.url + (log.fromList ? '/chat' : '/chat/ses_probechat') });
assert.ok(await waitFor(() => c.evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"), 30000));
// Mobile semantics activation requires a real pointer at the placeholder's
// centre; element.click() at (0,0) does not enable Android web semantics.
const rect = await c.evaluate("(() => { const r = document.querySelector('flt-semantics-placeholder').getBoundingClientRect(); return {x:(r.left+r.right)/2,y:(r.top+r.bottom)/2}; })()");
await c.send('Input.dispatchMouseEvent', { type: 'mousePressed', x: rect.x, y: rect.y, button: 'left', clickCount: 1 });
await c.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: rect.x, y: rect.y, button: 'left', clickCount: 1 });
if (log.fromList) {
const row = await waitFor(async () => (await axNodes(c)).find(n => n.name?.value?.startsWith('Probe conversation') && n.backendDOMNodeId && ['group', 'button'].includes(n.role?.value)), 30000);
assert.ok(row, 'chat list row');
const { model } = await c.send('DOM.getBoxModel', { backendNodeId: row.backendDOMNodeId });
const [x1, y1, , , x3, y3] = model.content;
await c.send('Input.dispatchMouseEvent', { type: 'mousePressed', x: (x1+x3)/2, y: (y1+y3)/2, button: 'left', clickCount: 1 });
await c.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: (x1+x3)/2, y: (y1+y3)/2, button: 'left', clickCount: 1 });
}
assert.ok(await waitFor(async () => (await axNodes(c)).some(n => n.name?.value === 'Back'), 45000), 'chat Back ready');
await sleep(1800);
await c.evaluate('new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))');
log.openAndSettleMs = performance.now() - opened;
await capture(c, output, log, 'chat-before-back');
if (log.roundTrip) {
await c.send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: width - 10, y: 400, deltaX: 0, deltaY: -1000000 });
await sleep(900);
await capture(c, output, log, 'chat-scrolled-top');
await c.send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: width - 10, y: 400, deltaX: 0, deltaY: 1000000 });
await sleep(Number(process.env.SCROLL_SETTLE_MS ?? 300));
await capture(c, output, log, 'chat-scrolled-bottom');
}
if (log.streaming) {
streamTimer = setInterval(() => {
const request = fetch(fixture.url + '/fixture/chat-event', { method: 'POST', body: JSON.stringify({
kind: 'text', message_id: `a${pairs-1}`, part: { type: 'text', id: `p${pairs-1}` }, delta: ' live',
}) });
requests.add(request);
request.finally(() => requests.delete(request));
}, 100);
await sleep(1200);
}
const backNodes = (await axNodes(c)).filter(n => /^Back(?:\s|$)/.test(n.name?.value ?? ''));
log.backControlNodes = backNodes.map(n => ({name:n.name?.value, role:n.role?.value, backend:n.backendDOMNodeId}));
const back = backNodes.find(n => n.backendDOMNodeId && n.role?.value === 'button') ?? backNodes.find(n => n.backendDOMNodeId);
assert.ok(back, 'native Back control');
const { model } = await c.send('DOM.getBoxModel', { backendNodeId: back.backendDOMNodeId });
const [x1, y1, , , x3, y3] = model.content;
await c.evaluate(`(() => {
window.__back = {started: performance.now(), frames: [], longTasks: []};
new PerformanceObserver(list => { for (const e of list.getEntries()) window.__back.longTasks.push({start:e.startTime,duration:e.duration}); }).observe({entryTypes:['longtask']});
const frame = t => { window.__back.frames.push(t); if (location.pathname === '/chat') window.__back.settled=t; else requestAnimationFrame(frame); };
requestAnimationFrame(frame);
})()`);
const started = performance.now();
await c.send('Input.dispatchMouseEvent', { type: 'mousePressed', x: (x1+x3)/2, y: (y1+y3)/2, button: 'left', clickCount: 1 });
await c.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: (x1+x3)/2, y: (y1+y3)/2, button: 'left', clickCount: 1 });
assert.ok(await waitFor(() => c.evaluate('Boolean(window.__back.settled)'), 30000), 'Back returns to list without network response');
const metrics = await c.evaluate('window.__back');
log.backWallMs = performance.now() - started;
log.routeSettledMs = metrics.settled - metrics.started;
log.maxFrameGapMs = Math.max(0, ...metrics.frames.slice(1).map((t, i) => t - metrics.frames[i]));
log.longTasks = metrics.longTasks;
if (process.env.MAX_BACK_MS) assert.ok(log.routeSettledMs <= Number(process.env.MAX_BACK_MS), `Back exceeded ${process.env.MAX_BACK_MS} ms: ${log.routeSettledMs}`);
if (process.env.MAX_FRAME_GAP_MS) assert.ok(log.maxFrameGapMs <= Number(process.env.MAX_FRAME_GAP_MS), `Frame gap exceeded ${process.env.MAX_FRAME_GAP_MS} ms: ${log.maxFrameGapMs}`);
log.checks.push('pop restored chat list');
await sleep(150);
await capture(c, output, log, 'chat-after-back');
console.log(JSON.stringify(log, null, 2));
} catch (error) {
if (c) {
await capture(c, output, log, 'failure');
await writeFile(join(output, 'failure-ax.json'), JSON.stringify(await axNodes(c), null, 2));
}
throw error;
} finally {
clearInterval(streamTimer);
await Promise.allSettled([...requests]);
await writeFile(join(output, 'result.json'), JSON.stringify(log, null, 2));
c?.close(); const exited = once(chrome, 'exit'); chrome.kill(); await exited;
await fixture.close();
}