Files
pipelines/app/tool/attention_probe.mjs
alan b60ad6d98f Remove alternate hosts from tooling probes
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
2026-11-11 21:38:30 +00:00

290 lines
17 KiB
JavaScript

// Real-browser acceptance for the reusable Needs you bell and native sheets.
//
// flutter build web -t tool/attention_probe.dart -o build/attention_probe_web
// OPENCODE_ARTIFACT_DIR=<dir> node tool/attention_probe.mjs
//
// The fixture is in-process; it never opens a socket or asks for notification
// permission. The drive checks the accessible bell label, both native hosts at
// 320 logical px / 2x text in dark, the row wording and per-row dismiss names, a
// dismissal that redraws the open sheet, Clear all leaving an honest empty sheet,
// opening a Decision and a Failed row, Escape back to the page, the honest empty
// state, and light at the default text scale. The Material count badge is also
// captured at 390 logical px / 2x, where it must keep its trailing margin. One
// capture is written per state it inspects.
import { createServer } from 'node:http';
import { spawn } from 'node:child_process';
import { once } from 'node:events';
import { existsSync } from 'node:fs';
import { setTimeout as delay } from 'node:timers/promises';
import { join, extname, resolve, sep } from 'node:path';
import { tmpdir } from 'node:os';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
const root = resolve(process.env.PROBE_WEB_ROOT ?? 'build/attention_probe_web');
const artifactDir = process.env.OPENCODE_ARTIFACT_DIR;
if (!artifactDir) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures');
await mkdir(artifactDir, { recursive: true });
const defaultChrome = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
const chromePath = process.env.CHROME_PATH ?? defaultChrome;
if (!existsSync(chromePath)) throw new Error(`Chrome not found at ${chromePath}; set CHROME_PATH`);
const mime = { '.html': 'text/html', '.js': 'text/javascript', '.json': 'application/json', '.wasm': 'application/wasm', '.png': 'image/png', '.ico': 'image/x-icon' };
const server = createServer(async (req, res) => {
try {
const path = resolve(root, '.' + decodeURIComponent(new URL(req.url, 'http://localhost').pathname));
if (path !== root && !path.startsWith(root + sep)) return res.writeHead(403).end();
const filePath = path === root || path.endsWith('/') ? join(path, 'index.html') : path;
const file = await readFile(filePath);
res.writeHead(200, { 'content-type': mime[extname(filePath)] ?? 'application/octet-stream' }).end(file);
} catch { res.writeHead(404).end('not found'); }
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const port = server.address().port;
const profile = await mkdtemp(join(tmpdir(), 'attention-chrome-'));
const chrome = spawn(chromePath, [
'--headless=new', '--no-first-run', '--no-default-browser-check', '--disable-gpu',
'--remote-debugging-port=0', `--user-data-dir=${profile}`, 'about:blank',
], { stdio: ['ignore', 'ignore', 'pipe'] });
let stderr = '';
chrome.stderr.on('data', (data) => { stderr += data; });
const waitFor = async (check) => {
for (let i = 0; i < 120; i++) { const value = await check(); if (value) return value; await delay(100); }
throw new Error(`Timed out waiting for Chrome DevTools\n${stderr.slice(-1200)}`);
};
let ws;
try {
const endpoint = await waitFor(() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1]);
ws = new WebSocket(endpoint);
await once(ws, 'open');
let id = 0;
const pending = new Map();
ws.addEventListener('message', (event) => {
const message = JSON.parse(event.data);
if (message.id && pending.has(message.id)) { pending.get(message.id)(message); pending.delete(message.id); }
});
const send = (method, params = {}, sessionId) => new Promise((resolveCall) => {
const callId = ++id; pending.set(callId, resolveCall);
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
});
const target = (await send('Target.createTarget', { url: 'about:blank' })).result.targetId;
const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId;
await send('Page.enable', {}, session);
await send('Runtime.enable', {}, session);
const evaluate = async (expression) => {
const response = await send('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, session);
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
return response.result?.result?.value;
};
const semantics = () => evaluate("[...document.querySelectorAll('flt-semantics')].map(x=>{const r=x.getBoundingClientRect();return{label:(x.getAttribute('aria-label')||x.innerText||'').trim(), role:x.getAttribute('role'), x:r.x, y:r.y, w:r.width, h:r.height}})");
const labels = async () => (await semantics()).map((item) => item.label).filter(Boolean);
const requireLabels = (all, needles, state) => {
for (const needle of needles) {
if (!all.some((label) => label.includes(needle))) {
throw new Error(`Missing ${JSON.stringify(needle)} in ${state}: ${JSON.stringify(all)}`);
}
}
};
const clickLabel = async (needle, exact = false, minY = 0) => {
const point = await evaluate(`(() => { const all=[...document.querySelectorAll('flt-semantics')].filter(x => x.getBoundingClientRect().top >= ${minY}); const match=x => { const l=(x.getAttribute('aria-label')||x.innerText||'').trim(); return ${exact ? 'l === ' : 'l.includes('}${JSON.stringify(needle)}${exact ? '' : ')'}; }; const e=all.find(x => x.getAttribute('role') === 'button' && match(x)) || all.find(match); if (!e) return null; const r=e.getBoundingClientRect(); return {x:r.x+r.width/2,y:r.y+r.height/2,w:r.width,h:r.height}; })()`);
if (!point) return false;
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
return true;
};
// Row-open and Clear-all controls can sit below the fold at 2x text. Find the
// node, wheel the sheet until its centre is on screen, then click it. Row
// labels are matched without their `Dismiss ...` sibling, whose label also
// contains the row text.
const clickScrolled = async (needle, { exact = false, minY = 0 } = {}) => {
for (let attempt = 0; attempt < 12; attempt++) {
const point = await evaluate(`(() => { const all=[...document.querySelectorAll('flt-semantics')].filter(x => x.getBoundingClientRect().top >= ${minY}); const match=x => { const l=(x.getAttribute('aria-label')||x.innerText||'').trim(); if (l.startsWith('Dismiss ')) return false; return ${exact ? 'l === ' : 'l.includes('}${JSON.stringify(needle)}${exact ? '' : ')'}; }; const e=all.find(x => x.getAttribute('role') === 'button' && match(x)) || all.find(match); if (!e) return null; const r=e.getBoundingClientRect(); return {x:r.x+r.width/2,y:r.y+r.height/2,w:r.width,h:r.height}; })()`);
if (point && point.y > 4 && point.y < viewport.height - 4) {
await send('Input.dispatchMouseEvent', { type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
await send('Input.dispatchMouseEvent', { type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 }, session);
return true;
}
const deltaY = point && point.y < 0 ? -240 : 240;
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: viewport.width / 2, y: viewport.height - 80, deltaX: 0, deltaY }, session);
await delay(500);
}
return false;
};
const pressEscape = async () => {
const key = { key: 'Escape', code: 'Escape', windowsVirtualKeyCode: 27, nativeVirtualKeyCode: 27 };
await send('Input.dispatchKeyEvent', { type: 'keyDown', ...key }, session);
await send('Input.dispatchKeyEvent', { type: 'keyUp', ...key }, session);
};
let viewport = { width: 320, height: 640 };
const navigate = async ({ mode = '', theme = 'dark', scale = 2, width = 320, height = 640 } = {}) => {
viewport = { width, height };
await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-color-scheme', value: theme }] }, session);
const query = new URLSearchParams({ theme, scale: String(scale) });
if (mode) query.set('mode', mode);
await send('Page.navigate', { url: `http://127.0.0.1:${port}/?${query}` }, session);
await delay(3000);
await send('Emulation.setDeviceMetricsOverride', { width, height, deviceScaleFactor: 2, mobile: true }, session);
await delay(1200);
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
await delay(1200);
};
const capture = async (name) => {
const data = (await send('Page.captureScreenshot', { format: 'png', captureBeyondViewport: false }, session)).result.data;
await writeFile(join(artifactDir, name), Buffer.from(data, 'base64'));
};
const captures = [];
const record = async (name) => {
await capture(name);
captures.push(name);
};
{
const host = 'material';
// The hardest case: narrow, scaled text in dark.
await navigate();
requireLabels(await labels(), ['Notifications, 2 needing attention'], `${host} bell`);
await record(`attention-${host}-dark-320-2x-bell.png`);
if (!(await clickLabel('Notifications, 2 needing attention', true))) throw new Error(`${host}: bell not clickable`);
await delay(1000);
const sheet = await semantics();
requireLabels(sheet.map((item) => item.label), [
'Needs you', 'Clear all', 'Approve the release candidate',
'release · Waiting for your decision',
'Nightly audit with a label long enough to wrap at 320 logical px',
'nightly-audit · Stage 3 stopped',
], `${host} sheet`);
// Each row's destructive control is named after its row, never bare
// `Dismiss`. The Material drag handle legitimately carries the native
// `Dismiss` (modalBarrierDismissLabel), so only row buttons are collected.
const dismissLabels = sheet
.filter((item) => item.role === 'button' && item.label.startsWith('Dismiss '))
.map((item) => item.label);
requireLabels(dismissLabels, [
'Dismiss Approve the release candidate',
'Dismiss Nightly audit with a label long enough to wrap at 320 logical px',
], `${host} dismiss names`);
if (new Set(dismissLabels).size !== 2) {
throw new Error(`${host}: expected two distinct named row dismiss actions: ${JSON.stringify(dismissLabels)}`);
}
await record(`attention-${host}-dark-320-2x-sheet.png`);
// Dismissing redraws the open sheet: the row is gone, the sheet is still there.
if (!(await clickLabel('Dismiss Approve the release candidate', true, 100))) throw new Error(`${host}: dismiss not clickable`);
await delay(1000);
const afterDismiss = await labels();
if (afterDismiss.some((label) => label.includes('Approve the release candidate'))) {
throw new Error(`${host}: dismissed row is still shown in the open sheet`);
}
requireLabels(afterDismiss, [
'Needs you',
'Dismiss Nightly audit with a label long enough to wrap at 320 logical px',
], `${host} after dismiss`);
await record(`attention-${host}-dark-320-2x-after-dismiss.png`);
// Escape closes the sheet natively and the page reflects the one dismissal.
await pressEscape();
await delay(900);
requireLabels(await labels(), [
'Dismissed 7:Decision',
'1 needing attention now',
'Notifications, 1 needing attention',
], `${host} closed sheet`);
// Light and the default text scale on a normal phone width.
await navigate({ theme: 'light', scale: 1, width: 390, height: 844 });
requireLabels(await labels(), ['Notifications, 2 needing attention'], `${host} light bell`);
await record(`attention-${host}-light-390-1x-bell.png`);
if (!(await clickLabel('Notifications, 2 needing attention', true))) throw new Error(`${host}: light bell not clickable`);
await delay(900);
requireLabels(await labels(), [
'Needs you', 'Clear all', 'Dismiss Approve the release candidate',
'Dismiss Nightly audit with a label long enough to wrap at 320 logical px',
], `${host} light sheet`);
await record(`attention-${host}-light-390-1x-sheet.png`);
await navigate({ mode: 'empty' });
const emptyBell = await labels();
requireLabels(emptyBell, ['Notifications'], `${host} empty bell`);
if (emptyBell.some((label) => label.startsWith('Notifications,'))) throw new Error(`${host}: empty bell announced a count`);
if (!(await clickLabel('Notifications', true))) throw new Error(`${host}: empty bell not clickable`);
await delay(900);
const emptySheet = await semantics();
const emptyLabels = emptySheet.map((item) => item.label);
requireLabels(emptyLabels, ['Needs you', 'Nothing needs you right now.'], `${host} empty sheet`);
if (emptyLabels.some((label) => label.includes('Clear all'))) throw new Error(`${host}: empty sheet offered Clear all`);
if (emptySheet.some((item) => item.role === 'button' && item.label.startsWith('Dismiss '))) throw new Error(`${host}: empty sheet offered Dismiss`);
await record(`attention-${host}-dark-320-2x-empty.png`);
// Clear all leaves the sheet open and honest: the empty copy, no Clear all
// and no row dismiss. Closing then shows a plain bell and no remaining work.
await navigate();
if (!(await clickLabel('Notifications, 2 needing attention', true))) throw new Error(`${host}: bell not clickable for Clear all`);
await delay(1000);
if (!(await clickScrolled('Clear all', { exact: true }))) throw new Error(`${host}: Clear all not clickable`);
await delay(1000);
const clearedSheet = await semantics();
const clearedLabels = clearedSheet.map((item) => item.label);
requireLabels(clearedLabels, ['Needs you', 'Nothing needs you right now.'], `${host} cleared sheet`);
if (clearedLabels.some((label) => label.includes('Clear all'))) throw new Error(`${host}: cleared sheet still offered Clear all`);
if (clearedSheet.some((item) => item.role === 'button' && item.label.startsWith('Dismiss '))) throw new Error(`${host}: cleared sheet offered Dismiss`);
await record(`attention-${host}-dark-320-2x-cleared.png`);
await pressEscape();
await delay(900);
const clearedPage = await labels();
requireLabels(clearedPage, ['Cleared 2 of the open snapshot', '0 needing attention now', 'Notifications'], `${host} cleared page`);
if (clearedPage.some((label) => label.startsWith('Notifications,'))) throw new Error(`${host}: cleared page announced a count`);
// Opening a Decision row closes the sheet and leaves the identity in place:
// only answering the decision clears it.
await navigate();
if (!(await clickLabel('Notifications, 2 needing attention', true))) throw new Error(`${host}: bell not clickable for Decision row`);
await delay(1000);
if (!(await clickScrolled('Approve the release candidate'))) throw new Error(`${host}: Decision row not clickable`);
await delay(1000);
const decisionPage = await labels();
requireLabels(decisionPage, [
'Opened 7:Decision', '2 needing attention now',
'Notifications, 2 needing attention',
], `${host} Decision row open`);
if (decisionPage.some((label) => label.includes('Clear all'))) throw new Error(`${host}: Decision row left the sheet open`);
await record(`attention-${host}-dark-320-2x-row-open-decision.png`);
// Opening a Failed row dismisses its identity before opening it, so the
// bell no longer asks for a run the operator has already looked at.
await navigate();
if (!(await clickLabel('Notifications, 2 needing attention', true))) throw new Error(`${host}: bell not clickable for Failed row`);
await delay(1000);
if (!(await clickScrolled('Nightly audit with a label long enough'))) throw new Error(`${host}: Failed row not clickable`);
await delay(1000);
const failedPage = await labels();
requireLabels(failedPage, [
'Opened 8:Failed', '1 needing attention now',
'Notifications, 1 needing attention',
], `${host} Failed row open`);
await record(`attention-${host}-dark-320-2x-row-open-failed.png`);
}
// The Material count badge keeps its trailing margin at a normal phone width
// and doubled text, not only at the narrow 320 px fixture.
await navigate('material', { width: 390, height: 844 });
requireLabels(await labels(), ['Notifications, 2 needing attention'], 'material 390 bell');
await record('attention-material-dark-390-2x-bell.png');
if (!(await clickLabel('Notifications, 2 needing attention', true))) throw new Error('material 390: bell not clickable');
await delay(1000);
if (!(await clickLabel('Dismiss Approve the release candidate', true, 100))) throw new Error('material 390: dismiss not clickable');
await delay(1000);
requireLabels(await labels(), [
'Needs you',
'Dismiss Nightly audit with a label long enough to wrap at 320 logical px',
], 'material 390 after dismiss');
await record('attention-material-dark-390-2x-after-dismiss.png');
console.log(JSON.stringify({ captures }, null, 2));
} finally {
ws?.close();
chrome.kill('SIGTERM');
await Promise.race([once(chrome, 'exit'), delay(3000)]);
server.close();
await rm(profile, { recursive: true, force: true });
}