b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
365 lines
12 KiB
JavaScript
365 lines
12 KiB
JavaScript
// Real-browser evidence for the chat approval and question cards.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_approvals_probe.dart \
|
|
// --dart-define=CHAT_APPROVALS_PROBE_STYLE=material \
|
|
// --output build/probe-chat-approvals-material
|
|
// node tool/chat_approvals_probe.mjs \
|
|
// --root build/probe-chat-approvals-material --out DIR --style material
|
|
//
|
|
// The same fixture is built with CHAT_APPROVALS_PROBE_STYLE=material. The
|
|
// script drives the real widget/controller through Chrome DevTools semantics,
|
|
// approves one permission and answers one question, and asserts the recorded
|
|
// answered states plus saved captures.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(args.get('root') ?? 'build/probe-chat-approvals-material');
|
|
const out = resolve(args.get('out') ?? 'build/chat-approvals-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, {
|
|
'content-type': mime[extname(filePath)] ?? 'application/octet-stream',
|
|
});
|
|
res.end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'chat-approvals-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 240; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) throw new Error('Target creation failed');
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 320, height: 1500, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
const nodes = () =>
|
|
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
x: r.x, y: r.y, w: r.width, h: r.height,
|
|
};
|
|
})`);
|
|
const hasText = async (fragment) => {
|
|
const all = await nodes();
|
|
return all.some((node) =>
|
|
`${node.label ?? ''} ${node.text ?? ''}`
|
|
.toLowerCase()
|
|
.includes(fragment.toLowerCase()),
|
|
);
|
|
};
|
|
const click = async (name) => {
|
|
const point = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(name)};
|
|
const matches = [...document.querySelectorAll('flt-semantics')]
|
|
.map(x => {
|
|
const label = x.getAttribute('aria-label') || '';
|
|
const text = (x.innerText || '').trim();
|
|
const r = x.getBoundingClientRect();
|
|
const labelHit = label.includes(wanted) ? 0 : 1;
|
|
return { x, label, text, labelHit, area: r.width * r.height };
|
|
})
|
|
.filter(x => x.labelHit === 0 || x.text.includes(wanted))
|
|
.sort((a, b) => a.labelHit - b.labelHit || a.area - b.area);
|
|
const node = matches[0]?.x;
|
|
if (!node) return null;
|
|
const r = node.getBoundingClientRect();
|
|
return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
|
|
})()`);
|
|
if (!point) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send(
|
|
'Input.dispatchMouseEvent',
|
|
{ type, x: point.x, y: point.y, button: 'left', clickCount: 1 },
|
|
session,
|
|
);
|
|
}
|
|
await delay(700);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${style}-${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
// Flutter web paints static text on the canvas, so semantics exposes the
|
|
// controls but not the card's prose. OCR reads the real rendered pixels.
|
|
const screenOCR = async () => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, 'screen-ocr.png');
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
const { stdout } = await promisify(execFile)('tesseract', [file, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
// OCR sometimes merges words across a line ("A demo question" reads as
|
|
// "Ademo question"), so comparisons ignore whitespace entirely.
|
|
const flat = (value) => value.toLowerCase().replace(/\s+/g, '');
|
|
const waitForOCR = async (text) => {
|
|
for (let i = 0; i < 120; i++) {
|
|
const read = await screenOCR();
|
|
if (flat(read).includes(flat(text))) return read;
|
|
await delay(200);
|
|
}
|
|
throw new Error(`Timed out waiting for OCR ${JSON.stringify(text)}`);
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
await waitForOCR('Approve bash').catch(async (error) => {
|
|
await capture('failure');
|
|
console.log(JSON.stringify(await nodes(), null, 2));
|
|
throw error;
|
|
});
|
|
// The two cards need not fit one screen; scroll the conversation like an
|
|
// operator until the question is rendered, then read the whole card view.
|
|
let pendingRead = await screenOCR();
|
|
for (let attempt = 0; attempt < 12 && !flat(pendingRead).includes(flat('A demo question')); attempt++) {
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mouseWheel', x: 195, y: 420, deltaX: 0, deltaY: 320,
|
|
}, session);
|
|
await delay(350);
|
|
pendingRead = await screenOCR();
|
|
}
|
|
if (!flat(pendingRead).includes(flat('A demo question'))) {
|
|
pendingRead = await waitForOCR('A demo question');
|
|
}
|
|
|
|
// Prose comes from the painted pixels, controls from the semantics tree; a
|
|
// white-on-tint filled button is reliable in semantics even when OCR skips it.
|
|
const pendingSemantics = flat(
|
|
(await nodes()).map((entry) => `${entry.label ?? ''} ${entry.text ?? ''}`).join(' '),
|
|
);
|
|
for (const text of [
|
|
// The web card headline is the recorded action; it has no separate
|
|
// "Permission request" heading, so none is invented here either.
|
|
'Approve bash',
|
|
'Waiting for your response',
|
|
'RISK REVIEW',
|
|
'This deletes the build output.',
|
|
'Allow once',
|
|
'Always allow',
|
|
'Reject',
|
|
'A demo question',
|
|
'The agent is waiting for your answer.',
|
|
'Colour',
|
|
'Red',
|
|
'Blue',
|
|
]) {
|
|
if (!flat(pendingRead).includes(flat(text)) && !pendingSemantics.includes(flat(text))) {
|
|
throw new Error(`Pending cards lack ${JSON.stringify(text)}`);
|
|
}
|
|
}
|
|
const pendingCapture = await capture('cards');
|
|
evidence.checks.push({
|
|
check: 'pending approval and question render with their recorded text',
|
|
style,
|
|
capture: pendingCapture,
|
|
labels: (await nodes()).map((node) => node.label).filter(Boolean),
|
|
});
|
|
|
|
await click('Allow once');
|
|
// The answered request folds into its turn chips, like tool calls.
|
|
await waitForOCR('1 permission');
|
|
await capture('approval-answered');
|
|
evidence.checks.push({
|
|
check: 'the approval reply was sent and the request folded into a permissions chip',
|
|
style,
|
|
allowOnceStillVisible: await hasText('Allow once'),
|
|
});
|
|
|
|
await click('Red');
|
|
await click('Send answer');
|
|
const answeredRead = await waitForOCR('Answer recorded.');
|
|
if (!answeredRead.toLowerCase().includes('red')) {
|
|
throw new Error('Answered question lost the recorded Red answer');
|
|
}
|
|
await capture('question-answered');
|
|
evidence.checks.push({
|
|
check: 'the question answer was sent and the card is read-only',
|
|
style,
|
|
sendAnswerStillVisible: await hasText('Send answer'),
|
|
});
|
|
|
|
await click('1 permission request');
|
|
await click('bash · Allowed once');
|
|
const sheetRead = await waitForOCR('Recorded by the probe fixture');
|
|
if (!flat(sheetRead).includes(flat('rm -rf build'))) {
|
|
throw new Error('The permissions sheet lost the recorded request');
|
|
}
|
|
await capture('permissions-sheet');
|
|
evidence.checks.push({
|
|
check: 'the permissions chip opens the recorded request and its outcome',
|
|
style,
|
|
});
|
|
|
|
if (evidence.checks.at(-2).sendAnswerStillVisible) {
|
|
throw new Error('Send answer stayed visible after the question resolved');
|
|
}
|
|
if (evidence.checks.at(-3).allowOnceStillVisible) {
|
|
throw new Error('Allow once stayed visible after the approval resolved');
|
|
}
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|