357 lines
12 KiB
JavaScript
357 lines
12 KiB
JavaScript
// Real-browser evidence for the chat approval and question cards.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_approvals_probe.dart \
|
|
// --dart-define=CHAT_APPROVALS_PROBE_STYLE=material \
|
|
// --output build/probe-chat-approvals-material
|
|
// node tool/chat_approvals_probe.mjs \
|
|
// --root build/probe-chat-approvals-material --out DIR --style material
|
|
//
|
|
// The same fixture is built with CHAT_APPROVALS_PROBE_STYLE=cupertino. The
|
|
// script drives the real widget/controller through Chrome DevTools semantics,
|
|
// approves one permission and answers one question, and asserts the recorded
|
|
// answered states plus saved captures.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(args.get('root') ?? 'build/probe-chat-approvals-material');
|
|
const out = resolve(args.get('out') ?? 'build/chat-approvals-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, {
|
|
'content-type': mime[extname(filePath)] ?? 'application/octet-stream',
|
|
});
|
|
res.end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'chat-approvals-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 240; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) throw new Error('Target creation failed');
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 320, height: 1500, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
const nodes = () =>
|
|
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
x: r.x, y: r.y, w: r.width, h: r.height,
|
|
};
|
|
})`);
|
|
const hasText = async (fragment) => {
|
|
const all = await nodes();
|
|
return all.some((node) =>
|
|
`${node.label ?? ''} ${node.text ?? ''}`
|
|
.toLowerCase()
|
|
.includes(fragment.toLowerCase()),
|
|
);
|
|
};
|
|
const click = async (name) => {
|
|
const point = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(name)};
|
|
const matches = [...document.querySelectorAll('flt-semantics')]
|
|
.map(x => {
|
|
const label = x.getAttribute('aria-label') || '';
|
|
const text = (x.innerText || '').trim();
|
|
const r = x.getBoundingClientRect();
|
|
const labelHit = label.includes(wanted) ? 0 : 1;
|
|
return { x, label, text, labelHit, area: r.width * r.height };
|
|
})
|
|
.filter(x => x.labelHit === 0 || x.text.includes(wanted))
|
|
.sort((a, b) => a.labelHit - b.labelHit || a.area - b.area);
|
|
const node = matches[0]?.x;
|
|
if (!node) return null;
|
|
const r = node.getBoundingClientRect();
|
|
return { x: r.x + r.width / 2, y: r.y + r.height / 2 };
|
|
})()`);
|
|
if (!point) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send(
|
|
'Input.dispatchMouseEvent',
|
|
{ type, x: point.x, y: point.y, button: 'left', clickCount: 1 },
|
|
session,
|
|
);
|
|
}
|
|
await delay(700);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${style}-${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
// Flutter web paints static text on the canvas, so semantics exposes the
|
|
// controls but not the card's prose. OCR reads the real rendered pixels.
|
|
const screenOCR = async () => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, 'screen-ocr.png');
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
const { stdout } = await promisify(execFile)('tesseract', [file, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
// OCR sometimes merges words across a line ("A demo question" reads as
|
|
// "Ademo question"), so comparisons ignore whitespace entirely.
|
|
const flat = (value) => value.toLowerCase().replace(/\s+/g, '');
|
|
const waitForOCR = async (text) => {
|
|
for (let i = 0; i < 120; i++) {
|
|
const read = await screenOCR();
|
|
if (flat(read).includes(flat(text))) return read;
|
|
await delay(200);
|
|
}
|
|
throw new Error(`Timed out waiting for OCR ${JSON.stringify(text)}`);
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
await waitForOCR('Approve bash').catch(async (error) => {
|
|
await capture('failure');
|
|
console.log(JSON.stringify(await nodes(), null, 2));
|
|
throw error;
|
|
});
|
|
// The two cards need not fit one screen; scroll the conversation like an
|
|
// operator until the question is rendered, then read the whole card view.
|
|
let pendingRead = await screenOCR();
|
|
for (let attempt = 0; attempt < 12 && !flat(pendingRead).includes(flat('A demo question')); attempt++) {
|
|
await send('Input.dispatchMouseEvent', {
|
|
type: 'mouseWheel', x: 195, y: 420, deltaX: 0, deltaY: 320,
|
|
}, session);
|
|
await delay(350);
|
|
pendingRead = await screenOCR();
|
|
}
|
|
if (!flat(pendingRead).includes(flat('A demo question'))) {
|
|
pendingRead = await waitForOCR('A demo question');
|
|
}
|
|
|
|
// Prose comes from the painted pixels, controls from the semantics tree; a
|
|
// white-on-tint filled button is reliable in semantics even when OCR skips it.
|
|
const pendingSemantics = flat(
|
|
(await nodes()).map((entry) => `${entry.label ?? ''} ${entry.text ?? ''}`).join(' '),
|
|
);
|
|
for (const text of [
|
|
// The web card headline is the recorded action; it has no separate
|
|
// "Permission request" heading, so none is invented here either.
|
|
'Approve bash',
|
|
'Waiting for your response',
|
|
'RISK REVIEW',
|
|
'This deletes the build output.',
|
|
'Allow once',
|
|
'Always allow',
|
|
'Reject',
|
|
'A demo question',
|
|
'The agent is waiting for your answer.',
|
|
'Colour',
|
|
'Red',
|
|
'Blue',
|
|
]) {
|
|
if (!flat(pendingRead).includes(flat(text)) && !pendingSemantics.includes(flat(text))) {
|
|
throw new Error(`Pending cards lack ${JSON.stringify(text)}`);
|
|
}
|
|
}
|
|
const pendingCapture = await capture('cards');
|
|
evidence.checks.push({
|
|
check: 'pending approval and question render with their recorded text',
|
|
style,
|
|
capture: pendingCapture,
|
|
labels: (await nodes()).map((node) => node.label).filter(Boolean),
|
|
});
|
|
|
|
await click('Allow once');
|
|
const approvedRead = await waitForOCR('Allowed once');
|
|
for (const text of ['Recorded by the probe fixture']) {
|
|
if (!approvedRead.toLowerCase().includes(text.toLowerCase())) {
|
|
throw new Error(`Answered approval lacks ${JSON.stringify(text)}`);
|
|
}
|
|
}
|
|
await capture('approval-answered');
|
|
evidence.checks.push({
|
|
check: 'the approval reply was sent and the recorded resolution cleared controls',
|
|
style,
|
|
allowOnceStillVisible: await hasText('Allow once'),
|
|
});
|
|
|
|
await click('Red');
|
|
await click('Send answer');
|
|
const answeredRead = await waitForOCR('Answer recorded.');
|
|
if (!answeredRead.toLowerCase().includes('red')) {
|
|
throw new Error('Answered question lost the recorded Red answer');
|
|
}
|
|
await capture('question-answered');
|
|
evidence.checks.push({
|
|
check: 'the question answer was sent and the card is read-only',
|
|
style,
|
|
sendAnswerStillVisible: await hasText('Send answer'),
|
|
});
|
|
|
|
if (evidence.checks.at(-1).sendAnswerStillVisible) {
|
|
throw new Error('Send answer stayed visible after the question resolved');
|
|
}
|
|
if (evidence.checks.at(-2).allowOnceStillVisible) {
|
|
throw new Error('Allow once stayed visible after the approval resolved');
|
|
}
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|