439 lines
15 KiB
JavaScript
439 lines
15 KiB
JavaScript
// Real-browser evidence for the C2 message-view fixture.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_message_probe.dart \
|
|
// --dart-define=CHAT_MESSAGE_PROBE_STYLE=material \
|
|
// --output build/probe-chat-message-material
|
|
// node tool/chat_message_probe.mjs --root build/probe-chat-message-material \
|
|
// --out DIR --style material
|
|
//
|
|
// The Cupertino host is the same build with CHAT_MESSAGE_PROBE_STYLE=cupertino.
|
|
// Requires Google Chrome and Tesseract: Flutter web paints selectable text on
|
|
// the canvas, so OCR checks the rendered body and its leading edge while the
|
|
// semantics tree checks the controls and the expanded state.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(args.get('root') ?? 'build/probe-chat-message-material');
|
|
const out = resolve(args.get('out') ?? 'build/chat-message-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.mjs': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
const type = mime[extname(filePath)] ?? 'application/octet-stream';
|
|
res.writeHead(200, { 'content-type': type }).end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'c2-message-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 150; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) {
|
|
throw new Error(`Target creation failed: ${JSON.stringify(created)}`);
|
|
}
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 390, height: 844, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
const nodes = () =>
|
|
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
expanded: x.getAttribute('aria-expanded'),
|
|
role: x.getAttribute('role'),
|
|
x: r.x, y: r.y, w: r.width, h: r.height,
|
|
};
|
|
})`);
|
|
const findNode = async (name) => {
|
|
const all = await nodes();
|
|
return all.find(
|
|
(node) => node.label === name || node.text === name,
|
|
);
|
|
};
|
|
const requireNode = async (name) => {
|
|
const node = await findNode(name);
|
|
if (!node) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(`Missing semantics ${JSON.stringify(name)}: ${JSON.stringify(shown)}`);
|
|
}
|
|
return node;
|
|
};
|
|
// A real pointer tap at the node's centre, for controls inside a modal sheet.
|
|
const tap = async (name) => {
|
|
const node = await findNode(name);
|
|
if (!node) throw new Error(`Could not tap ${JSON.stringify(name)}`);
|
|
const x = node.x + node.w / 2;
|
|
const y = node.y + node.h / 2;
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 }, session);
|
|
}
|
|
await delay(450);
|
|
};
|
|
const click = async (name) => {
|
|
const clicked = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(name)};
|
|
const all = [...document.querySelectorAll('flt-semantics')];
|
|
const node = all.find(x => x.getAttribute('aria-label') === wanted)
|
|
|| all.find(x => (x.innerText || '').trim() === wanted);
|
|
if (!node) return false;
|
|
node.click();
|
|
return true;
|
|
})()`);
|
|
if (!clicked) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(`Could not click ${JSON.stringify(name)}: ${JSON.stringify(shown)}`);
|
|
}
|
|
await delay(450);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
const assertOcr = async (path, text) => {
|
|
const read = await ocr(path);
|
|
if (!read.toLowerCase().includes(text.toLowerCase())) {
|
|
throw new Error(
|
|
`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 2000)}`,
|
|
);
|
|
}
|
|
};
|
|
const words = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [
|
|
path,
|
|
'stdout',
|
|
'tsv',
|
|
]);
|
|
return stdout
|
|
.split('\n')
|
|
.slice(1)
|
|
.map((line) => line.split('\t'))
|
|
.filter((cells) => cells.length > 11 && cells[11].trim())
|
|
.map((cells) => ({
|
|
text: cells[11].trim(),
|
|
x: Number(cells[6]),
|
|
y: Number(cells[7]),
|
|
w: Number(cells[8]),
|
|
h: Number(cells[9]),
|
|
}));
|
|
};
|
|
// The leading edge of the first word of a body that starts below `label` and
|
|
// above `next`, against the label's own word. A centred body lands near the
|
|
// middle of a 390-wide phone, so the tolerance is a few glyph bearings.
|
|
const assertBodyEdge = async (path, labelText, nextText) => {
|
|
const all = await words(path);
|
|
const label = all.find((word) => word.text === labelText);
|
|
if (!label) throw new Error(`OCR found no ${labelText} label`);
|
|
const next = nextText
|
|
? all.find((word) => word.text === nextText && word.y > label.y)
|
|
: null;
|
|
const body = all.filter(
|
|
(word) =>
|
|
word.y > label.y + label.h &&
|
|
(!next || word.y < next.y - 4) &&
|
|
word.w < 700,
|
|
);
|
|
if (body.length === 0) {
|
|
throw new Error(`OCR found no ${labelText} body words`);
|
|
}
|
|
const left = Math.min(...body.map((word) => word.x));
|
|
evidence.checks.push({
|
|
check: `${labelText} body starts at its label edge`,
|
|
labelX: label.x,
|
|
bodyX: left,
|
|
delta: left - label.x,
|
|
});
|
|
if (Math.abs(left - label.x) > 12) {
|
|
throw new Error(
|
|
`${labelText} body starts at ${left} but its label at ${label.x}`,
|
|
);
|
|
}
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
const contentType = await evaluate(
|
|
"fetch('/').then(r => r.headers.get('content-type'))",
|
|
);
|
|
if (!String(contentType).toLowerCase().startsWith('text/html')) {
|
|
throw new Error(`Root index is not text/html: ${contentType}`);
|
|
}
|
|
evidence.checks.push({
|
|
check: 'fixture index is served as text/html',
|
|
value: contentType,
|
|
});
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
await waitFor(async () => Boolean(await findNode('Assistant message')), 'semantics');
|
|
|
|
for (const name of [
|
|
'User message',
|
|
'Assistant message',
|
|
'Reasoning, expand for details',
|
|
'1 tool call, expand for details',
|
|
'Unrecognised part: future_part',
|
|
'Ada phone',
|
|
]) {
|
|
await requireNode(name);
|
|
}
|
|
const userGroup = await requireNode('User message');
|
|
const agentGroup = await requireNode('Assistant message');
|
|
if (userGroup.text.trim() === 'User' || agentGroup.text.trim() === 'Assistant') {
|
|
throw new Error('Row semantics repeat the visible role label');
|
|
}
|
|
evidence.checks.push({
|
|
check: 'roles, client, and collapsed disclosures are present once',
|
|
userText: userGroup.text,
|
|
agentText: agentGroup.text,
|
|
});
|
|
|
|
const top = await capture(`${style}-top`);
|
|
for (const text of [
|
|
'Please keep every word of this prompt',
|
|
'plan.md',
|
|
// `report.csv` covers both hosts; Cupertino's small paperclip glyph makes
|
|
// OCR drop the leading `r`, so the suffix is the stable part.
|
|
'eport.csv',
|
|
'checked the transcript and kept the original order.',
|
|
]) {
|
|
await assertOcr(top, text);
|
|
}
|
|
const topText = await ocr(top);
|
|
for (const hidden of ['The user asked for exact', 'command', 'payload']) {
|
|
if (topText.toLowerCase().includes(hidden.toLowerCase())) {
|
|
throw new Error(`Closed disclosure leaked ${JSON.stringify(hidden)}`);
|
|
}
|
|
}
|
|
// Reasoning is a chip beside the tool chip; it opens a sheet.
|
|
await click('Reasoning, expand for details');
|
|
const openReasoning = await capture(`${style}-reasoning-expanded`);
|
|
await assertOcr(openReasoning, 'The user asked for exact text');
|
|
// Dismiss the sheet through its barrier.
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x: 10, y: 10, button: 'left', clickCount: 1 }, session);
|
|
}
|
|
await delay(800);
|
|
|
|
await click('Unrecognised part: future_part');
|
|
const openUnknown = await capture(`${style}-unknown-expanded`);
|
|
for (const text of ['future_part', 'payload', 'kept', 'count']) {
|
|
await assertOcr(openUnknown, text);
|
|
}
|
|
|
|
// As the web: the turn's tool calls are one chip below the answer that
|
|
// opens a sheet; a call there opens to its formatted input.
|
|
await click('1 tool call, expand for details');
|
|
await waitFor(async () => Boolean(await findNode('shell · ls -la')), 'tool sheet');
|
|
// Let the sheet finish its entrance before the row is activated.
|
|
await delay(800);
|
|
await tap('shell · ls -la');
|
|
await delay(600);
|
|
const openTool = await capture(`${style}-tool-expanded`);
|
|
for (const text of ['INPUT', 'command', '-la']) {
|
|
await assertOcr(openTool, text);
|
|
}
|
|
|
|
// Dismiss the tool sheet, then bring the live turn into view.
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x: 10, y: 10, button: 'left', clickCount: 1 }, session);
|
|
}
|
|
await delay(800);
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: 195, y: 420, deltaX: 0, deltaY: 3000 }, session);
|
|
await delay(800);
|
|
const liveName = 'Reasoning · 2 steps, expand for details';
|
|
await requireNode(liveName);
|
|
const liveChip = await capture(`${style}-live-chip`);
|
|
await assertOcr(liveChip, 'Live answer so far');
|
|
await assertOcr(liveChip, 'Reasoning');
|
|
// The open sheet follows the live turn: its newest step keeps growing.
|
|
// A real pointer tap on the chip's own node (the smallest one carrying its
|
|
// name; its row group repeats the text).
|
|
const chipNode = (await nodes())
|
|
.filter((node) => node.label === liveName || node.text === liveName)
|
|
.sort((a, b) => a.w * a.h - b.w * b.h)[0];
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', {
|
|
type,
|
|
x: chipNode.x + chipNode.w / 2,
|
|
y: chipNode.y + chipNode.h / 2,
|
|
button: 'left',
|
|
clickCount: 1,
|
|
}, session);
|
|
}
|
|
await delay(800);
|
|
const tickOf = async (path) => {
|
|
const match = (await ocr(path)).match(/Live tick (\d+)/);
|
|
if (!match) throw new Error(`OCR of ${path} shows no live tick`);
|
|
return Number(match[1]);
|
|
};
|
|
const liveA = await capture(`${style}-live-sheet-a`);
|
|
await assertOcr(liveA, 'Live step one settled');
|
|
const first = await tickOf(liveA);
|
|
await delay(2000);
|
|
const liveB = await capture(`${style}-live-sheet-b`);
|
|
const second = await tickOf(liveB);
|
|
evidence.checks.push({ check: 'open reasoning sheet follows live deltas', first, second });
|
|
if (second <= first) {
|
|
throw new Error(`Open sheet did not follow live deltas: ${first} then ${second}`);
|
|
}
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|