b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
439 lines
15 KiB
JavaScript
439 lines
15 KiB
JavaScript
// Real-browser evidence for the C2 message-view fixture.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_message_probe.dart \
|
|
// --dart-define=CHAT_MESSAGE_PROBE_STYLE=material \
|
|
// --output build/probe-chat-message-material
|
|
// node tool/chat_message_probe.mjs --root build/probe-chat-message-material \
|
|
// --out DIR --style material
|
|
//
|
|
// The Material host is the same build with CHAT_MESSAGE_PROBE_STYLE=material.
|
|
// Requires Google Chrome and Tesseract: Flutter web paints selectable text on
|
|
// the canvas, so OCR checks the rendered body and its leading edge while the
|
|
// semantics tree checks the controls and the expanded state.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(args.get('root') ?? 'build/probe-chat-message-material');
|
|
const out = resolve(args.get('out') ?? 'build/chat-message-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.mjs': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
const type = mime[extname(filePath)] ?? 'application/octet-stream';
|
|
res.writeHead(200, { 'content-type': type }).end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'c2-message-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 150; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) {
|
|
throw new Error(`Target creation failed: ${JSON.stringify(created)}`);
|
|
}
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 390, height: 844, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
const nodes = () =>
|
|
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
expanded: x.getAttribute('aria-expanded'),
|
|
role: x.getAttribute('role'),
|
|
x: r.x, y: r.y, w: r.width, h: r.height,
|
|
};
|
|
})`);
|
|
const findNode = async (name) => {
|
|
const all = await nodes();
|
|
return all.find(
|
|
(node) => node.label === name || node.text === name,
|
|
);
|
|
};
|
|
const requireNode = async (name) => {
|
|
const node = await findNode(name);
|
|
if (!node) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(`Missing semantics ${JSON.stringify(name)}: ${JSON.stringify(shown)}`);
|
|
}
|
|
return node;
|
|
};
|
|
// A real pointer tap at the node's centre, for controls inside a modal sheet.
|
|
const tap = async (name) => {
|
|
const node = await findNode(name);
|
|
if (!node) throw new Error(`Could not tap ${JSON.stringify(name)}`);
|
|
const x = node.x + node.w / 2;
|
|
const y = node.y + node.h / 2;
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x, y, button: 'left', clickCount: 1 }, session);
|
|
}
|
|
await delay(450);
|
|
};
|
|
const click = async (name) => {
|
|
const clicked = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(name)};
|
|
const all = [...document.querySelectorAll('flt-semantics')];
|
|
const node = all.find(x => x.getAttribute('aria-label') === wanted)
|
|
|| all.find(x => (x.innerText || '').trim() === wanted);
|
|
if (!node) return false;
|
|
node.click();
|
|
return true;
|
|
})()`);
|
|
if (!clicked) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(`Could not click ${JSON.stringify(name)}: ${JSON.stringify(shown)}`);
|
|
}
|
|
await delay(450);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
const assertOcr = async (path, text) => {
|
|
const read = await ocr(path);
|
|
if (!read.toLowerCase().includes(text.toLowerCase())) {
|
|
throw new Error(
|
|
`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 2000)}`,
|
|
);
|
|
}
|
|
};
|
|
const words = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [
|
|
path,
|
|
'stdout',
|
|
'tsv',
|
|
]);
|
|
return stdout
|
|
.split('\n')
|
|
.slice(1)
|
|
.map((line) => line.split('\t'))
|
|
.filter((cells) => cells.length > 11 && cells[11].trim())
|
|
.map((cells) => ({
|
|
text: cells[11].trim(),
|
|
x: Number(cells[6]),
|
|
y: Number(cells[7]),
|
|
w: Number(cells[8]),
|
|
h: Number(cells[9]),
|
|
}));
|
|
};
|
|
// The leading edge of the first word of a body that starts below `label` and
|
|
// above `next`, against the label's own word. A centred body lands near the
|
|
// middle of a 390-wide phone, so the tolerance is a few glyph bearings.
|
|
const assertBodyEdge = async (path, labelText, nextText) => {
|
|
const all = await words(path);
|
|
const label = all.find((word) => word.text === labelText);
|
|
if (!label) throw new Error(`OCR found no ${labelText} label`);
|
|
const next = nextText
|
|
? all.find((word) => word.text === nextText && word.y > label.y)
|
|
: null;
|
|
const body = all.filter(
|
|
(word) =>
|
|
word.y > label.y + label.h &&
|
|
(!next || word.y < next.y - 4) &&
|
|
word.w < 700,
|
|
);
|
|
if (body.length === 0) {
|
|
throw new Error(`OCR found no ${labelText} body words`);
|
|
}
|
|
const left = Math.min(...body.map((word) => word.x));
|
|
evidence.checks.push({
|
|
check: `${labelText} body starts at its label edge`,
|
|
labelX: label.x,
|
|
bodyX: left,
|
|
delta: left - label.x,
|
|
});
|
|
if (Math.abs(left - label.x) > 12) {
|
|
throw new Error(
|
|
`${labelText} body starts at ${left} but its label at ${label.x}`,
|
|
);
|
|
}
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
const contentType = await evaluate(
|
|
"fetch('/').then(r => r.headers.get('content-type'))",
|
|
);
|
|
if (!String(contentType).toLowerCase().startsWith('text/html')) {
|
|
throw new Error(`Root index is not text/html: ${contentType}`);
|
|
}
|
|
evidence.checks.push({
|
|
check: 'fixture index is served as text/html',
|
|
value: contentType,
|
|
});
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
await waitFor(async () => Boolean(await findNode('Assistant message')), 'semantics');
|
|
|
|
for (const name of [
|
|
'User message',
|
|
'Assistant message',
|
|
'Reasoning, expand for details',
|
|
'1 tool call, expand for details',
|
|
'Unrecognised part: future_part',
|
|
'Ada phone',
|
|
]) {
|
|
await requireNode(name);
|
|
}
|
|
const userGroup = await requireNode('User message');
|
|
const agentGroup = await requireNode('Assistant message');
|
|
if (userGroup.text.trim() === 'User' || agentGroup.text.trim() === 'Assistant') {
|
|
throw new Error('Row semantics repeat the visible role label');
|
|
}
|
|
evidence.checks.push({
|
|
check: 'roles, client, and collapsed disclosures are present once',
|
|
userText: userGroup.text,
|
|
agentText: agentGroup.text,
|
|
});
|
|
|
|
const top = await capture(`${style}-top`);
|
|
for (const text of [
|
|
'Please keep every word of this prompt',
|
|
'plan.md',
|
|
// `report.csv` covers both hosts; Material's small paperclip glyph makes
|
|
// OCR drop the leading `r`, so the suffix is the stable part.
|
|
'eport.csv',
|
|
'checked the transcript and kept the original order.',
|
|
]) {
|
|
await assertOcr(top, text);
|
|
}
|
|
const topText = await ocr(top);
|
|
for (const hidden of ['The user asked for exact', 'command', 'payload']) {
|
|
if (topText.toLowerCase().includes(hidden.toLowerCase())) {
|
|
throw new Error(`Closed disclosure leaked ${JSON.stringify(hidden)}`);
|
|
}
|
|
}
|
|
// Reasoning is a chip beside the tool chip; it opens a sheet.
|
|
await click('Reasoning, expand for details');
|
|
const openReasoning = await capture(`${style}-reasoning-expanded`);
|
|
await assertOcr(openReasoning, 'The user asked for exact text');
|
|
// Dismiss the sheet through its barrier.
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x: 10, y: 10, button: 'left', clickCount: 1 }, session);
|
|
}
|
|
await delay(800);
|
|
|
|
await click('Unrecognised part: future_part');
|
|
const openUnknown = await capture(`${style}-unknown-expanded`);
|
|
for (const text of ['future_part', 'payload', 'kept', 'count']) {
|
|
await assertOcr(openUnknown, text);
|
|
}
|
|
|
|
// As the web: the turn's tool calls are one chip below the answer that
|
|
// opens a sheet; a call there opens to its formatted input.
|
|
await click('1 tool call, expand for details');
|
|
await waitFor(async () => Boolean(await findNode('shell · ls -la')), 'tool sheet');
|
|
// Let the sheet finish its entrance before the row is activated.
|
|
await delay(800);
|
|
await tap('shell · ls -la');
|
|
await delay(600);
|
|
const openTool = await capture(`${style}-tool-expanded`);
|
|
for (const text of ['INPUT', 'command', '-la']) {
|
|
await assertOcr(openTool, text);
|
|
}
|
|
|
|
// Dismiss the tool sheet, then bring the live turn into view.
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', { type, x: 10, y: 10, button: 'left', clickCount: 1 }, session);
|
|
}
|
|
await delay(800);
|
|
await send('Input.dispatchMouseEvent', { type: 'mouseWheel', x: 195, y: 420, deltaX: 0, deltaY: 3000 }, session);
|
|
await delay(800);
|
|
const liveName = 'Reasoning · 2 steps, expand for details';
|
|
await requireNode(liveName);
|
|
const liveChip = await capture(`${style}-live-chip`);
|
|
await assertOcr(liveChip, 'Live answer so far');
|
|
await assertOcr(liveChip, 'Reasoning');
|
|
// The open sheet follows the live turn: its newest step keeps growing.
|
|
// A real pointer tap on the chip's own node (the smallest one carrying its
|
|
// name; its row group repeats the text).
|
|
const chipNode = (await nodes())
|
|
.filter((node) => node.label === liveName || node.text === liveName)
|
|
.sort((a, b) => a.w * a.h - b.w * b.h)[0];
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send('Input.dispatchMouseEvent', {
|
|
type,
|
|
x: chipNode.x + chipNode.w / 2,
|
|
y: chipNode.y + chipNode.h / 2,
|
|
button: 'left',
|
|
clickCount: 1,
|
|
}, session);
|
|
}
|
|
await delay(800);
|
|
const tickOf = async (path) => {
|
|
const match = (await ocr(path)).match(/Live tick (\d+)/);
|
|
if (!match) throw new Error(`OCR of ${path} shows no live tick`);
|
|
return Number(match[1]);
|
|
};
|
|
const liveA = await capture(`${style}-live-sheet-a`);
|
|
await assertOcr(liveA, 'Live step one settled');
|
|
const first = await tickOf(liveA);
|
|
await delay(2000);
|
|
const liveB = await capture(`${style}-live-sheet-b`);
|
|
const second = await tickOf(liveB);
|
|
evidence.checks.push({ check: 'open reasoning sheet follows live deltas', first, second });
|
|
if (second <= first) {
|
|
throw new Error(`Open sheet did not follow live deltas: ${first} then ${second}`);
|
|
}
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|