Files
pipelines/app/tool/chat_wire_probe.mjs
T

531 lines
17 KiB
JavaScript

// Real-browser evidence for the Chat destination wiring fixture.
//
// Build one host, then run from app/:
// flutter build web -t tool/chat_wire_probe.dart \
// --dart-define=CHAT_WIRE_PROBE_STYLE=material \
// --output build/probe-chat-wire-material
// node tool/chat_wire_probe.mjs --root build/probe-chat-wire-material \
// --out DIR --style material
//
// The Cupertino host is the same fixture built with CHAT_WIRE_PROBE_STYLE=cupertino.
// Requires Google Chrome and Tesseract: Flutter web paints selectable text on
// the canvas, so OCR confirms the actual message body, not just nav labels.
import { createServer } from 'node:http';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { spawn, execFile } from 'node:child_process';
import { once } from 'node:events';
import { promisify } from 'node:util';
import { setTimeout as delay } from 'node:timers/promises';
import { extname, join, resolve, sep } from 'node:path';
import { tmpdir } from 'node:os';
const args = new Map();
for (let i = 2; i < process.argv.length; i++) {
const value = process.argv[i];
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
}
const root = resolve(args.get('root') ?? 'build/probe-chat-wire-material');
const out = resolve(args.get('out') ?? 'build/chat-wire-probe');
const style = args.get('style') ?? 'material';
const chromePath =
process.env.CHROME_PATH ??
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
await mkdir(out, { recursive: true });
const mime = {
'.html': 'text/html; charset=utf-8',
'.js': 'text/javascript; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.wasm': 'application/wasm',
'.png': 'image/png',
'.svg': 'image/svg+xml',
'.ico': 'image/x-icon',
'.ttf': 'font/ttf',
'.otf': 'font/otf',
'.woff': 'font/woff',
'.woff2': 'font/woff2',
'.bin': 'application/octet-stream',
};
const server = createServer(async (req, res) => {
try {
const pathname = decodeURIComponent(
new URL(req.url, 'http://localhost').pathname,
);
let filePath = resolve(root, '.' + pathname);
if (filePath !== root && !filePath.startsWith(root + sep)) {
res.writeHead(403).end();
return;
}
if (pathname === '/' || pathname.endsWith('/')) {
filePath = join(filePath, 'index.html');
}
const file = await readFile(filePath);
res.writeHead(200, {
'content-type': mime[extname(filePath)] ?? 'application/octet-stream',
});
res.end(file);
} catch {
res.writeHead(404).end('not found');
}
});
server.listen(0, '127.0.0.1');
await once(server, 'listening');
const origin = `http://127.0.0.1:${server.address().port}`;
const profile = await mkdtemp(join(tmpdir(), 'c2-wire-chrome-'));
const chrome = spawn(
chromePath,
[
'--headless=new',
'--no-first-run',
'--no-default-browser-check',
'--disable-gpu',
'--remote-debugging-port=0',
`--user-data-dir=${profile}`,
'about:blank',
],
{ stdio: ['ignore', 'ignore', 'pipe'] },
);
let stderr = '';
chrome.stderr.on('data', (data) => {
stderr += data;
});
const waitFor = async (fn, what) => {
for (let i = 0; i < 180; i++) {
const value = await fn();
if (value) return value;
await delay(100);
}
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
};
let ws;
const evidence = { origin, style, captures: [], checks: [] };
try {
const endpoint = await waitFor(
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
'Chrome DevTools',
);
ws = new WebSocket(endpoint);
await once(ws, 'open');
let id = 0;
const pending = new Map();
ws.addEventListener('message', (event) => {
const message = JSON.parse(event.data);
if (message.id && pending.has(message.id)) {
pending.get(message.id)(message);
pending.delete(message.id);
}
});
const send = (method, params = {}, sessionId) =>
new Promise((resolveCall) => {
const callId = ++id;
pending.set(callId, resolveCall);
ws.send(
JSON.stringify({
id: callId,
method,
params,
...(sessionId ? { sessionId } : {}),
}),
);
});
const created = await send('Target.createTarget', { url: 'about:blank' });
if (!created.result) throw new Error('Target creation failed');
const target = created.result.targetId;
const session = (
await send('Target.attachToTarget', { targetId: target, flatten: true })
).result.sessionId;
await send('Page.enable', {}, session);
await send('Runtime.enable', {}, session);
await send(
'Emulation.setDeviceMetricsOverride',
{ width: 390, height: 844, deviceScaleFactor: 2, mobile: true },
session,
);
await send(
'Emulation.setEmulatedMedia',
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
session,
);
const evaluate = async (expression) => {
const response = await send(
'Runtime.evaluate',
{ expression, returnByValue: true, awaitPromise: true },
session,
);
if (response.error) {
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
}
return response.result?.result?.value;
};
const nodes = () =>
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
const r = x.getBoundingClientRect();
return {
label: x.getAttribute('aria-label'),
text: (x.innerText || '').trim(),
x: r.x, y: r.y, w: r.width, h: r.height,
};
})`);
const findNode = async (name) => {
const all = await nodes();
return all.find((node) => node.label === name || node.text === name);
};
const requireNode = async (name) => {
const node = await findNode(name);
if (!node) {
const shown = (await nodes())
.map((entry) => entry.label ?? entry.text)
.filter(Boolean);
throw new Error(
`Missing semantics ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
);
}
return node;
};
const click = async (name, { large = false } = {}) => {
const point = await evaluate(`(() => {
const wanted = ${JSON.stringify(name)};
const large = ${large};
const matches = [...document.querySelectorAll('flt-semantics')]
.map(x => {
const label = x.getAttribute('aria-label') || '';
const text = (x.innerText || '').trim();
const r = x.getBoundingClientRect();
const labelHit = label.includes(wanted) ? 0 : 1;
return { x, label, text, labelHit, area: r.width * r.height };
})
.filter(x => large ? x.labelHit === 0 : (x.labelHit === 0 || x.text.includes(wanted)))
.sort(large
? (a, b) => b.area - a.area
: (a, b) => a.labelHit - b.labelHit || a.area - b.area);
const node = matches[0]?.x;
if (!node) return null;
const r = node.getBoundingClientRect();
return {
x: large ? r.x + Math.min(r.width / 2, 100) : r.x + r.width / 2,
y: r.y + r.height / 2,
};
})()`);
if (!point) {
const shown = (await nodes())
.map((entry) => entry.label ?? entry.text)
.filter(Boolean);
throw new Error(
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
);
}
await send(
'Input.dispatchMouseEvent',
{ type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 },
session,
);
await send(
'Input.dispatchMouseEvent',
{ type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 },
session,
);
await delay(700);
};
// The first matching control in document order. A folded assistant turn can
// hold several `Reasoning` labels; the newest message renders first.
const clickFirst = async (name) => {
const node = (await nodes()).find(
(entry) => entry.label === name || entry.text === name,
);
if (!node) {
const shown = (await nodes())
.map((entry) => entry.label ?? entry.text)
.filter(Boolean);
throw new Error(
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
);
}
const x = node.x + node.w / 2;
const y = node.y + node.h / 2;
for (const type of ['mousePressed', 'mouseReleased']) {
await send(
'Input.dispatchMouseEvent',
{ type, x, y, button: 'left', clickCount: 1 },
session,
);
}
await delay(700);
};
const capture = async (name) => {
const data = (
await send('Page.captureScreenshot', { format: 'png' }, session)
).result.data;
const file = join(out, `${style}-${name}.png`);
await writeFile(file, Buffer.from(data, 'base64'));
evidence.captures.push(file);
return file;
};
const ocr = async (path) => {
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
return stdout.replace(/\s+/g, ' ');
};
const assertOcr = async (path, text) => {
const read = await ocr(path);
if (!read.toLowerCase().includes(text.toLowerCase())) {
throw new Error(
`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 1600)}`,
);
}
};
const hasText = async (fragment) => {
const all = await nodes();
return all.some((node) =>
`${node.label ?? ''} ${node.text ?? ''}`
.toLowerCase()
.includes(fragment.toLowerCase()),
);
};
await send('Page.navigate', { url: `${origin}/` }, session);
await delay(3500);
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
await delay(1200);
await waitFor(
async () => hasText('Probe conversation'),
'session list semantics',
);
const list = await capture('list');
await assertOcr(list, 'Probe conversation');
await assertOcr(list, 'Another stored chat');
if (!(await hasText('New chat'))) {
throw new Error('Embedded list lost its New chat action');
}
if (!(await hasText('Refresh'))) {
throw new Error('Embedded list lost its Refresh action');
}
evidence.checks.push({
check: 'embedded list renders server rows and actions',
labels: (await nodes())
.map((node) => node.label)
.filter(Boolean)
.slice(0, 20),
});
await click('Probe conversation', { large: true });
await waitFor(
async () => Boolean(await findNode('Assistant message')),
'thread semantics',
);
for (const name of [
'User message',
'Assistant message',
'Reasoning',
'shell',
]) {
await requireNode(name);
}
const thread = await capture('thread');
for (const text of [
'read-only message from the server',
'stored assistant reply',
'existing native message view',
'Probe conversation',
]) {
await assertOcr(thread, text);
}
const closed = await ocr(thread);
for (const hidden of [
'Reasoning stays behind',
'NEWEST reasoning body',
]) {
if (closed.toLowerCase().includes(hidden.toLowerCase())) {
throw new Error(`Collapsed disclosure leaked ${JSON.stringify(hidden)}`);
}
}
evidence.checks.push({
check: 'actual stored message body renders in the thread',
style,
});
await delay(1500);
const live = await capture('live');
await assertOcr(live, 'A live delta updated this thread');
evidence.checks.push({
check: 'a pushed chat_event delta updates the open thread',
style,
});
// Repro B: expand the newest reasoning, then load the one older page whose
// `has_more` is false, so the `Load older` child disappears.
await click('Reasoning');
const expanded = await capture('expanded');
await assertOcr(expanded, 'NEWEST reasoning body');
await click('Load older');
await delay(1200);
const afterOlder = await capture('older');
const afterOlderText = await ocr(afterOlder);
const newestKept = afterOlderText.includes('NEWEST reasoning body');
const olderLeaked = afterOlderText.includes('OLDER reasoning body');
const loadOlderGone = !(await findNode('Load older'));
evidence.observations = {
...evidence.observations,
afterOlderRemovesControl: { newestKept, olderLeaked, loadOlderGone },
};
if (!newestKept) {
throw new Error(
`The newest open reasoning was lost after Load older:\n${afterOlderText.slice(0, 1600)}`,
);
}
if (olderLeaked) {
throw new Error(
`The older reasoning stole the newest disclosure:\n${afterOlderText.slice(0, 1600)}`,
);
}
if (!loadOlderGone) {
throw new Error('Load older stayed visible after has_more became false');
}
await assertOcr(afterOlder, 'older stored message came from the opaque cursor page');
await assertOcr(afterOlder, 'Older reply kept in order');
evidence.checks.push({
check: 'Load older keeps the open disclosure and renders the opaque page',
style,
});
// Repro A: the same action while the server still reports `has_more`, so the
// paging control remains in the indexed list.
await click(style === 'cupertino' ? 'Back' : 'Back to chats');
await waitFor(
async () => hasText('Probe paging session'),
'list after Back',
);
await click('Probe paging session', { large: true });
await waitFor(
async () => Boolean(await findNode('Assistant message')),
'second thread semantics',
);
await click('Reasoning');
const moreExpanded = await capture('more-expanded');
await assertOcr(moreExpanded, 'MORE NEWEST reasoning body');
await click('Load older');
await delay(1200);
const moreAfterOlder = await capture('more-older');
const moreAfterOlderText = await ocr(moreAfterOlder);
const moreNewestKept = moreAfterOlderText.includes('MORE NEWEST reasoning body');
const moreOlderLeaked = moreAfterOlderText.includes('MORE OLDER reasoning body');
const moreControlStays = Boolean(await findNode('Load older'));
evidence.observations.afterOlderKeepsControl = {
newestKept: moreNewestKept,
olderLeaked: moreOlderLeaked,
loadOlderPresent: moreControlStays,
};
if (!moreNewestKept) {
throw new Error(
`The newest open reasoning was lost while has_more stayed true:\n${moreAfterOlderText.slice(0, 1600)}`,
);
}
if (moreOlderLeaked) {
throw new Error(
`The older reasoning stole the newest disclosure while has_more stayed true:\n${moreAfterOlderText.slice(0, 1600)}`,
);
}
if (!moreControlStays) {
throw new Error('Load older disappeared although has_more stayed true');
}
evidence.checks.push({
check: 'an open disclosure follows its part while paging keeps has_more',
style,
});
// New repro: the paged-history fixture's shape. Both pages hold only
// consecutive assistant messages, so an older page prepends into the single
// folded turn and changes its first message id. The row element must survive
// that identity change and keep the operator's open block.
for (const fold of [
{
title: 'Probe assistant fold session',
tag: 'fold',
keepsControl: true,
},
{
title: 'Probe assistant fold last page',
tag: 'fold-end',
keepsControl: false,
},
]) {
await click(style === 'cupertino' ? 'Back' : 'Back to chats');
await waitFor(
async () => hasText(fold.title),
`list before ${fold.tag}`,
);
await click(fold.title, { large: true });
await waitFor(
async () => Boolean(await findNode('Assistant message')),
`${fold.tag} thread semantics`,
);
await clickFirst('Reasoning');
await delay(400);
const foldExpanded = await capture(`${fold.tag}-expanded`);
await assertOcr(foldExpanded, 'FOLD NEWEST reasoning body');
await click('Load older');
await delay(1200);
const foldAfter = await capture(`${fold.tag}-older`);
const foldText = await ocr(foldAfter);
const foldNewestKept = foldText.includes('FOLD NEWEST reasoning body');
const foldOlderLeaked = [
'FOLD SECOND reasoning body',
'FOLD OLDEST reasoning body',
].some((body) => foldText.includes(body));
const foldControlStays = Boolean(await findNode('Load older'));
evidence.observations = {
...evidence.observations,
[`${fold.tag}AfterOlder`]: {
newestKept: foldNewestKept,
olderLeaked: foldOlderLeaked,
loadOlderPresent: foldControlStays,
},
};
if (!foldNewestKept) {
throw new Error(
`The folded turn lost the open reasoning after Load older:\n${foldText.slice(0, 1600)}`,
);
}
if (foldOlderLeaked) {
throw new Error(
`An older block in the folded turn opened itself:\n${foldText.slice(0, 1600)}`,
);
}
if (foldControlStays !== fold.keepsControl) {
throw new Error(
`Load older presence ${foldControlStays} did not match has_more ${fold.keepsControl}`,
);
}
evidence.checks.push({
check:
'an open disclosure survives paging inside a folded assistant turn',
style,
case: fold.tag,
});
}
const report = JSON.stringify(evidence, null, 2);
await writeFile(join(out, `${style}-evidence.json`), report);
console.log(report);
} finally {
ws?.close();
chrome.kill();
server.close();
for (let attempt = 0; attempt < 10; attempt++) {
try {
await rm(profile, { recursive: true, force: true });
break;
} catch {
await delay(200);
}
}
}