b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
531 lines
17 KiB
JavaScript
531 lines
17 KiB
JavaScript
// Real-browser evidence for the Chat destination wiring fixture.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_wire_probe.dart \
|
|
// --dart-define=CHAT_WIRE_PROBE_STYLE=material \
|
|
// --output build/probe-chat-wire-material
|
|
// node tool/chat_wire_probe.mjs --root build/probe-chat-wire-material \
|
|
// --out DIR --style material
|
|
//
|
|
// The Material host is the same fixture built with CHAT_WIRE_PROBE_STYLE=material.
|
|
// Requires Google Chrome and Tesseract: Flutter web paints selectable text on
|
|
// the canvas, so OCR confirms the actual message body, not just nav labels.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(args.get('root') ?? 'build/probe-chat-wire-material');
|
|
const out = resolve(args.get('out') ?? 'build/chat-wire-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, {
|
|
'content-type': mime[extname(filePath)] ?? 'application/octet-stream',
|
|
});
|
|
res.end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'c2-wire-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 180; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) throw new Error('Target creation failed');
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 390, height: 844, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
const nodes = () =>
|
|
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
x: r.x, y: r.y, w: r.width, h: r.height,
|
|
};
|
|
})`);
|
|
const findNode = async (name) => {
|
|
const all = await nodes();
|
|
return all.find((node) => node.label === name || node.text === name);
|
|
};
|
|
const requireNode = async (name) => {
|
|
const node = await findNode(name);
|
|
if (!node) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Missing semantics ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
return node;
|
|
};
|
|
const click = async (name, { large = false } = {}) => {
|
|
const point = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(name)};
|
|
const large = ${large};
|
|
const matches = [...document.querySelectorAll('flt-semantics')]
|
|
.map(x => {
|
|
const label = x.getAttribute('aria-label') || '';
|
|
const text = (x.innerText || '').trim();
|
|
const r = x.getBoundingClientRect();
|
|
const labelHit = label.includes(wanted) ? 0 : 1;
|
|
return { x, label, text, labelHit, area: r.width * r.height };
|
|
})
|
|
.filter(x => large ? x.labelHit === 0 : (x.labelHit === 0 || x.text.includes(wanted)))
|
|
.sort(large
|
|
? (a, b) => b.area - a.area
|
|
: (a, b) => a.labelHit - b.labelHit || a.area - b.area);
|
|
const node = matches[0]?.x;
|
|
if (!node) return null;
|
|
const r = node.getBoundingClientRect();
|
|
return {
|
|
x: large ? r.x + Math.min(r.width / 2, 100) : r.x + r.width / 2,
|
|
y: r.y + r.height / 2,
|
|
};
|
|
})()`);
|
|
if (!point) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
await send(
|
|
'Input.dispatchMouseEvent',
|
|
{ type: 'mousePressed', x: point.x, y: point.y, button: 'left', clickCount: 1 },
|
|
session,
|
|
);
|
|
await send(
|
|
'Input.dispatchMouseEvent',
|
|
{ type: 'mouseReleased', x: point.x, y: point.y, button: 'left', clickCount: 1 },
|
|
session,
|
|
);
|
|
await delay(700);
|
|
};
|
|
// The first matching control in document order. A folded assistant turn can
|
|
// hold several `Reasoning` labels; the newest message renders first.
|
|
const clickFirst = async (name) => {
|
|
const node = (await nodes()).find(
|
|
(entry) => entry.label === name || entry.text === name,
|
|
);
|
|
if (!node) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
const x = node.x + node.w / 2;
|
|
const y = node.y + node.h / 2;
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send(
|
|
'Input.dispatchMouseEvent',
|
|
{ type, x, y, button: 'left', clickCount: 1 },
|
|
session,
|
|
);
|
|
}
|
|
await delay(700);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${style}-${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
const assertOcr = async (path, text) => {
|
|
const read = await ocr(path);
|
|
if (!read.toLowerCase().includes(text.toLowerCase())) {
|
|
throw new Error(
|
|
`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
};
|
|
|
|
const hasText = async (fragment) => {
|
|
const all = await nodes();
|
|
return all.some((node) =>
|
|
`${node.label ?? ''} ${node.text ?? ''}`
|
|
.toLowerCase()
|
|
.includes(fragment.toLowerCase()),
|
|
);
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
await waitFor(
|
|
async () => hasText('Probe conversation'),
|
|
'session list semantics',
|
|
);
|
|
|
|
const list = await capture('list');
|
|
await assertOcr(list, 'Probe conversation');
|
|
await assertOcr(list, 'Another stored chat');
|
|
if (!(await hasText('New chat'))) {
|
|
throw new Error('Embedded list lost its New chat action');
|
|
}
|
|
if (!(await hasText('Refresh'))) {
|
|
throw new Error('Embedded list lost its Refresh action');
|
|
}
|
|
evidence.checks.push({
|
|
check: 'embedded list renders server rows and actions',
|
|
labels: (await nodes())
|
|
.map((node) => node.label)
|
|
.filter(Boolean)
|
|
.slice(0, 20),
|
|
});
|
|
|
|
await click('Probe conversation', { large: true });
|
|
await waitFor(
|
|
async () => Boolean(await findNode('Assistant message')),
|
|
'thread semantics',
|
|
);
|
|
for (const name of [
|
|
'User message',
|
|
'Assistant message',
|
|
'Reasoning',
|
|
'shell',
|
|
]) {
|
|
await requireNode(name);
|
|
}
|
|
const thread = await capture('thread');
|
|
for (const text of [
|
|
'read-only message from the server',
|
|
'stored assistant reply',
|
|
'existing native message view',
|
|
'Probe conversation',
|
|
]) {
|
|
await assertOcr(thread, text);
|
|
}
|
|
const closed = await ocr(thread);
|
|
for (const hidden of [
|
|
'Reasoning stays behind',
|
|
'NEWEST reasoning body',
|
|
]) {
|
|
if (closed.toLowerCase().includes(hidden.toLowerCase())) {
|
|
throw new Error(`Collapsed disclosure leaked ${JSON.stringify(hidden)}`);
|
|
}
|
|
}
|
|
evidence.checks.push({
|
|
check: 'actual stored message body renders in the thread',
|
|
style,
|
|
});
|
|
|
|
await delay(1500);
|
|
const live = await capture('live');
|
|
await assertOcr(live, 'A live delta updated this thread');
|
|
evidence.checks.push({
|
|
check: 'a pushed chat_event delta updates the open thread',
|
|
style,
|
|
});
|
|
|
|
// Repro B: expand the newest reasoning, then load the one older page whose
|
|
// `has_more` is false, so the `Load older` child disappears.
|
|
await click('Reasoning');
|
|
const expanded = await capture('expanded');
|
|
await assertOcr(expanded, 'NEWEST reasoning body');
|
|
|
|
await click('Load older');
|
|
await delay(1200);
|
|
const afterOlder = await capture('older');
|
|
const afterOlderText = await ocr(afterOlder);
|
|
const newestKept = afterOlderText.includes('NEWEST reasoning body');
|
|
const olderLeaked = afterOlderText.includes('OLDER reasoning body');
|
|
const loadOlderGone = !(await findNode('Load older'));
|
|
evidence.observations = {
|
|
...evidence.observations,
|
|
afterOlderRemovesControl: { newestKept, olderLeaked, loadOlderGone },
|
|
};
|
|
if (!newestKept) {
|
|
throw new Error(
|
|
`The newest open reasoning was lost after Load older:\n${afterOlderText.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
if (olderLeaked) {
|
|
throw new Error(
|
|
`The older reasoning stole the newest disclosure:\n${afterOlderText.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
if (!loadOlderGone) {
|
|
throw new Error('Load older stayed visible after has_more became false');
|
|
}
|
|
await assertOcr(afterOlder, 'older stored message came from the opaque cursor page');
|
|
await assertOcr(afterOlder, 'Older reply kept in order');
|
|
evidence.checks.push({
|
|
check: 'Load older keeps the open disclosure and renders the opaque page',
|
|
style,
|
|
});
|
|
|
|
// Repro A: the same action while the server still reports `has_more`, so the
|
|
// paging control remains in the indexed list.
|
|
await click(style === 'material' ? 'Back' : 'Back to chats');
|
|
await waitFor(
|
|
async () => hasText('Probe paging session'),
|
|
'list after Back',
|
|
);
|
|
await click('Probe paging session', { large: true });
|
|
await waitFor(
|
|
async () => Boolean(await findNode('Assistant message')),
|
|
'second thread semantics',
|
|
);
|
|
await click('Reasoning');
|
|
const moreExpanded = await capture('more-expanded');
|
|
await assertOcr(moreExpanded, 'MORE NEWEST reasoning body');
|
|
|
|
await click('Load older');
|
|
await delay(1200);
|
|
const moreAfterOlder = await capture('more-older');
|
|
const moreAfterOlderText = await ocr(moreAfterOlder);
|
|
const moreNewestKept = moreAfterOlderText.includes('MORE NEWEST reasoning body');
|
|
const moreOlderLeaked = moreAfterOlderText.includes('MORE OLDER reasoning body');
|
|
const moreControlStays = Boolean(await findNode('Load older'));
|
|
evidence.observations.afterOlderKeepsControl = {
|
|
newestKept: moreNewestKept,
|
|
olderLeaked: moreOlderLeaked,
|
|
loadOlderPresent: moreControlStays,
|
|
};
|
|
if (!moreNewestKept) {
|
|
throw new Error(
|
|
`The newest open reasoning was lost while has_more stayed true:\n${moreAfterOlderText.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
if (moreOlderLeaked) {
|
|
throw new Error(
|
|
`The older reasoning stole the newest disclosure while has_more stayed true:\n${moreAfterOlderText.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
if (!moreControlStays) {
|
|
throw new Error('Load older disappeared although has_more stayed true');
|
|
}
|
|
evidence.checks.push({
|
|
check: 'an open disclosure follows its part while paging keeps has_more',
|
|
style,
|
|
});
|
|
|
|
// New repro: the paged-history fixture's shape. Both pages hold only
|
|
// consecutive assistant messages, so an older page prepends into the single
|
|
// folded turn and changes its first message id. The row element must survive
|
|
// that identity change and keep the operator's open block.
|
|
for (const fold of [
|
|
{
|
|
title: 'Probe assistant fold session',
|
|
tag: 'fold',
|
|
keepsControl: true,
|
|
},
|
|
{
|
|
title: 'Probe assistant fold last page',
|
|
tag: 'fold-end',
|
|
keepsControl: false,
|
|
},
|
|
]) {
|
|
await click(style === 'material' ? 'Back' : 'Back to chats');
|
|
await waitFor(
|
|
async () => hasText(fold.title),
|
|
`list before ${fold.tag}`,
|
|
);
|
|
await click(fold.title, { large: true });
|
|
await waitFor(
|
|
async () => Boolean(await findNode('Assistant message')),
|
|
`${fold.tag} thread semantics`,
|
|
);
|
|
await clickFirst('Reasoning');
|
|
await delay(400);
|
|
const foldExpanded = await capture(`${fold.tag}-expanded`);
|
|
await assertOcr(foldExpanded, 'FOLD NEWEST reasoning body');
|
|
|
|
await click('Load older');
|
|
await delay(1200);
|
|
const foldAfter = await capture(`${fold.tag}-older`);
|
|
const foldText = await ocr(foldAfter);
|
|
const foldNewestKept = foldText.includes('FOLD NEWEST reasoning body');
|
|
const foldOlderLeaked = [
|
|
'FOLD SECOND reasoning body',
|
|
'FOLD OLDEST reasoning body',
|
|
].some((body) => foldText.includes(body));
|
|
const foldControlStays = Boolean(await findNode('Load older'));
|
|
evidence.observations = {
|
|
...evidence.observations,
|
|
[`${fold.tag}AfterOlder`]: {
|
|
newestKept: foldNewestKept,
|
|
olderLeaked: foldOlderLeaked,
|
|
loadOlderPresent: foldControlStays,
|
|
},
|
|
};
|
|
if (!foldNewestKept) {
|
|
throw new Error(
|
|
`The folded turn lost the open reasoning after Load older:\n${foldText.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
if (foldOlderLeaked) {
|
|
throw new Error(
|
|
`An older block in the folded turn opened itself:\n${foldText.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
if (foldControlStays !== fold.keepsControl) {
|
|
throw new Error(
|
|
`Load older presence ${foldControlStays} did not match has_more ${fold.keepsControl}`,
|
|
);
|
|
}
|
|
evidence.checks.push({
|
|
check:
|
|
'an open disclosure survives paging inside a folded assistant turn',
|
|
style,
|
|
case: fold.tag,
|
|
});
|
|
}
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|