b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
354 lines
11 KiB
JavaScript
354 lines
11 KiB
JavaScript
// Real-browser evidence for the C2 markdown wiring fixture.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_markdown_wire_probe.dart \
|
|
// --dart-define=CHAT_MARKDOWN_WIRE_PROBE_STYLE=material \
|
|
// --output build/probe-chat-markdown-wire-material
|
|
// node tool/chat_markdown_wire_probe.mjs \
|
|
// --root build/probe-chat-markdown-wire-material \
|
|
// --out DIR --style material
|
|
//
|
|
// The Material host is the same fixture built with the material define.
|
|
// Requires Google Chrome and Tesseract: Flutter web paints text on the canvas,
|
|
// so OCR confirms the rendered answer while the semantics tree confirms the
|
|
// heading and the row structure.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(
|
|
args.get('root') ?? 'build/probe-chat-markdown-wire-material',
|
|
);
|
|
const out = resolve(args.get('out') ?? 'build/chat-markdown-wire-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
res.writeHead(200, {
|
|
'content-type': mime[extname(filePath)] ?? 'application/octet-stream',
|
|
});
|
|
res.end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'c2-markdown-wire-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 180; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
if (!created.result) throw new Error('Target creation failed');
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 390, height: 844, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
const nodes = () =>
|
|
evaluate(`[...document.querySelectorAll('flt-semantics')].map(x => {
|
|
const r = x.getBoundingClientRect();
|
|
return {
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
level: x.getAttribute('aria-level'),
|
|
role: x.getAttribute('role'),
|
|
x: r.x, y: r.y, w: r.width, h: r.height,
|
|
};
|
|
})`);
|
|
const findNode = async (name) => {
|
|
const all = await nodes();
|
|
return all.find((node) => node.label === name || node.text === name);
|
|
};
|
|
const requireNode = async (name) => {
|
|
const node = await findNode(name);
|
|
if (!node) {
|
|
const shown = (await nodes())
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Missing semantics ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
return node;
|
|
};
|
|
const hasText = async (fragment) => {
|
|
const all = await nodes();
|
|
return all.some((node) =>
|
|
`${node.label ?? ''} ${node.text ?? ''}`
|
|
.toLowerCase()
|
|
.includes(fragment.toLowerCase()),
|
|
);
|
|
};
|
|
const click = async (name, { skipButtons = false } = {}) => {
|
|
const all = await nodes();
|
|
const score = (node) => {
|
|
const label = node.label ?? '';
|
|
const text = node.text ?? '';
|
|
if (label === name || text === name) return 0;
|
|
if (label.startsWith(name) || text.startsWith(name)) return 1;
|
|
if (label.includes(name) || text.includes(name)) return 2;
|
|
return -1;
|
|
};
|
|
const matches = all
|
|
.filter((node) => score(node) >= 0)
|
|
.filter((node) => !skipButtons || node.role !== 'button')
|
|
.sort((a, b) => score(a) - score(b) || a.w * a.h - b.w * b.h);
|
|
const node = matches[0];
|
|
if (!node) {
|
|
const shown = all
|
|
.map((entry) => entry.label ?? entry.text)
|
|
.filter(Boolean);
|
|
throw new Error(
|
|
`Could not find ${JSON.stringify(name)}: ${JSON.stringify(shown)}`,
|
|
);
|
|
}
|
|
const x = node.x + node.w / 2;
|
|
const y = node.y + node.h / 2;
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await send(
|
|
'Input.dispatchMouseEvent',
|
|
{ type, x, y, button: 'left', clickCount: 1 },
|
|
session,
|
|
);
|
|
}
|
|
await delay(900);
|
|
};
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${style}-${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
const assertOcr = async (path, text) => {
|
|
const read = await ocr(path);
|
|
if (!read.toLowerCase().includes(text.toLowerCase())) {
|
|
throw new Error(
|
|
`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 1600)}`,
|
|
);
|
|
}
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
await waitFor(
|
|
async () => hasText('Markdown wire'),
|
|
'session list semantics',
|
|
);
|
|
// The row's trailing actions are separate buttons whose labels also contain
|
|
// the title; open the row itself, never its overflow menu.
|
|
await click('Markdown wire', { skipButtons: true });
|
|
await waitFor(
|
|
async () => Boolean(await findNode('Assistant message')),
|
|
'thread semantics',
|
|
);
|
|
|
|
// The user row keeps its literal characters and the folded assistant turn
|
|
// keeps the unchanged Reasoning disclosure.
|
|
const user = await requireNode('User message');
|
|
const assistant = await requireNode('Assistant message');
|
|
await requireNode('Reasoning');
|
|
evidence.checks.push({
|
|
check: 'user and folded assistant rows are present once',
|
|
userLabel: user.label,
|
|
assistantLabel: assistant.label,
|
|
});
|
|
|
|
const shot = await capture('markdown-thread');
|
|
for (const text of [
|
|
'Quarterly status',
|
|
'release',
|
|
'review pending',
|
|
'dart',
|
|
'count = 3',
|
|
'unsafe',
|
|
// The user's own message is not parsed.
|
|
'Keep **this** exactly',
|
|
]) {
|
|
await assertOcr(shot, text);
|
|
}
|
|
const read = await ocr(shot);
|
|
for (const raw of [
|
|
'## Quarterly',
|
|
'```dart',
|
|
'| --- |',
|
|
'**release**',
|
|
'unsafe](javascript',
|
|
]) {
|
|
if (read.toLowerCase().includes(raw.toLowerCase())) {
|
|
throw new Error(`Raw markdown marker rendered: ${JSON.stringify(raw)}`);
|
|
}
|
|
}
|
|
await assertOcr(shot, 'unsafe stay visible');
|
|
|
|
const headings = await evaluate(`[...document.querySelectorAll(
|
|
'flt-semantics[aria-level], h1, h2, h3, h4, h5, h6')]
|
|
.map(x => ({
|
|
label: x.getAttribute('aria-label'),
|
|
text: (x.innerText || '').trim(),
|
|
level: x.getAttribute('aria-level') ??
|
|
(/^H[1-6]$/.test(x.tagName) ? x.tagName.slice(1) : null),
|
|
}))`);
|
|
const heading = headings.find((node) =>
|
|
`${node.label ?? ''}${node.text ?? ''}`.includes('Quarterly'),
|
|
);
|
|
if (!heading) {
|
|
throw new Error(
|
|
`The formatted heading is missing its level: ${JSON.stringify(headings)}`,
|
|
);
|
|
}
|
|
evidence.checks.push({
|
|
check: 'assistant answer renders formatted markdown blocks',
|
|
host: style,
|
|
headingLevel: heading.level,
|
|
rawMarkersAbsent: true,
|
|
capture: shot,
|
|
});
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|