b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
285 lines
8.8 KiB
JavaScript
285 lines
8.8 KiB
JavaScript
// Real-browser evidence for the C2 markdown fixture.
|
|
//
|
|
// Build one host, then run from app/:
|
|
// flutter build web -t tool/chat_markdown_probe.dart \
|
|
// --dart-define=CHAT_MARKDOWN_PROBE_STYLE=material \
|
|
// --output build/probe-chat-markdown-material
|
|
// node tool/chat_markdown_probe.mjs --root build/probe-chat-markdown-material \
|
|
// --out DIR --style material
|
|
//
|
|
// The Material host is the same build with CHAT_MARKDOWN_PROBE_STYLE=material.
|
|
// Flutter web paints SelectableText on the canvas, so OCR checks the rendered
|
|
// prose while the semantics tree checks headings and the dark host.
|
|
import { createServer } from 'node:http';
|
|
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|
import { spawn, execFile } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { promisify } from 'node:util';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
|
|
const args = new Map();
|
|
for (let i = 2; i < process.argv.length; i++) {
|
|
const value = process.argv[i];
|
|
if (value.startsWith('--')) args.set(value.slice(2), process.argv[++i]);
|
|
}
|
|
const root = resolve(args.get('root') ?? 'build/probe-chat-markdown-material');
|
|
const out = resolve(args.get('out') ?? 'build/chat-markdown-probe');
|
|
const style = args.get('style') ?? 'material';
|
|
const chromePath =
|
|
process.env.CHROME_PATH ??
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
|
|
await mkdir(out, { recursive: true });
|
|
|
|
const mime = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.mjs': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
'.bin': 'application/octet-stream',
|
|
};
|
|
|
|
const server = createServer(async (req, res) => {
|
|
try {
|
|
const pathname = decodeURIComponent(
|
|
new URL(req.url, 'http://localhost').pathname,
|
|
);
|
|
let filePath = resolve(root, '.' + pathname);
|
|
if (filePath !== root && !filePath.startsWith(root + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
if (pathname === '/' || pathname.endsWith('/')) {
|
|
filePath = join(filePath, 'index.html');
|
|
}
|
|
const file = await readFile(filePath);
|
|
res
|
|
.writeHead(200, {
|
|
'content-type': mime[extname(filePath)] ?? 'application/octet-stream',
|
|
})
|
|
.end(file);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const origin = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'c2-markdown-chrome-'));
|
|
const chrome = spawn(
|
|
chromePath,
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
const waitFor = async (fn, what) => {
|
|
for (let i = 0; i < 150; i++) {
|
|
const value = await fn();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${what}: ${stderr.slice(-1500)}`);
|
|
};
|
|
|
|
let ws;
|
|
const evidence = { origin, style, captures: [], checks: [] };
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(
|
|
JSON.stringify({
|
|
id: callId,
|
|
method,
|
|
params,
|
|
...(sessionId ? { sessionId } : {}),
|
|
}),
|
|
);
|
|
});
|
|
|
|
const created = await send('Target.createTarget', { url: 'about:blank' });
|
|
const target = created.result.targetId;
|
|
const session = (
|
|
await send('Target.attachToTarget', { targetId: target, flatten: true })
|
|
).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width: 390, height: 844, deviceScaleFactor: 2, mobile: true },
|
|
session,
|
|
);
|
|
await send(
|
|
'Emulation.setEmulatedMedia',
|
|
{ features: [{ name: 'prefers-color-scheme', value: 'dark' }] },
|
|
session,
|
|
);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) {
|
|
throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
|
|
await send('Page.navigate', { url: `${origin}/` }, session);
|
|
await delay(3500);
|
|
const contentType = await evaluate(
|
|
"fetch('/').then(r => r.headers.get('content-type'))",
|
|
);
|
|
if (!String(contentType).toLowerCase().startsWith('text/html')) {
|
|
throw new Error(`Root index is not text/html: ${contentType}`);
|
|
}
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(1200);
|
|
|
|
const capture = async (name) => {
|
|
const data = (
|
|
await send('Page.captureScreenshot', { format: 'png' }, session)
|
|
).result.data;
|
|
const file = join(out, `${name}.png`);
|
|
await writeFile(file, Buffer.from(data, 'base64'));
|
|
evidence.captures.push(file);
|
|
return file;
|
|
};
|
|
const ocr = async (path) => {
|
|
const { stdout } = await promisify(execFile)('tesseract', [path, 'stdout']);
|
|
return stdout.replace(/\s+/g, ' ');
|
|
};
|
|
const requireOcr = (text, wanted) => {
|
|
if (!text.toLowerCase().includes(wanted.toLowerCase())) {
|
|
throw new Error(`OCR lacks ${JSON.stringify(wanted)}: ${text.slice(0, 2500)}`);
|
|
}
|
|
};
|
|
|
|
const shot = await capture(`${style}-markdown`);
|
|
const text = await ocr(shot);
|
|
for (const wanted of [
|
|
'Heading one',
|
|
'Heading two',
|
|
'Heading three',
|
|
'Heading four',
|
|
'Heading five',
|
|
'Heading six',
|
|
'paragraph with bold',
|
|
'italic',
|
|
'inline code',
|
|
'snake_case_name',
|
|
'quoted line one',
|
|
'bullet one',
|
|
'bullet two',
|
|
'first item',
|
|
'Name',
|
|
'Value',
|
|
'alpha',
|
|
'beta',
|
|
'dart',
|
|
'const answer = 42',
|
|
'Open docs',
|
|
'unsafe',
|
|
'example.com/auto',
|
|
'literal html',
|
|
'strikethrough',
|
|
]) {
|
|
requireOcr(text, wanted);
|
|
}
|
|
for (const absent of ['**', '```', '| ---', 'javascript:alert']) {
|
|
if (text.includes(absent)) {
|
|
throw new Error(`OCR leaked raw marker ${JSON.stringify(absent)}`);
|
|
}
|
|
}
|
|
|
|
const nodes = await evaluate(`[...document.querySelectorAll(
|
|
'flt-semantics, h1, h2, h3, h4, h5, h6')]
|
|
.map(x => ({ label: x.getAttribute('aria-label'), role: x.getAttribute('role'),
|
|
level: x.getAttribute('aria-level') ??
|
|
(/^H[1-6]$/.test(x.tagName) ? x.tagName.slice(1) : null),
|
|
text: (x.innerText || '').trim() }))
|
|
.filter(x => x.label || x.text || x.role || x.level)`);
|
|
const serialized = JSON.stringify(nodes);
|
|
for (const wanted of ['dart', '\u2022', '1.']) {
|
|
if (!serialized.includes(wanted)) {
|
|
throw new Error(`Semantics lacks ${JSON.stringify(wanted)}`);
|
|
}
|
|
}
|
|
const levels = new Set(nodes.map((node) => node.level).filter(Boolean));
|
|
for (const level of ['1', '2', '3', '4', '5', '6']) {
|
|
if (!levels.has(level)) {
|
|
throw new Error(`Semantics lacks heading level ${level}: ${serialized}`);
|
|
}
|
|
}
|
|
const roles = new Set(nodes.map((node) => node.role).filter(Boolean));
|
|
for (const role of ['list', 'listitem', 'columnheader']) {
|
|
if (!roles.has(role)) {
|
|
throw new Error(`Semantics lacks role ${role}: ${serialized}`);
|
|
}
|
|
}
|
|
evidence.checks.push({
|
|
check:
|
|
'OCR prose, unsafe label, no raw markup, code/list labels, heading levels 1-6, list and columnheader roles',
|
|
headings: [...levels].sort(),
|
|
roleSample: [...roles].filter((role) =>
|
|
['list', 'listitem', 'columnheader', 'link', 'table', 'row', 'cell'].includes(role),
|
|
),
|
|
nodes: nodes.length,
|
|
});
|
|
|
|
const report = JSON.stringify(evidence, null, 2);
|
|
await writeFile(join(out, `${style}-evidence.json`), report);
|
|
console.log(report);
|
|
} finally {
|
|
ws?.close();
|
|
chrome.kill();
|
|
server.close();
|
|
for (let attempt = 0; attempt < 10; attempt++) {
|
|
try {
|
|
await rm(profile, { recursive: true, force: true });
|
|
break;
|
|
} catch {
|
|
await delay(200);
|
|
}
|
|
}
|
|
}
|