b60ad6d98f
Keep the probe suite on the Material composition so browser fixtures and capture drivers no longer render or select Cupertino hosts. Record the final residual legacy arguments and implementation checks for the delivery gate.
542 lines
18 KiB
JavaScript
542 lines
18 KiB
JavaScript
import { spawn, execFile } from 'node:child_process';
|
|
import { promisify } from 'node:util';
|
|
import { mkdtemp, mkdir, rm, writeFile, readFile, realpath, stat } from 'node:fs/promises';
|
|
import { once } from 'node:events';
|
|
import { createServer } from 'node:http';
|
|
import { createHash } from 'node:crypto';
|
|
import { tmpdir } from 'node:os';
|
|
import { extname, join, resolve, sep } from 'node:path';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
|
|
// Real-browser acceptance for the run title wiring.
|
|
//
|
|
// Usage (from app/):
|
|
// flutter build web --target=tool/title_wiring_probe.dart -o build/title_probe_web
|
|
// OPENCODE_ARTIFACT_DIR=<dir> node tool/title_wiring_probe.mjs build/title_probe_web
|
|
//
|
|
// The fixture serves the real bundle over loopback and answers the same
|
|
// websocket protocol as the app server: one run, `get_run`, and
|
|
// `set_run_title`. It records the exact rename payload and returns the stored
|
|
// run detail. Requires Google Chrome and Tesseract (OCR checks cover the
|
|
// canvas-rendered titles).
|
|
const output = process.env.OPENCODE_ARTIFACT_DIR;
|
|
if (!output) throw new Error('Set OPENCODE_ARTIFACT_DIR for the probe captures');
|
|
await mkdir(output, { recursive: true });
|
|
const root = await realpath(process.argv[2] ?? 'build/title_probe_web');
|
|
|
|
const RUN = {
|
|
id: 1,
|
|
title: 'Fixture live run',
|
|
name: 'fixture-live',
|
|
status: 'running',
|
|
created_at: '2026-10-21T12:00:00Z',
|
|
};
|
|
let storedTitle = RUN.title;
|
|
const renames = [];
|
|
const counters = { listRuns: 0, getRun: 0 };
|
|
|
|
function runRow() {
|
|
return { ...RUN, title: storedTitle };
|
|
}
|
|
|
|
function runDetail() {
|
|
return {
|
|
...runRow(),
|
|
input: { prompt: ' fixture prompt with spaces ' },
|
|
output: { answer: 42 },
|
|
steps: [
|
|
{ stage: 'build', sequence: 3, status: 'completed', outcome: 'success' },
|
|
],
|
|
};
|
|
}
|
|
|
|
const MIME = {
|
|
'.html': 'text/html; charset=utf-8',
|
|
'.js': 'text/javascript; charset=utf-8',
|
|
'.mjs': 'text/javascript; charset=utf-8',
|
|
'.json': 'application/json; charset=utf-8',
|
|
'.wasm': 'application/wasm',
|
|
'.css': 'text/css; charset=utf-8',
|
|
'.png': 'image/png',
|
|
'.svg': 'image/svg+xml',
|
|
'.ico': 'image/x-icon',
|
|
'.ttf': 'font/ttf',
|
|
'.otf': 'font/otf',
|
|
'.woff': 'font/woff',
|
|
'.woff2': 'font/woff2',
|
|
};
|
|
|
|
function accept(key) {
|
|
return createHash('sha1')
|
|
.update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`)
|
|
.digest('base64');
|
|
}
|
|
|
|
function frame(opcode, payload) {
|
|
const length = payload.length;
|
|
let header;
|
|
if (length < 126) {
|
|
header = Buffer.from([0x80 | opcode, length]);
|
|
} else if (length < 65536) {
|
|
header = Buffer.alloc(4);
|
|
header[0] = 0x80 | opcode;
|
|
header[1] = 126;
|
|
header.writeUInt16BE(length, 2);
|
|
} else {
|
|
header = Buffer.alloc(10);
|
|
header[0] = 0x80 | opcode;
|
|
header[1] = 127;
|
|
header.writeBigUInt64BE(BigInt(length), 2);
|
|
}
|
|
return Buffer.concat([header, payload]);
|
|
}
|
|
|
|
class Connection {
|
|
constructor(socket) {
|
|
this.socket = socket;
|
|
this.buffer = Buffer.alloc(0);
|
|
}
|
|
|
|
push(chunk) {
|
|
if (chunk?.length) this.buffer = Buffer.concat([this.buffer, chunk]);
|
|
while (this.buffer.length >= 2) {
|
|
const first = this.buffer[0];
|
|
const second = this.buffer[1];
|
|
const opcode = first & 0x0f;
|
|
let length = second & 0x7f;
|
|
let offset = 2;
|
|
if (length === 126) {
|
|
if (this.buffer.length < 4) return;
|
|
length = this.buffer.readUInt16BE(2);
|
|
offset = 4;
|
|
} else if (length === 127) {
|
|
if (this.buffer.length < 10) return;
|
|
length = Number(this.buffer.readBigUInt64BE(2));
|
|
offset = 10;
|
|
}
|
|
const masked = (second & 0x80) !== 0;
|
|
const maskOffset = offset;
|
|
if (masked) {
|
|
if (this.buffer.length < offset + 4) return;
|
|
offset += 4;
|
|
}
|
|
if (this.buffer.length < offset + length) return;
|
|
let payload = Buffer.from(this.buffer.subarray(offset, offset + length));
|
|
if (masked) {
|
|
const mask = this.buffer.subarray(maskOffset, maskOffset + 4);
|
|
for (let index = 0; index < payload.length; index++) {
|
|
payload[index] ^= mask[index % 4];
|
|
}
|
|
}
|
|
this.buffer = this.buffer.subarray(offset + length);
|
|
if (opcode === 0x8) return this.close();
|
|
if (opcode === 0x9) {
|
|
this.socket.write(frame(0x0a, payload));
|
|
continue;
|
|
}
|
|
if (opcode !== 0x1) continue;
|
|
let request;
|
|
try {
|
|
request = JSON.parse(payload.toString('utf8'));
|
|
} catch {
|
|
continue;
|
|
}
|
|
this.message(request);
|
|
}
|
|
}
|
|
|
|
message(request) {
|
|
if (request.type === 'subscribe' || request.type === 'unsubscribe') return;
|
|
if (request.type === 'ping') {
|
|
this.send({ id: request.id, type: 'pong', build: { version: 'Title fixture' } });
|
|
return;
|
|
}
|
|
if (request.type === 'list_runs') {
|
|
counters.listRuns++;
|
|
this.send({ id: request.id, type: 'runs', runs: [runRow()], usage: null });
|
|
return;
|
|
}
|
|
if (request.type === 'get_run') {
|
|
counters.getRun++;
|
|
this.send({ id: request.id, type: 'run', run_detail: runDetail() });
|
|
return;
|
|
}
|
|
if (request.type === 'set_run_title') {
|
|
renames.push({ run_id: request.run_id, title: request.title });
|
|
storedTitle = request.title;
|
|
this.send({ id: request.id, type: 'run', run_detail: runDetail() });
|
|
return;
|
|
}
|
|
this.send({
|
|
id: request.id,
|
|
type: 'error',
|
|
error: `Fixture does not implement ${request.type}`,
|
|
});
|
|
}
|
|
|
|
send(value) {
|
|
if (!this.socket.destroyed) {
|
|
this.socket.write(frame(0x1, Buffer.from(JSON.stringify(value))));
|
|
}
|
|
}
|
|
|
|
close() {
|
|
if (!this.socket.destroyed) this.socket.end();
|
|
}
|
|
}
|
|
|
|
async function staticFile(base, pathname, res) {
|
|
let file = resolve(base, `.${pathname}`);
|
|
if (file !== base && !file.startsWith(base + sep)) {
|
|
res.writeHead(403).end();
|
|
return;
|
|
}
|
|
try {
|
|
if ((await stat(file)).isDirectory()) file = join(file, 'index.html');
|
|
} catch {
|
|
if (extname(pathname)) {
|
|
res.writeHead(404).end('not found');
|
|
return;
|
|
}
|
|
file = join(base, 'index.html');
|
|
}
|
|
try {
|
|
const body = await readFile(file);
|
|
res
|
|
.writeHead(200, {
|
|
'content-type': MIME[extname(file)] ?? 'application/octet-stream',
|
|
'cache-control': 'no-store',
|
|
})
|
|
.end(body);
|
|
} catch {
|
|
res.writeHead(404).end('not found');
|
|
}
|
|
}
|
|
|
|
const sockets = new Set();
|
|
const server = createServer(async (req, res) => {
|
|
const pathname = new URL(req.url, 'http://fixture').pathname;
|
|
if (pathname === '/fixture/state') {
|
|
res
|
|
.writeHead(200, { 'content-type': 'application/json' })
|
|
.end(JSON.stringify({ storedTitle, renames, ...counters }));
|
|
return;
|
|
}
|
|
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
|
res.writeHead(405).end();
|
|
return;
|
|
}
|
|
await staticFile(root, pathname, res);
|
|
});
|
|
server.on('upgrade', (req, socket, head) => {
|
|
const url = new URL(req.url, 'http://fixture');
|
|
if (
|
|
url.pathname !== '/connect' ||
|
|
req.headers.upgrade?.toLowerCase() !== 'websocket' ||
|
|
!req.headers['sec-websocket-key']
|
|
) {
|
|
socket.end('HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n');
|
|
return;
|
|
}
|
|
socket.write(
|
|
'HTTP/1.1 101 Switching Protocols\r\n' +
|
|
'Upgrade: websocket\r\n' +
|
|
'Connection: Upgrade\r\n' +
|
|
`Sec-WebSocket-Accept: ${accept(req.headers['sec-websocket-key'])}\r\n\r\n`,
|
|
);
|
|
const connection = new Connection(socket);
|
|
socket.on('data', (chunk) => connection.push(chunk));
|
|
socket.on('error', () => socket.destroy());
|
|
sockets.add(socket);
|
|
socket.on('close', () => sockets.delete(socket));
|
|
connection.push(head);
|
|
});
|
|
server.listen(0, '127.0.0.1');
|
|
await once(server, 'listening');
|
|
const fixtureUrl = `http://127.0.0.1:${server.address().port}`;
|
|
|
|
const profile = await mkdtemp(join(tmpdir(), 'r2-title-chrome-'));
|
|
const chrome = spawn(
|
|
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
[
|
|
'--headless=new',
|
|
'--no-first-run',
|
|
'--no-default-browser-check',
|
|
'--disable-gpu',
|
|
'--remote-debugging-port=0',
|
|
`--user-data-dir=${profile}`,
|
|
'about:blank',
|
|
],
|
|
{ stdio: ['ignore', 'ignore', 'pipe'] },
|
|
);
|
|
let stderr = '';
|
|
chrome.stderr.on('data', (data) => {
|
|
stderr += data;
|
|
});
|
|
|
|
async function waitFor(check, label, attempts = 160) {
|
|
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
const value = await check();
|
|
if (value) return value;
|
|
await delay(100);
|
|
}
|
|
throw new Error(`Timed out waiting for ${label}\n${stderr.slice(-1200)}`);
|
|
}
|
|
|
|
let ws;
|
|
try {
|
|
const endpoint = await waitFor(
|
|
() => stderr.match(/DevTools listening on (ws:\/\/[^\s]+)/)?.[1],
|
|
'Chrome DevTools',
|
|
);
|
|
ws = new WebSocket(endpoint);
|
|
await once(ws, 'open');
|
|
let id = 0;
|
|
const pending = new Map();
|
|
ws.addEventListener('message', (event) => {
|
|
const message = JSON.parse(event.data);
|
|
if (message.method === 'Runtime.exceptionThrown') {
|
|
console.error('PAGE EXCEPTION', JSON.stringify(message.params?.exceptionDetails));
|
|
}
|
|
if (message.id && pending.has(message.id)) {
|
|
pending.get(message.id)(message);
|
|
pending.delete(message.id);
|
|
}
|
|
});
|
|
const send = (method, params = {}, sessionId) =>
|
|
new Promise((resolveCall) => {
|
|
const callId = ++id;
|
|
pending.set(callId, resolveCall);
|
|
ws.send(JSON.stringify({ id: callId, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
});
|
|
|
|
const target = (await send('Target.createTarget', { url: 'about:blank' })).result.targetId;
|
|
const session = (await send('Target.attachToTarget', { targetId: target, flatten: true })).result.sessionId;
|
|
await send('Page.enable', {}, session);
|
|
await send('Runtime.enable', {}, session);
|
|
|
|
const evaluate = async (expression) => {
|
|
const response = await send(
|
|
'Runtime.evaluate',
|
|
{ expression, returnByValue: true, awaitPromise: true },
|
|
session,
|
|
);
|
|
if (response.error) throw new Error(`Runtime.evaluate failed: ${JSON.stringify(response)}`);
|
|
if (response.result?.exceptionDetails) {
|
|
throw new Error(`Page exception: ${JSON.stringify(response.result.exceptionDetails)}`);
|
|
}
|
|
return response.result?.result?.value;
|
|
};
|
|
|
|
const enableSemantics = async () => {
|
|
await waitFor(
|
|
() => evaluate("Boolean(document.querySelector('flt-semantics-placeholder'))"),
|
|
'Flutter semantics placeholder',
|
|
);
|
|
await evaluate("document.querySelector('flt-semantics-placeholder')?.click()");
|
|
await delay(700);
|
|
};
|
|
|
|
const semanticText = () =>
|
|
evaluate(`(() => {
|
|
const chunks = [];
|
|
for (const el of document.querySelectorAll('flt-semantics')) {
|
|
const label = el.getAttribute('aria-label');
|
|
if (label) chunks.push(label);
|
|
const value = el.getAttribute('aria-valuetext');
|
|
if (value) chunks.push(value);
|
|
if (el.value) chunks.push(String(el.value));
|
|
if (!el.children.length && el.innerText) chunks.push(el.innerText);
|
|
}
|
|
for (const el of document.querySelectorAll('input, textarea')) {
|
|
const label = el.getAttribute('aria-label');
|
|
if (label) chunks.push(label);
|
|
if (el.value) chunks.push(String(el.value));
|
|
}
|
|
if (document.body?.innerText) chunks.push(document.body.innerText);
|
|
return [...new Set(chunks)].join('\\n');
|
|
})()`);
|
|
|
|
const assertBody = async (text) => {
|
|
try {
|
|
await waitFor(
|
|
async () => (await semanticText()).includes(text),
|
|
`body to contain ${JSON.stringify(text)}`,
|
|
);
|
|
} catch (error) {
|
|
throw new Error(`${error.message}\nBody now:\n${(await semanticText()).slice(0, 3000)}`);
|
|
}
|
|
};
|
|
|
|
const clickText = async (text) => {
|
|
const clicked = await evaluate(`(() => {
|
|
const wanted = ${JSON.stringify(text)};
|
|
const candidates = [...document.querySelectorAll('flt-semantics')].filter((el) => {
|
|
const label = (el.getAttribute('aria-label') || el.innerText || '').trim();
|
|
return label === wanted;
|
|
});
|
|
const target = candidates.find((el) =>
|
|
['button', 'tab', 'link', 'checkbox', 'menuitem'].includes(el.getAttribute('role')),
|
|
) ?? candidates[0];
|
|
if (!target) return false;
|
|
target.click();
|
|
return true;
|
|
})()`);
|
|
if (!clicked) {
|
|
const labels = await evaluate(`[...document.querySelectorAll('flt-semantics')]
|
|
.map((el) => (el.getAttribute('aria-label') || el.innerText || '').trim()).filter(Boolean)`);
|
|
throw new Error(`Could not click ${JSON.stringify(text)}. Labels: ${JSON.stringify(labels)}`);
|
|
}
|
|
await delay(400);
|
|
};
|
|
|
|
const capture = async (name) => {
|
|
const data = (await send('Page.captureScreenshot', { format: 'png' }, session)).result.data;
|
|
const path = join(output, `${name}.png`);
|
|
await writeFile(path, Buffer.from(data, 'base64'));
|
|
return path;
|
|
};
|
|
|
|
const ocr = async (path) =>
|
|
(await promisify(execFile)('tesseract', [path, 'stdout'])).stdout;
|
|
|
|
const assertOcr = async (path, text) => {
|
|
const read = await ocr(path);
|
|
if (!read.toLowerCase().includes(text.toLowerCase())) {
|
|
throw new Error(`OCR of ${path} lacks ${JSON.stringify(text)}:\n${read.slice(0, 2000)}`);
|
|
}
|
|
};
|
|
|
|
const state = async () => (await (await fetch(`${fixtureUrl}/fixture/state`)).json());
|
|
|
|
const setSize = async (width, height) => {
|
|
await send(
|
|
'Emulation.setDeviceMetricsOverride',
|
|
{ width, height, deviceScaleFactor: 1, mobile: width < 600 },
|
|
session,
|
|
);
|
|
await delay(400);
|
|
};
|
|
|
|
const dialogOpen = async () => (await semanticText()).includes('Save');
|
|
|
|
const focusRenameField = async () => {
|
|
const focused = await evaluate(`(() => {
|
|
const node = document.querySelector('input.flt-text-editing, flt-semantics input, input, textarea');
|
|
if (!node) return false;
|
|
node.focus();
|
|
node.select?.();
|
|
node.setSelectionRange?.(0, (node.value || '').length);
|
|
return true;
|
|
})()`);
|
|
if (!focused) throw new Error('The rename field was not exposed as a semantics input');
|
|
await delay(350);
|
|
return evaluate(`(() => {
|
|
const node = document.querySelector('input.flt-text-editing, flt-semantics input, input, textarea');
|
|
return node ? { value: node.value, label: node.getAttribute('aria-label'), placeholder: node.getAttribute('placeholder') } : null;
|
|
})()`);
|
|
};
|
|
|
|
const rename = async (value, { initial } = {}) => {
|
|
await assertBody('Rename');
|
|
await clickText('Rename');
|
|
await waitFor(dialogOpen, 'the rename prompt');
|
|
const field = await focusRenameField();
|
|
if (initial != null && field?.value !== initial) {
|
|
throw new Error(`Rename field seeded with ${JSON.stringify(field)}, expected ${JSON.stringify(initial)}`);
|
|
}
|
|
await send('Input.insertText', { text: value }, session);
|
|
await delay(400);
|
|
const typed = await evaluate(`(() => {
|
|
const node = document.querySelector('input.flt-text-editing, flt-semantics input, input, textarea');
|
|
return node ? node.value : null;
|
|
})()`);
|
|
if (typed !== value) {
|
|
throw new Error(`Rename field holds ${JSON.stringify(typed)}, expected ${JSON.stringify(value)}`);
|
|
}
|
|
await clickText('Save');
|
|
await waitFor(
|
|
async () => {
|
|
const text = await semanticText();
|
|
return text.includes(value) && !text.includes('Save');
|
|
},
|
|
'the saved title from the server',
|
|
);
|
|
const now = await state();
|
|
const last = now.renames.at(-1);
|
|
if (last?.title !== value || last?.run_id !== 1) {
|
|
throw new Error(`Server did not receive the exact rename: ${JSON.stringify(last)}`);
|
|
}
|
|
return { ...now, field };
|
|
};
|
|
|
|
const captures = [];
|
|
const checks = [];
|
|
|
|
await setSize(390, 844);
|
|
const open = async (expectedTitle) => {
|
|
const navigation = await send('Page.navigate', { url: `${fixtureUrl}/runs/1` }, session);
|
|
if (navigation.error || navigation.result?.errorText) throw new Error(JSON.stringify(navigation));
|
|
await enableSemantics();
|
|
await assertBody(expectedTitle);
|
|
if ((await evaluate('location.pathname')) !== '/runs/1') {
|
|
throw new Error(`Deep link moved to ${await evaluate('location.pathname')}`);
|
|
}
|
|
};
|
|
|
|
// Material host (macOS browser): Cancel sends nothing, then Save applies the
|
|
// exact server reply to the detail and the list row.
|
|
await open('Fixture live run');
|
|
await assertBody('Original prompt');
|
|
captures.push(await capture('r2-title-material-detail'));
|
|
await assertOcr(captures[0], 'Fixture live run');
|
|
|
|
await clickText('Rename');
|
|
await waitFor(dialogOpen, 'the rename prompt');
|
|
const cancelField = await focusRenameField();
|
|
if (cancelField?.value !== 'Fixture live run') {
|
|
throw new Error(`Rename field seeded with ${JSON.stringify(cancelField)}`);
|
|
}
|
|
await send('Input.insertText', { text: 'Discarded title' }, session);
|
|
await delay(300);
|
|
captures.push(await capture('r2-title-material-dialog'));
|
|
await clickText('Cancel');
|
|
if ((await state()).renames.length !== 0) {
|
|
throw new Error('Cancel sent set_run_title');
|
|
}
|
|
await assertBody('Fixture live run');
|
|
await waitFor(async () => !(await dialogOpen()), 'the closed rename prompt');
|
|
checks.push({ check: 'Cancel sent nothing and kept the stored title', sent: 0 });
|
|
captures.push(await capture('r2-title-material-cancelled'));
|
|
|
|
const materialState = await rename('Material renamed run', {
|
|
initial: 'Fixture live run',
|
|
});
|
|
captures.push(await capture('r2-title-material-renamed'));
|
|
await assertOcr(captures.at(-1), 'Material renamed run');
|
|
checks.push({
|
|
check: 'Material Save sent the exact payload',
|
|
rename: materialState.renames.at(-1),
|
|
field: materialState.field,
|
|
});
|
|
|
|
await clickText('Back');
|
|
await assertBody('Material renamed run');
|
|
captures.push(await capture('r2-title-material-list'));
|
|
await assertOcr(captures.at(-1), 'Material renamed run');
|
|
checks.push({ check: 'The list row shows the server title', path: await evaluate('location.pathname') });
|
|
|
|
const finalState = await state();
|
|
if (finalState.renames.length !== 1) {
|
|
throw new Error(`Expected one rename, saw ${JSON.stringify(finalState.renames)}`);
|
|
}
|
|
console.log(JSON.stringify({ ok: true, fixture: fixtureUrl, checks, captures }, null, 2));
|
|
} finally {
|
|
ws?.close();
|
|
for (const socket of sockets) socket.destroy();
|
|
await new Promise((resolveClose) => server.close(resolveClose));
|
|
chrome.kill('SIGTERM');
|
|
await Promise.race([once(chrome, 'exit'), delay(3000)]);
|
|
await rm(profile, { recursive: true, force: true });
|
|
}
|